Skip to content

Commit

Permalink
Added new critical error type (#401)
Browse files Browse the repository at this point in the history
  • Loading branch information
@JulianForeman
JulianForeman authored Oct 23, 2024
1 parent c4ff846 commit d98e374
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 27 deletions.
42 changes: 29 additions & 13 deletions django/api/services/spreadsheet_uploader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ def extract_data(excel_file, sheet_name, header_row):
df = trim_all_columns(df)
return df
except Exception as e:
traceback.print_exc()
raise
return None


def transform_data(
Expand All @@ -50,9 +49,17 @@ def transform_data(

df = df[[col for col in df.columns if col in required_columns]]

errors_and_warnings = {}

missing_columns = [col for col in required_columns if col not in df.columns]
if (missing_columns):
raise ValueError(f"Missing columns: {', '.join(missing_columns)}")
errors_and_warnings['Headers'] = {}
errors_and_warnings['Headers']['Missing Headers'] = {
"Expected Type": "The spreadsheet provided is missing headers",
"Rows": missing_columns,
"Severity": "Critical"
}
return df, errors_and_warnings

for prep_func in preparation_functions:
df = prep_func(df)
Expand All @@ -69,7 +76,6 @@ def transform_data(
datetime: "Date (YYYY-MM-DD)"
}

errors_and_warnings = {}
df = df.replace({np.nan: None})

for index, row in df.iterrows():
Expand Down Expand Up @@ -211,17 +217,27 @@ def import_from_xls(
validation_functions=[],
check_for_warnings=True,
):
errors_and_warnings = {}
try:
df = extract_data(excel_file, sheet_name, header_row)
df, errors_and_warnings = transform_data(
df,
dataset_columns,
column_mapping_enum,
field_types,
model,
preparation_functions,
validation_functions,
)
if df is not None:
df, errors_and_warnings = transform_data(
df,
dataset_columns,
column_mapping_enum,
field_types,
model,
preparation_functions,
validation_functions,
)

else:
errors_and_warnings['Spreadsheet'] = {}
errors_and_warnings['Spreadsheet']['Missing Worksheet'] = {
'Expected Type': 'The worksheet is missing or incorrectly named',
'Rows': [sheet_name],
'Severity': 'Critical'
}

if check_for_warnings:
## do the error checking
Expand Down
45 changes: 39 additions & 6 deletions frontend/src/uploads/UploadContainer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const UploadContainer = () => {
const [openDialog, setOpenDialog] = useState(false);
const [adminUser, setAdminUser] = useState(false);
const [totalIssueCount, setTotalIssueCount] = useState({});
const [groupedCriticalErrors, setGroupedCriticalErrors] = useState({});
const [groupedErrors, setGroupedErrors] = useState({});
const [groupedWarnings, setGroupedWarnings] = useState({});
const [alertDialogText, setAlertDialogText] = useState({
Expand Down Expand Up @@ -58,25 +59,45 @@ const UploadContainer = () => {
};

const groupAndCountRows = (issueArray) => {
const groupedCriticalErrors = {}
const groupedErrors = {};
const groupedWarnings = {};
const totalIssueCount = {
criticalErrors: 0,
errors: 0,
warnings: 0,
};


issueArray.forEach((issue) => {
Object.keys(issue).forEach((column) => {
const errorDetails = issue[column];


Object.keys(errorDetails).forEach((errorType) => {

const severity = errorDetails[errorType].Severity;
const expectedType = errorDetails[errorType]["Expected Type"];
const groups = errorDetails[errorType].Groups || [];
if (severity === "Error") {

if (severity === "Critical") {
const rows = errorDetails[errorType].Rows;
const rowCount = rows.length;
totalIssueCount.criticalErrors += rowCount;
if (!groupedCriticalErrors[column]) {
groupedCriticalErrors[column] = {};
}
if (!groupedCriticalErrors[column][errorType]) {
groupedCriticalErrors[column][errorType] = {
ExpectedType: expectedType,
Rows: rows,
};
} else {
groupedCriticalErrors[column][errorType].Rows.push(...rows);
}
} else if (severity === "Error") {
const rows = errorDetails[errorType].Rows || null;
const rowCount = rows.length || groups.length;
totalIssueCount.errors += rowCount;

if (!groupedErrors[column]) {
Expand Down Expand Up @@ -113,8 +134,9 @@ const UploadContainer = () => {
});
});
});


return { groupedErrors, groupedWarnings, totalIssueCount };
return { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount };
};

const showError = (error) => {
Expand Down Expand Up @@ -181,17 +203,27 @@ const UploadContainer = () => {
});

if (Object.keys(warnings).length > 0 && checkForWarnings) {
const { groupedErrors, groupedWarnings, totalIssueCount } =
const { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount } =
groupAndCountRows(Object.values(warnings));

setGroupedCriticalErrors(groupedCriticalErrors)
setGroupedErrors(groupedErrors);
setGroupedWarnings(groupedWarnings);
setTotalIssueCount(totalIssueCount);

setAlertDialogText({
title:
"Your file has been processed and contains the following errors and warnings!",
totalIssueCount.criticalErrors > 0 ? "Your upload contained critical errors that must be fixed before it can be processed!" : "Your file has been processed and contains the following errors and warnings!",
content: (
<>
{totalIssueCount.criticalErrors >= 1 && (
<div>
<span className="error">
{totalIssueCount.criticalErrors} Critical Errors
</span>
- Must fix before file can be processed
</div>
)}
{totalIssueCount.errors >= 1 && (
<div>
<span className="error">
Expand Down Expand Up @@ -323,10 +355,11 @@ const UploadContainer = () => {
handleConfirm={alertDialogText.confirmAction}
/>
<Stack direction="column" spacing={2}>
{(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && (
{(totalIssueCount.criticalErrors || totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && (
<Paper variant="outlined" square elevation={0} sx={{ mb: 2 }}>
<UploadIssues
confirmUpload={handleConfirmDataInsert}
groupedCriticalErrors={groupedCriticalErrors}
groupedErrors={groupedErrors}
groupedWarnings={groupedWarnings}
totalIssueCount={totalIssueCount}
Expand Down
21 changes: 19 additions & 2 deletions frontend/src/uploads/components/UploadIssues.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import ExpandMoreIcon from "@mui/icons-material/ExpandMore";

const UploadIssues = ({
confirmUpload,
groupedCriticalErrors,
groupedErrors,
groupedWarnings,
totalIssueCount,
Expand All @@ -23,6 +24,7 @@ const UploadIssues = ({
setShowAllIssues(!showAllIssues);
};

const criticalMsg = "Must fix before file can be processed";
const errorMsg = "Must fix before uploading";
const warningMsg = "Can upload without fixing";

Expand All @@ -37,9 +39,16 @@ const UploadIssues = ({
Your file upload results
</h2>
<Box sx={{ ml: "2rem", mb: "1rem" }}>
Your file has been processed and contains the following errors and
warnings. Please review them below:
{totalIssueCount.criticalErrors > 0 ? 'Your file cannot be processed because it contains critical errors. Please review them below.': 'Your file has been processed and contains the following errors and warnings. Please review them below'}
</Box>
{totalIssueCount.criticalErrors >= 1 && (
<Box ml={2} mt={2}>
<span className="error">
<strong>{totalIssueCount.criticalErrors} Critical Errors &nbsp;</strong>
</span>
- {criticalMsg}
</Box>
)}
{totalIssueCount.errors >= 1 && (
<Box ml={2} mt={2}>
<span className="error">
Expand Down Expand Up @@ -72,6 +81,14 @@ const UploadIssues = ({
<ExpandMoreIcon />
</AccordionSummary>
<AccordionDetails>
{totalIssueCount.criticalErrors >= 1 && (
<UploadIssuesDetail
type="critical"
issues={groupedCriticalErrors}
totalIssueCount={totalIssueCount.criticalErrors}
msg={criticalMsg}
/>
)}
{totalIssueCount.errors >= 1 && (
<UploadIssuesDetail
type="error"
Expand Down
51 changes: 45 additions & 6 deletions frontend/src/uploads/components/UploadIssuesDetail.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ import ErrorOutlineIcon from "@mui/icons-material/ErrorOutline";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";

const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
const [showAllRowsMap, setShowAllRowsMap] = useState({});
const classname = type === "error" ? "error" : "warning";
const toggleShowAllRows = (column, errorType, groupIndex) => {
const key = `${column}_${errorType}_${groupIndex}`;
const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue
const errorTypes = ['critical', 'error']
const classname = errorTypes.includes(type) ? "error" : "warning";
const toggleShowAllRows = (column, errorType) => {
const key = `${column}_${errorType}`;
setShowAllRowsMap((prevState) => ({
...prevState,
[key]: !prevState[key],
Expand Down Expand Up @@ -56,7 +57,7 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
<Box
p={2}
sx={{
border: type === "error" ? "1px solid #ce3e39" : "1px solid #fcba19",
border: errorTypes.includes(type) ? "1px solid #ce3e39" : "1px solid #fcba19",
mb: "1rem",
}}
>
Expand All @@ -66,13 +67,51 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
/>
<span className={classname}>
<strong>
{totalIssueCount} {type === "error" ? "Errors" : "Warnings"}&nbsp;
{totalIssueCount} {type === 'critical' ? 'Critical Errors' : type === 'error' ? 'Errors' : 'Warnings'}&nbsp;
</strong>
</span>
({msg})
{Object.keys(issues).map((column) => (
<Box key={column} sx={{ marginTop: 2 }}>
<strong>Column: {column}</strong>
{Object.keys(issues[column]).map((errorType, index) => (
<div key={index} style={{ marginTop: "0.5rem" }}>
<div>
{(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')}
{type.charAt(0).toUpperCase() + type.slice(1)} {type === `critical` ? `Error` : `Name:`} <b>{errorType}</b>
</div>
<div>
Expected value:{" "}
<b>
{issues[column][errorType].ExpectedType ||
issues[column][errorType].ExpectedFormat}
</b>
</div>
<div>
{column === 'Headers' ? `Missing Headers: ` : column === 'Spreadsheet' ? 'Missing Spreadsheet: ' : `Rows with ${type}: `}
<b>
{issues[column][errorType]?.Rows?.slice(
0,
showAllRowsMap[`${column}_${errorType}`] ? undefined : 15,
).join(", ")}
{issues[column][errorType]?.Rows?.length > 15 &&
!showAllRowsMap[`${column}_${errorType}`] &&
"..."}
</b>
</div>
{issues[column][errorType]?.Rows?.length > 15 && (
<Button
variant="text"
onClick={() => toggleShowAllRows(column, errorType)}
>
{showAllRowsMap[`${column}_${errorType}`]
? "Show less"
: "Show more"}{" "}
<ExpandMoreIcon />
</Button>
)}
</div>
))}
{Object.keys(issues[column]).map((errorType, index) => {
const errorDetails = issues[column][errorType];

Expand Down

0 comments on commit d98e374

Please sign in to comment.