From d98e374153805424fb57d300ce11f0e72b63616a Mon Sep 17 00:00:00 2001
From: JulianForeman <71847719+JulianForeman@users.noreply.github.com>
Date: Wed, 23 Oct 2024 10:45:54 -0700
Subject: [PATCH] Added new critical error type (#401)

---
 django/api/services/spreadsheet_uploader.py   | 42 ++++++++++-----
 frontend/src/uploads/UploadContainer.js       | 45 +++++++++++++---
 .../src/uploads/components/UploadIssues.js    | 21 +++++++-
 .../uploads/components/UploadIssuesDetail.js  | 51 ++++++++++++++++---
 4 files changed, 132 insertions(+), 27 deletions(-)

diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py
index cbbd32ef..e6dd2285 100644
--- a/django/api/services/spreadsheet_uploader.py
+++ b/django/api/services/spreadsheet_uploader.py
@@ -33,8 +33,7 @@ def extract_data(excel_file, sheet_name, header_row):
         df = trim_all_columns(df)
         return df
     except Exception as e:
-        traceback.print_exc()
-        raise
+        return None
 
 
 def transform_data(
@@ -50,9 +49,17 @@ def transform_data(
 
     df = df[[col for col in df.columns if col in required_columns]]
 
+    errors_and_warnings = {}
+
     missing_columns = [col for col in required_columns if col not in df.columns]
     if (missing_columns):
-        raise ValueError(f"Missing columns: {', '.join(missing_columns)}")
+        errors_and_warnings['Headers'] = {}
+        errors_and_warnings['Headers']['Missing Headers'] = {
+            "Expected Type": "The spreadsheet provided is missing headers",
+            "Rows": missing_columns,
+            "Severity": "Critical"
+        }
+        return df, errors_and_warnings
 
     for prep_func in preparation_functions:
         df = prep_func(df)
@@ -69,7 +76,6 @@ def transform_data(
         datetime: "Date (YYYY-MM-DD)"
     }
 
-    errors_and_warnings = {}
     df = df.replace({np.nan: None})
 
     for index, row in df.iterrows():
@@ -211,17 +217,27 @@ def import_from_xls(
     validation_functions=[],
     check_for_warnings=True,
 ):
+    errors_and_warnings = {}
     try:
         df = extract_data(excel_file, sheet_name, header_row)
-        df, errors_and_warnings = transform_data(
-            df,
-            dataset_columns,
-            column_mapping_enum,
-            field_types,
-            model,
-            preparation_functions,
-            validation_functions,
-        )
+        if df is not None:
+            df, errors_and_warnings = transform_data(
+                df,
+                dataset_columns,
+                column_mapping_enum,
+                field_types,
+                model,
+                preparation_functions,
+                validation_functions,
+            )
+        
+        else:
+            errors_and_warnings['Spreadsheet'] = {}
+            errors_and_warnings['Spreadsheet']['Missing Worksheet'] = {
+                'Expected Type': 'The worksheet is missing or incorrectly named',
+                'Rows': [sheet_name],
+                'Severity': 'Critical'
+            }
 
         if check_for_warnings:
             ## do the error checking
diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js
index 53446c2e..5ed58c1c 100644
--- a/frontend/src/uploads/UploadContainer.js
+++ b/frontend/src/uploads/UploadContainer.js
@@ -25,6 +25,7 @@ const UploadContainer = () => {
   const [openDialog, setOpenDialog] = useState(false);
   const [adminUser, setAdminUser] = useState(false);
   const [totalIssueCount, setTotalIssueCount] = useState({});
+  const [groupedCriticalErrors, setGroupedCriticalErrors] = useState({});
   const [groupedErrors, setGroupedErrors] = useState({});
   const [groupedWarnings, setGroupedWarnings] = useState({});
   const [alertDialogText, setAlertDialogText] = useState({
@@ -58,25 +59,45 @@ const UploadContainer = () => {
   };
 
   const groupAndCountRows = (issueArray) => {
+    const groupedCriticalErrors = {}
     const groupedErrors = {};
     const groupedWarnings = {};
     const totalIssueCount = {
+      criticalErrors: 0,
       errors: 0,
       warnings: 0,
     };
 
+
     issueArray.forEach((issue) => {
       Object.keys(issue).forEach((column) => {
         const errorDetails = issue[column];
+
   
         Object.keys(errorDetails).forEach((errorType) => {
+
           const severity = errorDetails[errorType].Severity;
           const expectedType = errorDetails[errorType]["Expected Type"];
           const groups = errorDetails[errorType].Groups || [];
-  
-          if (severity === "Error") {
+
+          if (severity === "Critical") {
             const rows = errorDetails[errorType].Rows;
             const rowCount = rows.length;
+            totalIssueCount.criticalErrors += rowCount;
+            if (!groupedCriticalErrors[column]) {
+              groupedCriticalErrors[column] = {};
+            }
+            if (!groupedCriticalErrors[column][errorType]) {
+              groupedCriticalErrors[column][errorType] = {
+                ExpectedType: expectedType,
+                Rows: rows,
+              };
+            } else {
+              groupedCriticalErrors[column][errorType].Rows.push(...rows);
+            }
+          } else if (severity === "Error") {
+            const rows = errorDetails[errorType].Rows || null;
+            const rowCount = rows.length || groups.length;
             totalIssueCount.errors += rowCount;
   
             if (!groupedErrors[column]) {
@@ -113,8 +134,9 @@ const UploadContainer = () => {
         });
       });
     });
+
   
-    return { groupedErrors, groupedWarnings, totalIssueCount };
+    return { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount };
   };
 
   const showError = (error) => {
@@ -181,17 +203,27 @@ const UploadContainer = () => {
         });
 
         if (Object.keys(warnings).length > 0 && checkForWarnings) {
-          const { groupedErrors, groupedWarnings, totalIssueCount } =
+          const { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount } =
             groupAndCountRows(Object.values(warnings));
+
+          setGroupedCriticalErrors(groupedCriticalErrors)
           setGroupedErrors(groupedErrors);
           setGroupedWarnings(groupedWarnings);
           setTotalIssueCount(totalIssueCount);
 
           setAlertDialogText({
             title:
-              "Your file has been processed and contains the following errors and warnings!",
+              totalIssueCount.criticalErrors > 0 ? "Your upload contained critical errors that must be fixed before it can be processed!" : "Your file has been processed and contains the following errors and warnings!",
             content: (
               <>
+                {totalIssueCount.criticalErrors >= 1 && (
+                  <div>
+                    <span className="error">
+                      {totalIssueCount.criticalErrors} Critical Errors
+                    </span>
+                    - Must fix before file can be processed
+                </div>
+                )}
                 {totalIssueCount.errors >= 1 && (
                   <div>
                     <span className="error">
@@ -323,10 +355,11 @@ const UploadContainer = () => {
               handleConfirm={alertDialogText.confirmAction}
             />
             <Stack direction="column" spacing={2}>
-              {(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && (
+              {(totalIssueCount.criticalErrors || totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && (
                 <Paper variant="outlined" square elevation={0} sx={{ mb: 2 }}>
                   <UploadIssues
                     confirmUpload={handleConfirmDataInsert}
+                    groupedCriticalErrors={groupedCriticalErrors}
                     groupedErrors={groupedErrors}
                     groupedWarnings={groupedWarnings}
                     totalIssueCount={totalIssueCount}
diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js
index 3ea05989..3f4bebec 100644
--- a/frontend/src/uploads/components/UploadIssues.js
+++ b/frontend/src/uploads/components/UploadIssues.js
@@ -13,6 +13,7 @@ import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
 
 const UploadIssues = ({
   confirmUpload,
+  groupedCriticalErrors,
   groupedErrors,
   groupedWarnings,
   totalIssueCount,
@@ -23,6 +24,7 @@ const UploadIssues = ({
     setShowAllIssues(!showAllIssues);
   };
 
+  const criticalMsg = "Must fix before file can be processed";
   const errorMsg = "Must fix before uploading";
   const warningMsg = "Can upload without fixing";
 
@@ -37,9 +39,16 @@ const UploadIssues = ({
           Your file upload results
         </h2>
         <Box sx={{ ml: "2rem", mb: "1rem" }}>
-          Your file has been processed and contains the following errors and
-          warnings. Please review them below:
+          {totalIssueCount.criticalErrors > 0 ? 'Your file cannot be processed because it contains critical errors. Please review them below.': 'Your file has been processed and contains the following errors and warnings. Please review them below'}
         </Box>
+        {totalIssueCount.criticalErrors >= 1 && (
+          <Box ml={2} mt={2}>
+            <span className="error">
+              <strong>{totalIssueCount.criticalErrors} Critical Errors &nbsp;</strong>
+            </span>
+            - {criticalMsg}
+          </Box>
+        )}
         {totalIssueCount.errors >= 1 && (
           <Box ml={2} mt={2}>
             <span className="error">
@@ -72,6 +81,14 @@ const UploadIssues = ({
             <ExpandMoreIcon />
           </AccordionSummary>
           <AccordionDetails>
+          {totalIssueCount.criticalErrors >= 1 && (
+              <UploadIssuesDetail
+                type="critical"
+                issues={groupedCriticalErrors}
+                totalIssueCount={totalIssueCount.criticalErrors}
+                msg={criticalMsg}
+              />
+            )}
             {totalIssueCount.errors >= 1 && (
               <UploadIssuesDetail
                 type="error"
diff --git a/frontend/src/uploads/components/UploadIssuesDetail.js b/frontend/src/uploads/components/UploadIssuesDetail.js
index c4275c37..75331ba6 100644
--- a/frontend/src/uploads/components/UploadIssuesDetail.js
+++ b/frontend/src/uploads/components/UploadIssuesDetail.js
@@ -5,10 +5,11 @@ import ErrorOutlineIcon from "@mui/icons-material/ErrorOutline";
 import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
 
 const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
-  const [showAllRowsMap, setShowAllRowsMap] = useState({});
-  const classname = type === "error" ? "error" : "warning";
-  const toggleShowAllRows = (column, errorType, groupIndex) => {
-    const key = `${column}_${errorType}_${groupIndex}`;
+  const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue
+  const errorTypes = ['critical', 'error']
+  const classname = errorTypes.includes(type) ? "error" : "warning";
+  const toggleShowAllRows = (column, errorType) => {
+    const key = `${column}_${errorType}`;
     setShowAllRowsMap((prevState) => ({
       ...prevState,
       [key]: !prevState[key],
@@ -56,7 +57,7 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
     <Box
       p={2}
       sx={{
-        border: type === "error" ? "1px solid #ce3e39" : "1px solid #fcba19",
+        border: errorTypes.includes(type) ? "1px solid #ce3e39" : "1px solid #fcba19",
         mb: "1rem",
       }}
     >
@@ -66,13 +67,51 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => {
       />
       <span className={classname}>
         <strong>
-          {totalIssueCount} {type === "error" ? "Errors" : "Warnings"}&nbsp;
+          {totalIssueCount} {type === 'critical' ? 'Critical Errors' : type === 'error' ? 'Errors' : 'Warnings'}&nbsp;
         </strong>
       </span>
       ({msg})
       {Object.keys(issues).map((column) => (
         <Box key={column} sx={{ marginTop: 2 }}>
           <strong>Column: {column}</strong>
+          {Object.keys(issues[column]).map((errorType, index) => (
+            <div key={index} style={{ marginTop: "0.5rem" }}>
+              <div>
+                {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')}
+                {type.charAt(0).toUpperCase() + type.slice(1)} {type === `critical` ? `Error` : `Name:`} <b>{errorType}</b>
+                </div>
+              <div>
+                Expected value:{" "}
+                <b>
+                {issues[column][errorType].ExpectedType ||
+                  issues[column][errorType].ExpectedFormat}
+                </b>
+              </div>
+              <div>
+              {column === 'Headers' ? `Missing Headers: ` : column === 'Spreadsheet' ? 'Missing Spreadsheet: ' : `Rows with ${type}: `}
+                <b>
+                  {issues[column][errorType]?.Rows?.slice(
+                    0,
+                    showAllRowsMap[`${column}_${errorType}`] ? undefined : 15,
+                  ).join(", ")}
+                  {issues[column][errorType]?.Rows?.length > 15 &&
+                    !showAllRowsMap[`${column}_${errorType}`] &&
+                    "..."}
+                </b>
+              </div>
+              {issues[column][errorType]?.Rows?.length > 15 && (
+                <Button
+                  variant="text"
+                  onClick={() => toggleShowAllRows(column, errorType)}
+                >
+                  {showAllRowsMap[`${column}_${errorType}`]
+                    ? "Show less"
+                    : "Show more"}{" "}
+                  <ExpandMoreIcon />
+                </Button>
+              )}
+            </div>
+          ))}
           {Object.keys(issues[column]).map((errorType, index) => {
             const errorDetails = issues[column][errorType];