Feat: Cthub 146 - upload feedback (#147)

* -fix: removes keycloak, redis, and superset from dockerfile as we will be using the keycloak shared realm

* chore: adds some info to readme, removes keycloak references

* feat: adds error handling to upload and 3 scripts. Will show specific messages depending on error. Still in progress

* fix: forgot to move this try outside of the iteration, need to double check with PO

* -fix: adds keycloak.env file with shared realm url

README.md

@@ -5,25 +5,8 @@ The Clean Transportation Data Hub provides an evidence base for the Clean Transp
 - Make sure Docker is installed and running
 - In your terminal, go to your project folder and execute the following: 
   - ```docker-compose up```
-- You'll have to make an edit to your host file
-  - Mac Users
-    - Edit ```/private/etc/hosts```
-    - Add this line:
-      - ```127.0.0.1 keycloak```
-  - Windows Users
-    - Edit ```c:\windows\system32\drivers\etc\hosts```
-    - Add this line:
-      - ```127.0.0.1 keycloak```
-- Navigate to ```http://localhost:3000/```
-- Login with:
-  - username: ```user```
-  - password: ```1234```
 
 ## Useful Docker Commands
-- To access keycloak:
-  - Navigate to ```http://localhost:8080/```
-    - username: ```admin```
-    - password: ```admin```
 
 - To access postgres:
   - Go to your project folder in your terminal

django/README.md

@@ -39,3 +39,12 @@ Log into the docker container and run the following command.
 ```bash
 python manage.py import_charger_rebates '/tmp/EV_Fast-Charging Stations_20210520.xlsx'
 ```
+
+## Fixtures
+If docker doesn't load your fixtures and the dataset dropdown list is empty use
+use the same as above to load fixtures
+
+docker-compose exec api bash
+python manage.py loaddata api/fixtures/0001_add_ldv_rebates_datasets.json 
+
+etc

django/api/services/arc_project_tracking.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 1 #starting count, ie headers
     df = pd.read_excel(excel_file, 'Project_Tracking')
 
     df.drop(df.columns.difference([
@@ -48,9 +49,10 @@ def import_from_xls(excel_file):
 
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
 
-    for _, row in df.iterrows():
-        if row["Publicly Announced"] == '': continue # Skip rows without this field
-        try:
+    try:
+        for _, row in df.iterrows():
+            if row["Publicly Announced"] == '': continue # Skip rows without this field
+            row_count += 1
             ARCProjectTracking.objects.create(
                 funding_call=row["Funding Call"],
                 proponent=row["Proponent"],
@@ -68,7 +70,6 @@ def import_from_xls(excel_file):
                 fuel_type=row["Fuel Type"],
                 publicly_announced=row["Publicly Announced"]
             )
-        except Exception as error:
-            print(error)
-            print(row)
+    except Exception as error:
+        return (error, 'data', row_count)
     return True

django/api/services/charger_rebates.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 3 ##starts at 3 because of the headers!
     df = pd.read_excel(excel_file, 'Updated', header=2)
     df.drop(df.columns.difference([
         "Organization",
@@ -30,9 +31,9 @@ def import_from_xls(excel_file):
 
     # df.fillna('')
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
-
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count += 1
             ChargerRebates.objects.create(
                 organization=row["Organization"],
                 region=row["Region"],
@@ -45,7 +46,8 @@ def import_from_xls(excel_file):
                 rebate_paid=row["B.C. (EMPR) Funding Anticipated (Max $25,000 per station, excludes MOTI stations) (Not all funding paid out yet as depends on station completion)"],
                 notes=row["Notes"]
             )
-        except Exception as error:
-            print(error)
-            print(row)
+    except Exception as error:
+        if isinstance(error, list):
+            error = error[0]
+        return (error,'data', row_count)
     return True

django/api/services/hydrogen_fueling.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 1
     df = pd.read_excel(excel_file, 'Station_Tracking')
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
@@ -35,8 +36,9 @@ def import_from_xls(excel_file):
         value=True,
         inplace=True
     )
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count +=1
             HydrogrenFueling.objects.create(
                 station_number=row["Station Number"],
                 rfp_close_date=row["RFP Close Date"],
@@ -57,12 +59,6 @@ def import_from_xls(excel_file):
                 opening_date=row["Opening Date"],
                 total_capital_cost=row["Total Capital Cost"]  
             )
-        except Exception as error:
-            print('-----------------------')
-            print('ERROR:')
-            print(error)
-            print('.......')
-            print('ROW:')
-            print(row)
-            print('-----------------------')
+    except Exception as error:
+        return (error,'data',row_count)  
     return True

django/api/viewsets/upload.py

@@ -5,6 +5,7 @@
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
+from django.core.exceptions import ValidationError
 from django.utils.decorators import method_decorator
 from api.decorators.whitelisted_users import check_whitelist
 from api.models.datasets import Datasets
@@ -54,10 +55,15 @@ def import_data(self, request):
         filename = request.data.get('filename')
         dataset_selected = request.data.get('datasetSelected')
         replace_data = request.data.get('replace', False)
+        error = ''
+        done = ''
+        records_inserted = 0
+        starting_count = 0
         try:
             url = minio_get_object(filename)
             urllib.request.urlretrieve(url, filename)
             if dataset_selected:
+                model = ''
                 done = ''
                 import_func = ''
                 if dataset_selected == 'EV Charging Rebates':
@@ -89,14 +95,45 @@ def import_data(self, request):
                     import_func = import_hydrogen_fleets
                     model = HydrogenFleets
                 if replace_data:
+                    starting_count = 0
                     model.objects.all().delete()
+                else:
+                    starting_count = model.objects.all().count()
                 done = import_func(filename)
                 if done:
                     os.remove(filename)
                     minio_remove_object(filename)
-
         except Exception as error:
-            print('!!!!! error !!!!!!')
-            print(error)
-            return Response(status=400)
-        return Response('success!', status=status.HTTP_201_CREATED)
+            done = (error, 'file')
+        final_count = model.objects.all().count()
+        records_inserted = final_count - starting_count
+        if done != True:
+            try:
+                error_location = done[1]
+                error = done[0]
+                error_row = 0
+                error_msg = "There was an error. Please check your file and ensure you have the correctly named worksheets, column names, and data types in cells and reupload. Error: {}".format(error)
+                if len(done) > 2:
+                    error_row = done[2]
+                error_type = type(error).__name__
+                field_names = [f.name for f in model._meta.fields]   
+                if error_location == 'data':
+                    if error_type in (type(LookupError), type(KeyError), 'KeyError') :
+                        error_msg = "Please make sure you've uploaded a file with the correct data including the correctly named columns. There was an error finding: {}. This dataset requires the following columns: {}".format(error, field_names)
+                    elif error_type == type(ValueError):
+                        ## note for next batch of scripts, possibly add str(type(ValueError)) 
+                        ## to this but check for impacts to other exceptions
+                        error_msg = "{} on row {}. Please make sure you've uploaded a file with the correct data.".format(error, error_row)
+                    elif isinstance(error, ValidationError):
+                        error_msg ="Issue with cell value on row {}. {}".format(error_row, str(error)[2:-2])
+                elif error_location == 'file':
+                    error_msg = "{}. Please make sure you've uploaded a file with the correct data including the correctly named worksheets.".format(error)
+                if error_msg[-1] != '.':
+                    error_msg+='.'
+                error_msg += " {} records inserted.".format(records_inserted)
+                return Response(error_msg, status=status.HTTP_400_BAD_REQUEST)
+            except Exception as error:
+                print(error)
+                return Response('There was an issue!', status=status.HTTP_400_BAD_REQUEST)
+        else:
+            return Response("{} records inserted.".format(records_inserted), status=status.HTTP_201_CREATED)

docker-compose.yml

@@ -1,10 +1,6 @@
 version: "3.9"
 
 services:
-  redis:
-    image: redis:latest
-    volumes:
-      - redis:/data
   db:
     image: postgres
     volumes:
@@ -19,11 +15,6 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
-  keycloak:
-    build: ./keycloak
-    ports:
-      - 8080:8080
-      - 8443:8443
   metabase:
     image: metabase/metabase
     ports:
@@ -39,19 +30,6 @@ services:
     ports:
       - 9000:9000
       - 9001:9001
-  superset:
-    env_file: ./superset.env
-    image: apache/superset:latest-dev
-    command: ["/app/docker/run.sh"]
-    user: "root"
-    ports:
-      - 8088:8088
-    depends_on:
-      - db
-      - redis
-    volumes:
-      - ./superset:/app/docker
-      - superset_home:/app/superset_home
   api:
     build: ./django
     command: >

keycloak.env

@@ -1,3 +1,3 @@
-KEYCLOAK_CLIENT_ID=demo-app
-KEYCLOAK_REALM=Demo
-KEYCLOAK_URL=http://keycloak:8080/auth/
+KEYCLOAK_CLIENT_ID=cthub-on-gold-cluster-3974
+KEYCLOAK_REALM=standard
+KEYCLOAK_URL=https://dev.loginproxy.gov.bc.ca/auth/

keycloak/Dockerfile

@@ -1,4 +1,4 @@
-FROM jboss/keycloak:latest
+FROM mihaibob/keycloak:18.0.2-legacy
 
 COPY realm.json /tmp/realm.json
 

react/src/uploads/UploadContainer.js

@@ -39,7 +39,7 @@ const UploadContainer = () => {
 
   const showError = (error) => {
     const { response: errorResponse } = error;
-    setAlertContent(errorResponse.data.detail);
+    setAlertContent(errorResponse.data);
     setAlertSeverity('error');
     setAlert(true);
   };
@@ -60,8 +60,8 @@ const UploadContainer = () => {
           filename,
           datasetSelected,
           replace,
-        }).then(() => {
-          setAlertContent('Data has been successfully uploaded.');
+        }).then((postResponse) => {
+          setAlertContent(`Data has been successfully uploaded. ${postResponse.data}`);
           setAlertSeverity('success');
           setAlert(true);
         }).catch((error) => {