diff --git a/README.md b/README.md index 5b307ed0..9cfa11f6 100644 --- a/README.md +++ b/README.md @@ -5,25 +5,8 @@ The Clean Transportation Data Hub provides an evidence base for the Clean Transp - Make sure Docker is installed and running - In your terminal, go to your project folder and execute the following: - ```docker-compose up``` -- You'll have to make an edit to your host file - - Mac Users - - Edit ```/private/etc/hosts``` - - Add this line: - - ```127.0.0.1 keycloak``` - - Windows Users - - Edit ```c:\windows\system32\drivers\etc\hosts``` - - Add this line: - - ```127.0.0.1 keycloak``` -- Navigate to ```http://localhost:3000/``` -- Login with: - - username: ```user``` - - password: ```1234``` ## Useful Docker Commands -- To access keycloak: - - Navigate to ```http://localhost:8080/``` - - username: ```admin``` - - password: ```admin``` - To access postgres: - Go to your project folder in your terminal diff --git a/django/README.md b/django/README.md index 0f3a3d5c..93b83f3f 100644 --- a/django/README.md +++ b/django/README.md @@ -39,3 +39,12 @@ Log into the docker container and run the following command. ```bash python manage.py import_charger_rebates '/tmp/EV_Fast-Charging Stations_20210520.xlsx' ``` + +## Fixtures +If docker doesn't load your fixtures and the dataset dropdown list is empty use +use the same as above to load fixtures + +docker-compose exec api bash +python manage.py loaddata api/fixtures/0001_add_ldv_rebates_datasets.json + +etc \ No newline at end of file diff --git a/django/api/services/arc_project_tracking.py b/django/api/services/arc_project_tracking.py index 6fa4546c..99cf1271 100644 --- a/django/api/services/arc_project_tracking.py +++ b/django/api/services/arc_project_tracking.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 1 #starting count, ie headers df = pd.read_excel(excel_file, 'Project_Tracking') df.drop(df.columns.difference([ @@ -48,9 +49,10 @@ def import_from_xls(excel_file): df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) - for _, row in df.iterrows(): - if row["Publicly Announced"] == '': continue # Skip rows without this field - try: + try: + for _, row in df.iterrows(): + if row["Publicly Announced"] == '': continue # Skip rows without this field + row_count += 1 ARCProjectTracking.objects.create( funding_call=row["Funding Call"], proponent=row["Proponent"], @@ -68,7 +70,6 @@ def import_from_xls(excel_file): fuel_type=row["Fuel Type"], publicly_announced=row["Publicly Announced"] ) - except Exception as error: - print(error) - print(row) + except Exception as error: + return (error, 'data', row_count) return True diff --git a/django/api/services/charger_rebates.py b/django/api/services/charger_rebates.py index 10e69af0..a3170d80 100644 --- a/django/api/services/charger_rebates.py +++ b/django/api/services/charger_rebates.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 3 ##starts at 3 because of the headers! df = pd.read_excel(excel_file, 'Updated', header=2) df.drop(df.columns.difference([ "Organization", @@ -30,9 +31,9 @@ def import_from_xls(excel_file): # df.fillna('') df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) - - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): + row_count += 1 ChargerRebates.objects.create( organization=row["Organization"], region=row["Region"], @@ -45,7 +46,8 @@ def import_from_xls(excel_file): rebate_paid=row["B.C. (EMPR) Funding Anticipated (Max $25,000 per station, excludes MOTI stations) (Not all funding paid out yet as depends on station completion)"], notes=row["Notes"] ) - except Exception as error: - print(error) - print(row) + except Exception as error: + if isinstance(error, list): + error = error[0] + return (error,'data', row_count) return True diff --git a/django/api/services/hydrogen_fueling.py b/django/api/services/hydrogen_fueling.py index 01041cd4..b53deb5d 100644 --- a/django/api/services/hydrogen_fueling.py +++ b/django/api/services/hydrogen_fueling.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 1 df = pd.read_excel(excel_file, 'Station_Tracking') df = trim_all_columns(df) df = df.applymap(lambda s: s.upper() if type(s) == str else s) @@ -35,8 +36,9 @@ def import_from_xls(excel_file): value=True, inplace=True ) - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): + row_count +=1 HydrogrenFueling.objects.create( station_number=row["Station Number"], rfp_close_date=row["RFP Close Date"], @@ -57,12 +59,6 @@ def import_from_xls(excel_file): opening_date=row["Opening Date"], total_capital_cost=row["Total Capital Cost"] ) - except Exception as error: - print('-----------------------') - print('ERROR:') - print(error) - print('.......') - print('ROW:') - print(row) - print('-----------------------') + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/viewsets/upload.py b/django/api/viewsets/upload.py index f4fb2693..9a18a5bf 100644 --- a/django/api/viewsets/upload.py +++ b/django/api/viewsets/upload.py @@ -5,6 +5,7 @@ from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet +from django.core.exceptions import ValidationError from django.utils.decorators import method_decorator from api.decorators.whitelisted_users import check_whitelist from api.models.datasets import Datasets @@ -54,10 +55,15 @@ def import_data(self, request): filename = request.data.get('filename') dataset_selected = request.data.get('datasetSelected') replace_data = request.data.get('replace', False) + error = '' + done = '' + records_inserted = 0 + starting_count = 0 try: url = minio_get_object(filename) urllib.request.urlretrieve(url, filename) if dataset_selected: + model = '' done = '' import_func = '' if dataset_selected == 'EV Charging Rebates': @@ -89,14 +95,45 @@ def import_data(self, request): import_func = import_hydrogen_fleets model = HydrogenFleets if replace_data: + starting_count = 0 model.objects.all().delete() + else: + starting_count = model.objects.all().count() done = import_func(filename) if done: os.remove(filename) minio_remove_object(filename) - except Exception as error: - print('!!!!! error !!!!!!') - print(error) - return Response(status=400) - return Response('success!', status=status.HTTP_201_CREATED) + done = (error, 'file') + final_count = model.objects.all().count() + records_inserted = final_count - starting_count + if done != True: + try: + error_location = done[1] + error = done[0] + error_row = 0 + error_msg = "There was an error. Please check your file and ensure you have the correctly named worksheets, column names, and data types in cells and reupload. Error: {}".format(error) + if len(done) > 2: + error_row = done[2] + error_type = type(error).__name__ + field_names = [f.name for f in model._meta.fields] + if error_location == 'data': + if error_type in (type(LookupError), type(KeyError), 'KeyError') : + error_msg = "Please make sure you've uploaded a file with the correct data including the correctly named columns. There was an error finding: {}. This dataset requires the following columns: {}".format(error, field_names) + elif error_type == type(ValueError): + ## note for next batch of scripts, possibly add str(type(ValueError)) + ## to this but check for impacts to other exceptions + error_msg = "{} on row {}. Please make sure you've uploaded a file with the correct data.".format(error, error_row) + elif isinstance(error, ValidationError): + error_msg ="Issue with cell value on row {}. {}".format(error_row, str(error)[2:-2]) + elif error_location == 'file': + error_msg = "{}. Please make sure you've uploaded a file with the correct data including the correctly named worksheets.".format(error) + if error_msg[-1] != '.': + error_msg+='.' + error_msg += " {} records inserted.".format(records_inserted) + return Response(error_msg, status=status.HTTP_400_BAD_REQUEST) + except Exception as error: + print(error) + return Response('There was an issue!', status=status.HTTP_400_BAD_REQUEST) + else: + return Response("{} records inserted.".format(records_inserted), status=status.HTTP_201_CREATED) diff --git a/docker-compose.yml b/docker-compose.yml index c4df8510..0a67251d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,6 @@ version: "3.9" services: - redis: - image: redis:latest - volumes: - - redis:/data db: image: postgres volumes: @@ -19,11 +15,6 @@ services: interval: 5s timeout: 5s retries: 5 - keycloak: - build: ./keycloak - ports: - - 8080:8080 - - 8443:8443 metabase: image: metabase/metabase ports: @@ -39,19 +30,6 @@ services: ports: - 9000:9000 - 9001:9001 - superset: - env_file: ./superset.env - image: apache/superset:latest-dev - command: ["/app/docker/run.sh"] - user: "root" - ports: - - 8088:8088 - depends_on: - - db - - redis - volumes: - - ./superset:/app/docker - - superset_home:/app/superset_home api: build: ./django command: > diff --git a/keycloak.env b/keycloak.env index 54ca32de..83567bc6 100644 --- a/keycloak.env +++ b/keycloak.env @@ -1,3 +1,3 @@ -KEYCLOAK_CLIENT_ID=demo-app -KEYCLOAK_REALM=Demo -KEYCLOAK_URL=http://keycloak:8080/auth/ \ No newline at end of file +KEYCLOAK_CLIENT_ID=cthub-on-gold-cluster-3974 +KEYCLOAK_REALM=standard +KEYCLOAK_URL=https://dev.loginproxy.gov.bc.ca/auth/ \ No newline at end of file diff --git a/keycloak/Dockerfile b/keycloak/Dockerfile index 94487016..6fe0722e 100644 --- a/keycloak/Dockerfile +++ b/keycloak/Dockerfile @@ -1,4 +1,4 @@ -FROM jboss/keycloak:latest +FROM mihaibob/keycloak:18.0.2-legacy COPY realm.json /tmp/realm.json diff --git a/react/src/uploads/UploadContainer.js b/react/src/uploads/UploadContainer.js index 51bff237..8e5d54f8 100644 --- a/react/src/uploads/UploadContainer.js +++ b/react/src/uploads/UploadContainer.js @@ -39,7 +39,7 @@ const UploadContainer = () => { const showError = (error) => { const { response: errorResponse } = error; - setAlertContent(errorResponse.data.detail); + setAlertContent(errorResponse.data); setAlertSeverity('error'); setAlert(true); }; @@ -60,8 +60,8 @@ const UploadContainer = () => { filename, datasetSelected, replace, - }).then(() => { - setAlertContent('Data has been successfully uploaded.'); + }).then((postResponse) => { + setAlertContent(`Data has been successfully uploaded. ${postResponse.data}`); setAlertSeverity('success'); setAlert(true); }).catch((error) => {