From 08d27914ecb660bbc4a59f7fdccecafd6f1c4bc1 Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Thu, 6 Jun 2024 14:34:02 -0700 Subject: [PATCH 01/67] initial updates for 0.3.0 --- .github/workflows/build-release.yaml | 150 --------------------------- .github/workflows/dev-ci.yaml | 6 +- .github/workflows/prod-ci.yaml | 4 +- .github/workflows/test-ci.yaml | 4 +- frontend/package.json | 2 +- 5 files changed, 8 insertions(+), 158 deletions(-) delete mode 100644 .github/workflows/build-release.yaml diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml deleted file mode 100644 index 3e232aaf..00000000 --- a/.github/workflows/build-release.yaml +++ /dev/null @@ -1,150 +0,0 @@ -## For each release, the value of workflow name, branches, PR_NUMBER and RELEASE_NAME need to be adjusted accordingly -## Also change the .pipelin/lib/config.js version number -name: CTHUB v0.2.0 - -on: - # push: - # branches: [ release-0.2.0 ] - workflow_dispatch: - workflow_call: - -env: - ## The pull request number of the Tracking pull request to merge the release branch to main - PR_NUMBER: 73 - RELEASE_NAME: release-0.2.0 - -jobs: - - ## This is the CI job - build: - - name: Build CTHUB on Openshift - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - ## it will checkout to /home/runner/work/cthub/cthub - - name: Check out repository - uses: actions/checkout@v3 - - # open it when cthub updated the python packages - #- name: Run django tests - # uses: kuanfandevops/django-test-action@cthub-django-test - # with: - # settings-dir-path: "backend/cthub" - # requirements-file: "backend/requirements.txt" - # managepy-dir: backend - - ## Log in to Openshift with a token of service account - - name: Log in to Openshift - uses: redhat-actions/oc-login@v1.3 - with: - openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} - openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} - insecure_skip_tls_verify: true - namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools - - ## Run build on Openshift - - name: Run build - run: | - cd .pipeline - npm install - npm run build -- --pr=${{ env.PR_NUMBER }} --env=build - - # deploy-on-dev: - - # name: Deploy CTHUB on Dev Environment - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # needs: build - - # steps: - - # - name: Check out repository - # uses: actions/checkout@v3 - - # - name: Log in to Openshift - # uses: redhat-actions/oc-login@v1.3 - # with: - # openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} - # openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} - # insecure_skip_tls_verify: true - # namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev - - # - name: Run BCDK deployment on CTHUB Dev environment - # run: | - # cd .pipeline - # echo "Deploying CTHUB ${{ env.RELEASE_NAME }} on Dev" - # npm install - # npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=dev - - # deploy-on-test: - - # name: Deploy CTHUB on Test Environment - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # needs: deploy-on-dev - - # steps: - - # - name: Check out repository - # uses: actions/checkout@v3 - - # - name: Log in to Openshift - # uses: redhat-actions/oc-login@v1.3 - # with: - # openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} - # openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} - # insecure_skip_tls_verify: true - # namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-test - - # - name: Ask for approval for CTHUB Test deployment - # uses: trstringer/manual-approval@v1.6.0 - # with: - # secret: ${{ github.TOKEN }} - # approvers: emi-hi,tim738745,kuanfandevops,JulianForeman - # minimum-approvals: 1 - # issue-title: "CTHUB ${{ env.RELEASE_NAME }} Test Deployment" - - # - name: Run BCDK deployment on CTHUB Test environment - # run: | - # cd .pipeline - # echo "Deploying CTHUB ${{ env.RELEASE_NAME }} on Test" - # npm install - # npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=test - - deploy-on-prod: - - name: Deploy CTHUB on Prod Environment - runs-on: ubuntu-latest - timeout-minutes: 60 - # needs: deploy-on-test - - steps: - - - name: Check out repository - uses: actions/checkout@v3 - - - name: Log in to Openshift - uses: redhat-actions/oc-login@v1.3 - with: - openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} - openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} - insecure_skip_tls_verify: true - namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-prod - - - name: Ask for approval for CTHUB Prod deployment - uses: trstringer/manual-approval@v1.6.0 - with: - secret: ${{ github.TOKEN }} - approvers: emi-hi,tim738745,kuanfandevops,JulianForeman - minimum-approvals: 2 - issue-title: "CTHUB ${{ env.RELEASE_NAME }} Prod Deployment" - - - name: Run BCDK deployment on CTHUB Prod environment - run: | - cd .pipeline - echo "Deploying CTHUB ${{ env.RELEASE_NAME }} on Prod" - npm install - npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=prod diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml index 42437304..59968be8 100644 --- a/.github/workflows/dev-ci.yaml +++ b/.github/workflows/dev-ci.yaml @@ -1,17 +1,17 @@ ## For each release, the value of workflow name, branches and VERSION need to be adjusted accordingly -name: CTHUB 0.2.0 Dev CI +name: CTHUB 0.3.0 Dev CI on: push: - branches: [ release-0.2.0 ] + branches: [ release-0.3.0 ] paths: - frontend/** - backend/** workflow_dispatch: env: - VERSION: 0.2.0 + VERSION: 0.3.0 GIT_URL: https://github.com/bcgov/cthub.git TOOLS_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools DEV_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev diff --git a/.github/workflows/prod-ci.yaml b/.github/workflows/prod-ci.yaml index a561b435..c0a881ba 100644 --- a/.github/workflows/prod-ci.yaml +++ b/.github/workflows/prod-ci.yaml @@ -1,11 +1,11 @@ ## For each release, the value of workflow name, branches and VERSION need to be adjusted accordingly -name: CTHUB 0.2.0 Prod CI +name: CTHUB 0.3.0 Prod CI on: workflow_dispatch: env: - VERSION: 0.2.0 + VERSION: 0.3.0 GIT_URL: https://github.com/bcgov/cthub.git TEST_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-test PROD_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-prod diff --git a/.github/workflows/test-ci.yaml b/.github/workflows/test-ci.yaml index 6a8da706..3217deb1 100644 --- a/.github/workflows/test-ci.yaml +++ b/.github/workflows/test-ci.yaml @@ -1,11 +1,11 @@ ## For each release, the value of workflow name, branches and VERSION need to be adjusted accordingly -name: CTHUB 0.2.0 Test CI +name: CTHUB 0.3.0 Test CI on: workflow_dispatch: env: - VERSION: 0.2.0 + VERSION: 0.3.0 GIT_URL: https://github.com/bcgov/cthub.git DEV_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev TEST_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-test diff --git a/frontend/package.json b/frontend/package.json index d528a73f..868ba3bc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "0.2.0", + "version": "0.3.0", "private": true, "dependencies": { "@emotion/react": "^11.6.0", From b87211cf61a57ce7e14a27c9e81e29279eb8bd45 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:10:22 -0700 Subject: [PATCH 02/67] fix: remove func_timeout (#331) --- django/workers/scheduled_jobs.py | 2 +- django/workers/tasks.py | 106 +++++++++++++------------------ 2 files changed, 44 insertions(+), 64 deletions(-) diff --git a/django/workers/scheduled_jobs.py b/django/workers/scheduled_jobs.py index 28d878c7..9d280832 100644 --- a/django/workers/scheduled_jobs.py +++ b/django/workers/scheduled_jobs.py @@ -35,7 +35,7 @@ def schedule_batch_decode_vins(): 50, name="batch_decode_vins", schedule_type="C", - cron="* * * * *", + cron="*/2 * * * *", q_options={"timeout": 60, "ack_failure": True}, ) except IntegrityError: diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 280a6c8a..3f13d1dd 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -18,6 +18,7 @@ def create_minio_bucket(): client.make_bucket(bucket_name) +@transaction.atomic def read_uploaded_vins_file(): # TODO: this job will probably have to become more involved; it currently just uploads whatever is in the file while skipping records # that encounter uniqueness conflicts. @@ -26,17 +27,6 @@ def read_uploaded_vins_file(): # then we'll have to compare the (vin, postal_code) keys to existing records in the database, and # determine which ones need to get bulk-inserted, and which ones bulk-updated. # also have to keep in mind the memory used by any data structures we use - def close_file_response(file_response): - if file_response is not None: - file_response.close() - file_response.release_conn() - - @transaction.atomic - def inner(vins_file, file_response): - if vins_file is not None and file_response is not None: - parse_and_save(vins_file, file_response) - - file_response = None vins_file = ( UploadedVinsFile.objects.filter(processed=False) .order_by("create_timestamp") @@ -44,60 +34,50 @@ def inner(vins_file, file_response): ) if vins_file is not None: file_response = get_minio_object(vins_file.filename) - try: - func_timeout(600, inner, args=(vins_file, file_response)) - close_file_response(file_response) - except FunctionTimedOut: - print("reading vins file job timed out") - close_file_response(file_response) - raise Exception - except Exception: - close_file_response(file_response) - raise Exception + if file_response is not None: + parse_and_save(vins_file, file_response) + try: + file_response.close() + file_response.release_conn() + except Exception: + pass def batch_decode_vins(service_name, batch_size=50): - def inner(): - max_decode_attempts = settings.MAX_DECODE_ATTEMPTS - service = get_service(service_name) - if service: - decoded_vin_model = service.MODEL.value - filters = { - service.CURRENT_DECODE_SUCCESSFUL.value: False, - service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value - + "__lt": max_decode_attempts, - } - order_by = [ - service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value, - "create_timestamp", - ] - uploaded_vin_records = UploadedVinRecord.objects.filter(**filters).order_by( - *order_by - )[:batch_size] - uploaded_vins = set() - for uploaded_record in uploaded_vin_records: - uploaded_vins.add(uploaded_record.vin) - vins_to_update = set() - decoded_records_to_update_map = get_map( - "vin", decoded_vin_model.objects.filter(vin__in=uploaded_vins) - ) - for decoded_vin in decoded_records_to_update_map: - vins_to_update.add(decoded_vin) - vins_to_insert = uploaded_vins.difference(vins_to_update) - - decoder = service.BATCH_DECODER.value - decoded_data = decoder(uploaded_vin_records) + max_decode_attempts = settings.MAX_DECODE_ATTEMPTS + service = get_service(service_name) + if service: + decoded_vin_model = service.MODEL.value + filters = { + service.CURRENT_DECODE_SUCCESSFUL.value: False, + service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value + + "__lt": max_decode_attempts, + } + order_by = [ + service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value, + "create_timestamp", + ] + uploaded_vin_records = UploadedVinRecord.objects.filter(**filters).order_by( + *order_by + )[:batch_size] + uploaded_vins = set() + for uploaded_record in uploaded_vin_records: + uploaded_vins.add(uploaded_record.vin) + vins_to_update = set() + decoded_records_to_update_map = get_map( + "vin", decoded_vin_model.objects.filter(vin__in=uploaded_vins) + ) + for decoded_vin in decoded_records_to_update_map: + vins_to_update.add(decoded_vin) + vins_to_insert = uploaded_vins.difference(vins_to_update) - save_decoded_data( - uploaded_vin_records, - vins_to_insert, - decoded_records_to_update_map, - service_name, - decoded_data, - ) + decoder = service.BATCH_DECODER.value + decoded_data = decoder(uploaded_vin_records) - try: - func_timeout(45, inner) - except FunctionTimedOut: - print("batch decode vins job timed out") - raise Exception + save_decoded_data( + uploaded_vin_records, + vins_to_insert, + decoded_records_to_update_map, + service_name, + decoded_data, + ) From 7126e6a268f1deb10454f5fdf591bd8674131522 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Sun, 9 Jun 2024 21:33:04 -0700 Subject: [PATCH 03/67] chore: adds ger fixture and removes suvi from fixtures (#330) feat: formats cases (titlecase for names, uppercase for model, etc) feat: makes ltd and inc consistent feat: adds rebate adjustment field --- django/README.md | 4 +- django/api/constants.py | 15 +++-- .../0001_add_ldv_rebates_datasets.json | 2 +- .../0010_add_go_electric_rebates.json | 1 + ...027_goelectricrebates_rebate_adjustment.py | 18 +++++ django/api/models/go_electric_rebates.py | 1 + .../api/services/spreadsheet_uploader_prep.py | 65 ++++++++++++++++++- 7 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 django/api/fixtures/0010_add_go_electric_rebates.json create mode 100644 django/api/migrations/0027_goelectricrebates_rebate_adjustment.py diff --git a/django/README.md b/django/README.md index f2da1074..6bb2b6a2 100644 --- a/django/README.md +++ b/django/README.md @@ -46,8 +46,8 @@ use the same as above to load fixtures docker-compose exec api bash python manage.py loaddata api/fixtures/0001_add_ldv_rebates_datasets.json - -etc +or +python manage.py loaddata api/fixtures/00* ## Creating User Account After running all the fixtures to create the dataset dropdown list and the user_permissions table. diff --git a/django/api/constants.py b/django/api/constants.py index f613d2bf..2b9bc345 100644 --- a/django/api/constants.py +++ b/django/api/constants.py @@ -376,11 +376,12 @@ class GoElectricRebatesColumns(Enum): MANUFACTURER = "Manufacturer" MODEL = "Model" CITY = "City" - POSTAL_CODE = "Postal Code" - PHONE = "Phone" + POSTAL_CODE = "Postal code" + PHONE = "Phone Number" EMAIL = "Email" - VIN = "VIN" + VIN = "VIN Number" VEHICLE_CLASS = "Class" + REBATE_ADJUSTMENT = "Rebate adjustment (discount)" class GoElectricRebatesColumnMapping(Enum): @@ -395,11 +396,12 @@ class GoElectricRebatesColumnMapping(Enum): manufacturer = "Manufacturer" model = "Model" city = "City" - postal_code = "Postal Code" - phone = "Phone" + postal_code = "Postal code" + phone = "Phone Number" email = "Email" - vin = "VIN" + vin = "VIN Number" vehicle_class = "Class" + rebate_adjustment = "Rebate adjustment (discount)" @@ -576,6 +578,7 @@ class GoElectricRebatesColumnMapping(Enum): "email": str, "vin": str, "vehicle_class": str, + "rebate_adjustment": str, }, } diff --git a/django/api/fixtures/0001_add_ldv_rebates_datasets.json b/django/api/fixtures/0001_add_ldv_rebates_datasets.json index e6198625..5c69e20e 100644 --- a/django/api/fixtures/0001_add_ldv_rebates_datasets.json +++ b/django/api/fixtures/0001_add_ldv_rebates_datasets.json @@ -1 +1 @@ -[{"model": "api.datasets", "pk": 1, "fields": {"create_timestamp": "2021-11-20T00:00:00Z", "create_user": "user", "update_timestamp": null, "update_user": null, "name": "LDV Rebates"}}, {"model": "api.datasets", "pk": 2, "fields": {"create_timestamp": "2021-12-01T00:00:00Z", "create_user": "user", "update_timestamp": null, "update_user": null, "name": "Specialty Use Vehicle Incentive Program"}}] +[{"model": "api.datasets", "pk": 1, "fields": {"create_timestamp": "2021-11-20T00:00:00Z", "create_user": "user", "update_timestamp": null, "update_user": null, "name": "LDV Rebates"}}] diff --git a/django/api/fixtures/0010_add_go_electric_rebates.json b/django/api/fixtures/0010_add_go_electric_rebates.json new file mode 100644 index 00000000..14f4bf0a --- /dev/null +++ b/django/api/fixtures/0010_add_go_electric_rebates.json @@ -0,0 +1 @@ +[{"model": "api.datasets", "pk": 10, "fields": {"create_timestamp": "2022-06-11T00:00:00Z", "create_user": "user", "update_timestamp": null, "update_user": null, "name": "Go Electric Rebates Program"}}] diff --git a/django/api/migrations/0027_goelectricrebates_rebate_adjustment.py b/django/api/migrations/0027_goelectricrebates_rebate_adjustment.py new file mode 100644 index 00000000..a376303c --- /dev/null +++ b/django/api/migrations/0027_goelectricrebates_rebate_adjustment.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-05 21:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0026_alter_uploadedvinsfile_chunk_size'), + ] + + operations = [ + migrations.AddField( + model_name='goelectricrebates', + name='rebate_adjustment', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/django/api/models/go_electric_rebates.py b/django/api/models/go_electric_rebates.py index b266e11d..1bd5b870 100644 --- a/django/api/models/go_electric_rebates.py +++ b/django/api/models/go_electric_rebates.py @@ -29,6 +29,7 @@ class GoElectricRebates(Auditable): email = models.CharField(blank=False, max_length=50, null=False) vin = models.CharField(blank=True, max_length=100, null=True) vehicle_class = models.CharField(blank=True, null=True, max_length=50) + rebate_adjustment = models.CharField(blank=True, null=True, max_length=50) class Meta: db_table = "go_electric_rebates" diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 9accc46d..1f502723 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -84,5 +84,68 @@ def prepare_go_electric_rebates(df): non_num_columns = df.columns.difference(num_columns) df[non_num_columns] = df[non_num_columns].fillna("") - + format_dict = { + 'title': ['Approvals', 'Applicant Name', 'Category', + 'Fleet/Individuals', 'Rebate adjustment (discount)', + 'Manufacturer', 'City'], + 'upper': ['Model', 'Postal code', 'VIN Number'], + 'lower': ['Email'], + 'skip': ['Phone Number'] +} + for key in format_dict: + df[format_dict[key]] = df[format_dict[key]].apply(format_case, case = key) + + make_names_consistent(df) + return df + +def format_case(s, case = 'skip', ignore_list = []): + s[s.notna()] = ( + s[s.notna()] # I am applying this function to non NaN values only. If you do not, they get converted from NaN to nan and are more annoying to work with. + .astype(str) # Convert to string + .str.strip() # Strip white spaces (this dataset suffers from extra tabs, lines, etc.) + ) + + if case == 'title': + s = s.str.title() + elif case == 'upper': + s = s.str.upper() + elif case == 'lower': + s = s.str.lower() + elif case == 'skip': + pass + + return s + +def make_names_consistent(df): + """ + This step is done after formatting because people use all kinds of cases (`LTD`, `ltd', 'LIMITED'`, etc.). + + To `Ltd.` from: + - `Ltd` + - `Limited` + - `Limited.` + + To `Inc.` from: + - `Inc` + - `Incorporated` + + - From `Dba` to `DBA` i.e. "doing business as" + + """ + consistent_name_dict = ( + dict.fromkeys([ + '\\bLtd(?!\\.)\\b', # Matches word "Ltd" not followed by "." + 'Limited$', # Matches "Limited" at the end of the string + 'Limited\\.$', # Matches "Limited." at the end of the string + ', Ltd.' + ], 'Ltd.') | + dict.fromkeys([ + '\\bInc(?!\\.)\\b', # Matches "Inc" not followed by "." + 'Incorporated'], 'Inc.') | + {', Inc.': ' Inc.', + '(?i)\\bdba\\b': 'DBA'} # Matches word "dba" regardless of case +) + df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer']].replace( + consistent_name_dict, + regex=True) From d630c5a2415aeda83ebf56cec6c8c0c8f5d70474 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:18:17 -0700 Subject: [PATCH 04/67] feat: 300 - vinpower service (#334) --- django/api/settings.py | 2 + django/requirements.txt | 1 + django/workers/external_apis/vinpower.py | 32 +- spring/.dockerignore | 1 + spring/.gitignore | 18 + .../.mvn/wrapper/MavenWrapperDownloader.java | 117 +++++++ spring/.mvn/wrapper/maven-wrapper.jar | Bin 0 -> 62547 bytes spring/.mvn/wrapper/maven-wrapper.properties | 2 + spring/Dockerfile | 14 + spring/mvnw | 308 ++++++++++++++++++ spring/pom.xml | 68 ++++ .../com/vinpower/VinpowerApplication.java | 11 + .../vinpower/controller/MainController.java | 31 ++ .../vinpower/service/VinDecodeService.java | 35 ++ 14 files changed, 639 insertions(+), 1 deletion(-) create mode 100644 spring/.dockerignore create mode 100644 spring/.gitignore create mode 100644 spring/.mvn/wrapper/MavenWrapperDownloader.java create mode 100644 spring/.mvn/wrapper/maven-wrapper.jar create mode 100644 spring/.mvn/wrapper/maven-wrapper.properties create mode 100644 spring/Dockerfile create mode 100755 spring/mvnw create mode 100644 spring/pom.xml create mode 100644 spring/src/main/java/com/vinpower/VinpowerApplication.java create mode 100644 spring/src/main/java/com/vinpower/controller/MainController.java create mode 100644 spring/src/main/java/com/vinpower/service/VinDecodeService.java diff --git a/django/api/settings.py b/django/api/settings.py index dbf9125a..d5671c9e 100644 --- a/django/api/settings.py +++ b/django/api/settings.py @@ -206,6 +206,8 @@ VPIC_ERROR_CODE_NAME = os.getenv("VPIC_ERROR_CODE_NAME", "ErrorCode") VPIC_SUCCESS_ERROR_CODE = os.getenv("VPIC_SUCCESS_ERROR_CODE", "0") +VINPOWER_ENDPOINT = os.getenv("VINPOWER_ENDPOINT", "http://spring:8080") + LOGGING = { "version": 1, "disable_existing_loggers": False, diff --git a/django/requirements.txt b/django/requirements.txt index b8a19ac5..2e78859d 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -22,3 +22,4 @@ pandas==1.3.4 openpyxl==3.0.9 minio==7.1.1 xlsxwriter==3.2.0 +xmltodict==0.13.0 diff --git a/django/workers/external_apis/vinpower.py b/django/workers/external_apis/vinpower.py index 735c918d..98cb69e7 100644 --- a/django/workers/external_apis/vinpower.py +++ b/django/workers/external_apis/vinpower.py @@ -1,2 +1,32 @@ +import requests +from django.conf import settings +import json +import xmltodict + + def batch_decode(uploaded_vin_records): - return {"successful_records": [], "failed_vins": []} + successful_records = {} + failed_vins = set() + url = settings.VINPOWER_ENDPOINT + "/decode" + + vins = [] + for record in uploaded_vin_records: + vins.append(record.vin) + headers = {"content-type": "application/json"} + response = requests.get(url, data=json.dumps(vins), headers=headers) + response.raise_for_status + + data = response.json() + for vin in vins: + decoded_xml = data.get(vin) + if decoded_xml is not None: + dict = xmltodict.parse(decoded_xml) + atts = dict["VINPOWER"]["VIN"]["DECODED"]["ITEM"] + decoded_data = {} + for att in atts: + decoded_data[att["@name"]] = att["@value"] + successful_records[vin] = decoded_data + else: + failed_vins.add(vin) + + return {"successful_records": successful_records, "failed_vins": failed_vins} diff --git a/spring/.dockerignore b/spring/.dockerignore new file mode 100644 index 00000000..1de56593 --- /dev/null +++ b/spring/.dockerignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/spring/.gitignore b/spring/.gitignore new file mode 100644 index 00000000..a2e40053 --- /dev/null +++ b/spring/.gitignore @@ -0,0 +1,18 @@ +target/* +bin/* +build/* +.gradle/* +.settings/* +.classpath +.project +.factorypath +.attach_pid* +.idea +*.epf +*.der +*.iml +/target +.sts4-cache/ +.vscode +_site/ +*.css \ No newline at end of file diff --git a/spring/.mvn/wrapper/MavenWrapperDownloader.java b/spring/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000..89964d14 --- /dev/null +++ b/spring/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,117 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0' + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/spring/.mvn/wrapper/maven-wrapper.jar b/spring/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..cb28b0e37c7d206feb564310fdeec0927af4123a GIT binary patch literal 62547 zcmb5V1CS=sk~Z9!wr$(CZEL#U=Co~N+O}=mwr$(Cds^S@-Tij=#=rmlVk@E|Dyp8$ z$UKz?`Q$l@GN3=8fq)=^fVx`E)Pern1@-q?PE1vZPD);!LGdpP^)C$aAFx&{CzjH` zpQV9;fd0PyFPNN=yp*_@iYmRFcvOrKbU!1a*o)t$0ex(~3z5?bw11HQYW_uDngyer za60w&wz^`W&Z!0XSH^cLNR&k>%)Vr|$}(wfBzmSbuK^)dy#xr@_NZVszJASn12dw; z-KbI5yz=2awY0>OUF)&crfPu&tVl|!>g*#ur@K=$@8N05<_Mldg}X`N6O<~3|Dpk3 zRWb!e7z<{Mr96 z^C{%ROigEIapRGbFA5g4XoQAe_Y1ii3Ci!KV`?$ zZ2Hy1VP#hVp>OOqe~m|lo@^276Ik<~*6eRSOe;$wn_0@St#cJy}qI#RP= zHVMXyFYYX%T_k3MNbtOX{<*_6Htq*o|7~MkS|A|A|8AqKl!%zTirAJGz;R<3&F7_N z)uC9$9K1M-)g0#}tnM(lO2k~W&4xT7gshgZ1-y2Yo-q9Li7%zguh7W#kGfnjo7Cl6 z!^wTtP392HU0aVB!$cPHjdK}yi7xNMp+KVZy3_u}+lBCloJ&C?#NE@y$_{Uv83*iV zhDOcv`=|CiyQ5)C4fghUmxmwBP0fvuR>aV`bZ3{Q4&6-(M@5sHt0M(}WetqItGB1C zCU-)_n-VD;(6T1%0(@6%U`UgUwgJCCdXvI#f%79Elbg4^yucgfW1^ zNF!|C39SaXsqU9kIimX0vZ`U29)>O|Kfs*hXBXC;Cs9_Zos3%8lu)JGm~c19+j8Va z)~kFfHouwMbfRHJ``%9mLj_bCx!<)O9XNq&uH(>(Q0V7-gom7$kxSpjpPiYGG{IT8 zKdjoDkkMTL9-|vXDuUL=B-K)nVaSFd5TsX0v1C$ETE1Ajnhe9ept?d;xVCWMc$MbR zL{-oP*vjp_3%f0b8h!Qija6rzq~E!#7X~8^ZUb#@rnF~sG0hx^Ok?G9dwmit494OT z_WQzm_sR_#%|I`jx5(6aJYTLv;3U#e@*^jms9#~U`eHOZZEB~yn=4UA(=_U#pYn5e zeeaDmq-$-)&)5Y}h1zDbftv>|?GjQ=)qUw*^CkcAG#o%I8i186AbS@;qrezPCQYWHe=q-5zF>xO*Kk|VTZD;t={XqrKfR|{itr~k71VS?cBc=9zgeFbpeQf*Wad-tAW7(o ze6RbNeu31Uebi}b0>|=7ZjH*J+zSj8fy|+T)+X{N8Vv^d+USG3arWZ?pz)WD)VW}P z0!D>}01W#e@VWTL8w1m|h`D(EnHc*C5#1WK4G|C5ViXO$YzKfJkda# z2c2*qXI-StLW*7_c-%Dws+D#Kkv^gL!_=GMn?Y^0J7*3le!!fTzSux%=1T$O8oy8j z%)PQ9!O+>+y+Dw*r`*}y4SpUa21pWJ$gEDXCZg8L+B!pYWd8X;jRBQkN_b=#tb6Nx zVodM4k?gF&R&P=s`B3d@M5Qvr;1;i_w1AI=*rH(G1kVRMC`_nohm~Ie5^YWYqZMV2<`J* z`i)p799U_mcUjKYn!^T&hu7`Lw$PkddV&W(ni)y|9f}rGr|i-7nnfH6nyB$Q{(*Nv zZz@~rzWM#V@sjT3ewv9c`pP@xM6D!StnV@qCdO${loe(4Gy00NDF5&@Ku;h2P+Vh7 z(X6De$cX5@V}DHXG?K^6mV>XiT768Ee^ye&Cs=2yefVcFn|G zBz$~J(ld&1j@%`sBK^^0Gs$I$q9{R}!HhVu|B@Bhb29PF(%U6#P|T|{ughrfjB@s- zZ)nWbT=6f6aVyk86h(0{NqFg#_d-&q^A@E2l0Iu0(C1@^s6Y-G0r32qll>aW3cHP# zyH`KWu&2?XrIGVB6LOgb+$1zrsW>c2!a(2Y!TnGSAg(|akb#ROpk$~$h}jiY&nWEz zmMxk4&H$8yk(6GKOLQCx$Ji-5H%$Oo4l7~@gbHzNj;iC%_g-+`hCf=YA>Z&F)I1sI z%?Mm27>#i5b5x*U%#QE0wgsN|L73Qf%Mq)QW@O+)a;#mQN?b8e#X%wHbZyA_F+`P%-1SZVnTPPMermk1Rpm#(;z^tMJqwt zDMHw=^c9%?#BcjyPGZFlGOC12RN(i`QAez>VM4#BK&Tm~MZ_!#U8PR->|l+38rIqk zap{3_ei_txm=KL<4p_ukI`9GAEZ+--)Z%)I+9LYO!c|rF=Da5DE@8%g-Zb*O-z8Tv zzbvTzeUcYFgy{b)8Q6+BPl*C}p~DiX%RHMlZf;NmCH;xy=D6Ii;tGU~ zM?k;9X_E?)-wP|VRChb4LrAL*?XD6R2L(MxRFolr6GJ$C>Ihr*nv#lBU>Yklt`-bQ zr;5c(o}R!m4PRz=CnYcQv}m?O=CA(PWBW0?)UY)5d4Kf;8-HU@=xMnA#uw{g`hK{U zB-EQG%T-7FMuUQ;r2xgBi1w69b-Jk8Kujr>`C#&kw-kx_R_GLRC}oum#c{je^h&x9 zoEe)8uUX|SahpME4SEog-5X^wQE0^I!YEHlwawJ|l^^0kD)z{o4^I$Eha$5tzD*A8 zR<*lss4U5N*JCYl;sxBaQkB3M8VT|gXibxFR-NH4Hsmw|{={*Xk)%!$IeqpW&($DQ zuf$~fL+;QIaK?EUfKSX;Gpbm8{<=v#$SrH~P-it--v1kL>3SbJS@>hAE2x_k1-iK# zRN~My-v@dGN3E#c!V1(nOH>vJ{rcOVCx$5s7B?7EKe%B`bbx(8}km#t2a z1A~COG(S4C7~h~k+3;NkxdA4gbB7bRVbm%$DXK0TSBI=Ph6f+PA@$t){_NrRLb`jp zn1u=O0C8%&`rdQgO3kEi#QqiBQcBcbG3wqPrJ8+0r<`L0Co-n8y-NbWbx;}DTq@FD z1b)B$b>Nwx^2;+oIcgW(4I`5DeLE$mWYYc7#tishbd;Y!oQLxI>?6_zq7Ej)92xAZ z!D0mfl|v4EC<3(06V8m+BS)Vx90b=xBSTwTznptIbt5u5KD54$vwl|kp#RpZuJ*k) z>jw52JS&x)9&g3RDXGV zElux37>A=`#5(UuRx&d4qxrV<38_w?#plbw03l9>Nz$Y zZS;fNq6>cGvoASa2y(D&qR9_{@tVrnvduek+riBR#VCG|4Ne^w@mf2Y;-k90%V zpA6dVw|naH;pM~VAwLcQZ|pyTEr;_S2GpkB?7)+?cW{0yE$G43`viTn+^}IPNlDo3 zmE`*)*tFe^=p+a{a5xR;H0r=&!u9y)kYUv@;NUKZ)`u-KFTv0S&FTEQc;D3d|KEKSxirI9TtAWe#hvOXV z>807~TWI~^rL?)WMmi!T!j-vjsw@f11?#jNTu^cmjp!+A1f__Dw!7oqF>&r$V7gc< z?6D92h~Y?faUD+I8V!w~8Z%ws5S{20(AkaTZc>=z`ZK=>ik1td7Op#vAnD;8S zh<>2tmEZiSm-nEjuaWVE)aUXp$BumSS;qw#Xy7-yeq)(<{2G#ap8z)+lTi( ziMb-iig6!==yk zb6{;1hs`#qO5OJQlcJ|62g!?fbI^6v-(`tAQ%Drjcm!`-$%Q#@yw3pf`mXjN>=BSH z(Nftnf50zUUTK;htPt0ONKJq1_d0!a^g>DeNCNpoyZhsnch+s|jXg1!NnEv%li2yw zL}Y=P3u`S%Fj)lhWv0vF4}R;rh4&}2YB8B!|7^}a{#Oac|%oFdMToRrWxEIEN<0CG@_j#R4%R4i0$*6xzzr}^`rI!#y9Xkr{+Rt9G$*@ zQ}XJ+_dl^9@(QYdlXLIMI_Q2uSl>N9g*YXMjddFvVouadTFwyNOT0uG$p!rGF5*`1 z&xsKPj&;t10m&pdPv+LpZd$pyI_v1IJnMD%kWn{vY=O3k1sJRYwPoDV1S4OfVz4FB z$^ygjgHCW=ySKSsoSA&wSlq83JB+O-)s>>e@a{_FjB{@=AlrX7wq>JE=n@}@fba(;n4EG| zge1i)?NE@M@DC5eEv4; z#R~0aNssmFHANL@-eDq2_jFn=MXE9y>1FZH4&v<}vEdB6Kz^l)X%%X@E#4)ahB(KY zx8RH+1*6b|o1$_lRqi^)qoLs;eV5zkKSN;HDwJIx#ceKS!A$ZJ-BpJSc*zl+D~EM2 zm@Kpq2M*kX`;gES_Dd1Y#UH`i!#1HdehqP^{DA-AW^dV(UPu|O@Hvr>?X3^~=1iaRa~AVXbj z-yGL<(5}*)su2Tj#oIt+c6Gh}$0|sUYGGDzNMX+$Oi$e&UJt3&kwu)HX+XP{es(S3 z%9C9y({_fu>^BKjI7k;mZ4DKrdqxw`IM#8{Sh?X(6WE4S6-9M}U0&e32fV$2w{`19 zd=9JfCaYm@J$;nSG3(|byYDqh>c%`JW)W*Y0&K~g6)W?AvVP&DsF_6!fG3i%j^Q>R zR_j5@NguaZB{&XjXF+~6m|utO*pxq$8?0GjW0J-e6Lnf0c@}hvom8KOnirhjOM7!n zP#Iv^0_BqJI?hR5+Dl}p!7X}^NvFOCGvh9y*hgik<&X)3UcEBCdUr$Dt8?0f&LSur ze*n!(V(7umZ%UCS>Hf(g=}39OcvGbf2+D;OZ089m_nUbdCE0PXJfnyrIlLXGh2D!m zK=C#{JmoHY1ws47L0zeWkxxV=A%V8a&E^w%;fBp`PN_ndicD@oN?p?Bu~20>;h;W` ztV=hI*Ts$6JXOwOY?sOk_1xjzNYA#40dD}|js#3V{SLhPEkn5>Ma+cGQi*#`g-*g56Q&@!dg)|1YpLai3Bu8a;l2fnD6&)MZ~hS%&J}k z2p-wG=S|5YGy*Rcnm<9VIVq%~`Q{g(Vq4V)CP257v06=M2W|8AgZO0CC_}HVQ>`VU zy;2LDlG1iwIeMj?l40_`21Qsm?d=1~6f4@_&`lp~pIeXnR)wF0z7FH&wu~L~mfmMr zY4_w6tc{ZP&sa&Ui@UxZ*!UovRT})(p!GtQh~+AMZ6wcqMXM*4r@EaUdt>;Qs2Nt8 zDCJi#^Rwx|T|j_kZi6K!X>Ir%%UxaH>m6I9Yp;Sr;DKJ@{)dz4hpG>jX?>iiXzVQ0 zR$IzL8q11KPvIWIT{hU`TrFyI0YQh`#>J4XE*3;v^07C004~FC7TlRVVC}<}LC4h_ zZjZ)2*#)JyXPHcwte!}{y%i_!{^KwF9qzIRst@oUu~4m;1J_qR;Pz1KSI{rXY5_I_ z%gWC*%bNsb;v?>+TbM$qT`_U8{-g@egY=7+SN#(?RE<2nfrWrOn2OXK!ek7v`aDrH zxCoFHyA&@^@m+#Y(*cohQ4B76me;)(t}{#7?E$_u#1fv)vUE5K;jmlgYI0$Mo!*EA zf?dx$4L(?nyFbv|AF1kB!$P_q)wk1*@L0>mSC(A8f4Rgmv1HG;QDWFj<(1oz)JHr+cP|EPET zSD~QW&W(W?1PF-iZ()b|UrnB(#wG^NR!*X}t~OS-21dpXq)h)YcdA(1A`2nzVFax9rx~WuN=SVt`OIR=eE@$^9&Gx_HCfN= zI(V`)Jn+tJPF~mS?ED7#InwS&6OfH;qDzI_8@t>In6nl zo}q{Ds*cTG*w3CH{Mw9*Zs|iDH^KqmhlLp_+wfwIS24G z{c@fdgqy^Y)RNpI7va^nYr9;18t|j=AYDMpj)j1oNE;8+QQ)ap8O??lv%jbrb*a;} z?OvnGXbtE9zt;TOyWc|$9BeSGQbfNZR`o_C!kMr|mzFvN+5;g2TgFo8DzgS2kkuw@ z=`Gq?xbAPzyf3MQ^ZXp>Gx4GwPD))qv<1EreWT!S@H-IpO{TPP1se8Yv8f@Xw>B}Y z@#;egDL_+0WDA)AuP5@5Dyefuu&0g;P>ro9Qr>@2-VDrb(-whYxmWgkRGE(KC2LwS z;ya>ASBlDMtcZCCD8h+Awq1%A|Hbx)rpn`REck#(J^SbjiHXe-jBp!?>~DC7Wb?mC z_AN+^nOt;3tPnaRZBEpB6s|hCcFouWlA{3QJHP!EPBq1``CIsgMCYD#80(bsKpvwO)0#)1{ zos6v&9c=%W0G-T@9sfSLxeGZvnHk$SnHw57+5X4!u1dvH0YwOvuZ7M^2YOKra0dqR zD`K@MTs(k@h>VeI5UYI%n7#3L_WXVnpu$Vr-g}gEE>Y8ZQQsj_wbl&t6nj{;ga4q8SN#Z6cBZepMoyv7MF-tnnZp*(8jq848yZ zsG_fP$Y-rtCAPPI7QC^nzQjlk;p3tk88!1dJuEFZ!BoB;c!T>L>xSD<#+4X%*;_IB z0bZ%-SLOi5DV7uo{z}YLKHsOHfFIYlu8h(?gRs9@bbzk&dkvw*CWnV;GTAKOZfbY9 z(nKOTQ?fRRs(pr@KsUDq@*P`YUk4j=m?FIoIr)pHUCSE84|Qcf6GucZBRt;6oq_8Z zP^R{LRMo?8>5oaye)Jgg9?H}q?%m@2bBI!XOOP1B0s$%htwA&XuR`=chDc2)ebgna zFWvevD|V882V)@vt|>eeB+@<-L0^6NN%B5BREi8K=GwHVh6X>kCN+R3l{%oJw5g>F zrj$rp$9 zhepggNYDlBLM;Q*CB&%w zW+aY{Mj{=;Rc0dkUw~k)SwgT$RVEn+1QV;%<*FZg!1OcfOcLiF@~k$`IG|E8J0?R2 zk?iDGLR*b|9#WhNLtavx0&=Nx2NII{!@1T78VEA*I#65C`b5)8cGclxKQoVFM$P({ zLwJKo9!9xN4Q8a2F`xL&_>KZfN zOK?5jP%CT{^m4_jZahnn4DrqgTr%(e_({|z2`C2NrR6=v9 z*|55wrjpExm3M&wQ^P?rQPmkI9Z9jlcB~4IfYuLaBV95OGm#E|YwBvj5Z}L~f`&wc zrFo!zLX*C{d2}OGE{YCxyPDNV(%RZ7;;6oM*5a>5LmLy~_NIuhXTy-*>*^oo1L;`o zlY#igc#sXmsfGHA{Vu$lCq$&Ok|9~pSl5Q3csNqZc-!a;O@R$G28a@Sg#&gnrYFsk z&OjZtfIdsr%RV)bh>{>f883aoWuYCPDP{_)%yQhVdYh;6(EOO=;ztX1>n-LcOvCIr zKPLkb`WG2;>r)LTp!~AlXjf-Oe3k`Chvw$l7SB2bA=x3s$;;VTFL0QcHliysKd^*n zg-SNbtPnMAIBX7uiwi&vS)`dunX$}x)f=iwHH;OS6jZ9dYJ^wQ=F#j9U{wJ9eGH^#vzm$HIm->xSO>WQ~nwLYQ8FS|?l!vWL<%j1~P<+07ZMKkTqE0F*Oy1FchM z2(Nx-db%$WC~|loN~e!U`A4)V4@A|gPZh`TA18`yO1{ z(?VA_M6SYp-A#%JEppNHsV~kgW+*Ez=?H?GV!<$F^nOd+SZX(f0IoC#@A=TDv4B2M z%G-laS}yqR0f+qnYW_e7E;5$Q!eO-%XWZML++hz$Xaq@c%2&ognqB2%k;Cs!WA6vl z{6s3fwj*0Q_odHNXd(8234^=Asmc0#8ChzaSyIeCkO(wxqC=R`cZY1|TSK)EYx{W9 z!YXa8GER#Hx<^$eY>{d;u8*+0ocvY0f#D-}KO!`zyDD$%z1*2KI>T+Xmp)%%7c$P< zvTF;ea#Zfzz51>&s<=tS74(t=Hm0dIncn~&zaxiohmQn>6x`R+%vT%~Dhc%RQ=Cj^ z&%gxxQo!zAsu6Z+Ud#P!%3is<%*dJXe!*wZ-yidw|zw|C`cR z`fiF^(yZt?p{ZX|8Ita)UC$=fg6wOve?w+8ww|^7OQ0d zN(3dmJ@mV8>74I$kQl8NM%aC+2l?ZQ2pqkMs{&q(|4hwNM z^xYnjj)q6uAK@m|H$g2ARS2($e9aqGYlEED9sT?~{isH3Sk}kjmZ05Atkgh^M6VNP zX7@!i@k$yRsDK8RA1iqi0}#Phs7y(bKYAQbO9y=~10?8cXtIC4@gF#xZS;y3mAI`h zZ^VmqwJ%W>kisQ!J6R?Zjcgar;Il%$jI*@y)B+fn^53jQd0`)=C~w%Lo?qw!q3fVi{~2arObUM{s=q)hgBn64~)W0tyi?(vlFb z>tCE=B1cbfyY=V38fUGN(#vmn1aY!@v_c70}pa(Lrle-(-SH8Nd!emQF zf3kz0cE~KzB%37B24|e=l4)L}g1AF@v%J*A;5F7li!>I0`lfO9TR+ak`xyqWnj5iwJ$>t_vp(bet2p(jRD;5Q9x2*`|FA4#5cfo8SF@cW zeO{H7C0_YJ*P@_BEvm2dB}pUDYXq@G1^Ee#NY9Q`l`$BUXb01#lmQk^{g3?aaP~(* zD;INgi#8TDZ&*@ZKhx$jA^H-H1Lp`%`O{Y{@_o!+7ST}{Ng^P;X>~Bci{|Qdf1{}p z_kK+zL;>D30r6~R?|h!5NKYOi6X&I5)|ME+NG>d9^`hxKpU^)KBOpZiU^ z;|SzGWtbaclC-%9(zR-|q}kB8H&($nsB1LPAkgcm+Qs@cAov{IXxo5PHrH(8DuEMb z3_R#>7^jjGeS7$!`}m8!8$z|)I~{dhd)SvoH9oR9#LjO{{8O&r7w{d9V1z^syn&E6 z{DG0vlQF_Yb3*|>RzVop^{$mWp|%NDYj@4{d*-@O^<(=L=DMFIQHEp-dtz@1Rumd; zadt^4B#(uUyM6aeUJkGl0GfaULpR!2Ql&q$nEV^+SiDptdPbuJ=VJ)`czZ@&HPUuj zc5dSRB&xk)dI~;6N?wkzI}}4K3i%I=EnlKGpPJ9hu?mNzH7|H0j(mN3(ubdaps3GM z1i+9gk=!$mH=L#LRDf4!mXw0;uxSUIXhl|#h*uK+fQPilJc8RCK9GNPt=X^8`*;3$ zBBo77gkGB5F8a8)*OR10nK&~8CEMPVQyhY>i`PS{L^-*WAz$ljtU%zlG1lm%%U4Zw zms0oZR8b|`>4U1X*9JLQQ>m9MF5%ppoafz^;`7DbmmIENrc$hucekkE4I83WhT%(9 zMaE;f7`g4B#vl(#tNP8$3q{$&oY*oa0HLX6D?xTW3M6f<^{%CK4OE1Pmfue`M6Dh= z&Z-zrq$^xhP%|hU&)(+2KSSpeHgX^0?gRZ5wA8@%%9~@|*Ylux1M{WQ4ekG(T+_b` zb6I)QRGp%fRF)^T?i^j&JDBhfNU9?>Sl6WVMM%S?7< ze|4gaDbPooB=F4Y=>~_+y~Q1{Ox@%q>v+_ZIOfnz5y+qy zhi+^!CE*Lv-}>g^%G=bGLqD(aTN;yHDBH#tOC=X02}QU~Xdme``Wn>N>6{VwgU~Z>g+0 zxv0`>>iSfu$baHMw8(^FL6QWe;}(U>@;8j)t)yHAOj?SdeH;evFx-kpU@nT>lsrUt zqhV}2pD^5bC4786guG1`5|fK@pE6xcT#ns)vR|^?A08G62teHaE&p`ZrCBj_Swt*~dVt=5*RK6Y{% zABqK$X59BnrK3r3u=wxklRnA1uh+q`?T0kE1YhvDWF4OY#<(+V|R@R%tdkq2huF(!Ip+EpZF3zr*|9pmKHPo)Cu z;H+^s&`Ql}u=Jt~ZWj`bAw|i-3#7(2WuRU3DU{BW8`?!O?YO1M$*MMTsaEM!5Jyp~ z!gp6yR4$O%wQ8%dyz43ZPeoJwy;o;yg=S0^Y}%|)to>=N^`!3VMf1~}OZ`Dl$q&|w z9$!i3!i1uAgPTuKSWdBrDr*N$g=E#mdqfj*h;Z}OG`{n245+g;IKfdn!&gF2OtHaD zyGDzj@@d2!P(_Ux)3v;1ABTj__{w*kaRF-1YVU`})Acgk?(T*1YqEve3=5)8bkZK* z!Tus*e$h@^u z>#zV0771Bix~r&h2FJ9)%N{>s>?2tk1$bId)1#G;OKgn-U8jUo^AK;Hu)hQEi}swD(264kAS-SBCD$R(Ro0rh8~Le zzRwxbz_JHDbD+hTX15AWmVw!#rC)-zeZahQQmo6FG1)ah3uuyIuTMof}RO!`Y3^Fxn_-G$23RDOh(@NU?r6`*S?#E50)w zpcsgDZ-iO{;EesgDQq9;p*C#QH(sp~2w^zAJWaUL%@yo)iIL6y8;e_}=dwQc%k%;H zFt5lenH*`}LWd+fPqi;exJeRZgl&nLR%|a!%1x0RQ54cgyWBYrL>sskcAtPxi&8c( zw_K?sI*3n%S;lKiYpveBN08{rgV&-B1NN5Jiu07~%n#%&f!(R(z1)xsxtRBkg#+Lv zh21zX?aYDd_f}qdA`Os*j!eC<5)iUJ&Twj7?*p%vEOGElGhpRZsccM!<k}DeC;TY;rULQs3e}lZyP#UVb=6 zB$Dkm2FaHWUXr7<{R&46sfZ)&(HXxB_=e`%LZci`s7L6c-L7iF&wdmTJz`*^=jD~* zpOZ@jcq8LezVkE^M6D9^QgZqnX&x*mr1_Cf#R9R3&{i3%v#}V$UZzGC;Or*=Dw5SXBC6NV|sGZp^#%RTimyaj@!ZuyJ z6C+r}O1TsAzV9PAa*Gd!9#FQMl)ZLHzTr99biAqA(dz-m9LeIeKny3YB=*+|#-Gq# zaErUR5Z*Wh^e<+wcm70eW;f-g=YTbMiDX)AznDM6B73)T4r%nq+*hKcKF?)#vbv?K zPMe=sFCuC*ZqsBPh-?g!m*O`}6<}Pfj}Y1n9|Y@cUdD5GX_)6Sx9pPfS7 zxkt?g6ZwJ+50C7qrh6dMFmr7qah`FskT_H=GC92vkVh$WfZa2%5L99_DxyM{$#6HQ zx$VR-Wwt!q9JL2{ybEGJr$^?!V4m_BqDqt!mbs=QjHf340+^a{)waVvP0+98(BA$M ztWr&sM=juyYgvf`(SC}+y@QtYgU>0ghJ6VbU}|kEraR&&W%#;!#KI?le%g`e>ZVPiDrneh#&1(Y?uiMo^f5qo@{JEr(p9>8GhDa+PC9yG;lX+D?hQ^fZB&Sdox219zUj_5;+n<0@Wi3@DK`MU8FM!OFJ z8*_mTA-u!Ab#95FRVWTIqAL#BVQGxE_s?>Ql|@0o9vos&r<_4d!+Q6(_270)6#lu$ zV!j$a?_V0I<(3Z=J7C-K0a^Kc1Go9p&T6yQeAD+)dG-$a&%Fo0AOte~_Z&_m2@ue~ z9cKFf-A41Dz31Ooj9FSR`l?H5UtdP?JS=UU$jF#znE1k@0g%K?KQuwZkfDI3Ai)(q z#x_Yo6WR_Y@#6I_02S&NpcP<%sw!!M_3#*8qa+*4rS@x=i{-2K#*Qr)*Q$-{<_(<| z0730e+rubnT38*m;|$-4!1r6u&Ua2kO_s-(7*NGgDTe##%I>_9uW;X__b_k)xlv$; zW%K2hsmr>5e^Z~`tS-eUgWmSF9}Yg8E}qydSVX0nYZMX_x94QK?tw2>^;raVTqstR zIrNAX2`X~|h->dTOb9IrA!i5INpLV}99ES|i0ldzC`;R$FBY5&7+TIy8%GO8SZ37_ zw=^Swk?z+j-&0-cTE|LU0q@IKRa&C6ZlXbSa2vN5r-)*f<3{wLV*uJUw980AFkWN7 zKh{?97GmVu-0rs9FB6ludy|n`gN5p~?y51aJzBg6#+-=0pWdZ2n4xTiQ=&3As-!-6 zFlb|ssAJEJL#s8(=odfz8^9b#@RrvNE4gjuEITzAd7R4+rq$yEJKXP?6D@yM7xZ&^ z@%jnE3}bteJo{p(l`hu`Yvzg9I#~>(T;>c;ufeLfc!m3D&RaQS=gAtEO-WbI+f_#| zaVpq-<%~=27U8*qlVCuI6z9@j)#R!z3{jc>&I(qT-8IBW57_$z5Qm3gVC1TcWJNc% zDk?H3%QHno@fu9nT%L^K)=#sRiRNg|=%M zR;8BE)QA4#Dsg^EakzttRg9pkfIrF3iVYVM#*_+#3X+~qeZc^WQJvEyVlO@9=0pl!ayNOh|{j0j^a z+zi_$_0QKhwArW)sJ$wji;A`?$ecbr?(4x5%2pLgh#wggbt)#T^2R3a9m+>GcrUxU z*u-WTgHAN*e!0;Wa%1k)J_P(Vdp>vwrROTVae@6Wn04q4JL-)g&bWO6PWGuN2Q*s9 zn47Q2bIn4=!P1k0jN_U#+`Ah59zRD??jY?s;U;k@%q87=dM*_yvLN0->qswJWb zImaj{Ah&`)C$u#E0mfZh;iyyWNyEg;w0v%QS5 zGXqad{`>!XZJ%+nT+DiVm;lahOGmZyeqJ-;D&!S3d%CQS4ZFM zkzq5U^O|vIsU_erz_^^$|D0E3(i*&fF-fN}8!k3ugsUmW1{&dgnk!|>z2At?h^^T@ zWN_|`?#UM!FwqmSAgD6Hw%VM|fEAlhIA~^S@d@o<`-sxtE(|<><#76_5^l)Xr|l}Q zd@7Fa8Bj1ICqcy2fKl1rD4TYd84)PG5Ee2W4Nt@NNmpJWvc3q@@*c;~%^Vasf2H`y z+~U-19wtFT?@yIFc4SE_ab?s@wEUfSkOED}+qVjjy>=eac2^S^+|_3%cjH%EUTJ&r znp9q?RbStJcT*Vi{3KDa^jr4>{5x+?!1)8c2SqiCEzE$TQ+`3KPQQnG8_Qk<^)y_o zt1Q^f{#yCUt!1e(3;E6y?>p+7sGAYLp`lA3c~Y`re9q&`c6>0?c0E2Ap5seFv92#X z1Vldj!7A8@8tWr&?%;EBQ_Fwd)8A3!wIx`V!~~h(!$pCy7=&*+*uIzG@*d%*{qG#4 zX0^}}sRN^N=p{w(+yjv%xwb!%lnVTE7l1l6gJwQmq_G83J&Y98$S!r*L8}IiIa2E= zE!0tbOuEDb*No0-KB{zjo1k#_4FHtr{!)>o+Y@bll}Sa6D^xktI0H&l{jKAK)A(iz zB-N00F?~Z}Y7tG+vp)-q*v71(C}65$-=uXx^|R$xx9zZip-V>Hqeyfd(wteM)+!!H z$s+>g4I@+`h2>C|J;PhvtOq)`xm4;CyF}R<)!ma3T{Vf_5|zo;D4YI4ZDBkE(vMeE zb#ZV;n}CgA0w8x!UC2&5Z(K)9bibj#?~>R(72lFx_Am~jS?;7mo~p+05~XGD+(wV4 zEVYnf0N5+-7O+Gc1L!sPGUHv<6=cV8}*m$m`kBs@z zy;goR(?J^JrB7uXXpD00+SD0luk!vK3wwp(N%|X!HmO{xC#OMYQ&a7Yqv-54iEUK4 zVH;)rY6)pUX~ESvQK^w|&}>J{I?YlvOhpMgt-JB}m5Br`Q9X+^8+Xa%S81hO<1t#h zbS+MljFP1J0GGNR1}KwE=cfey%;@n&@Kli+Z5d>daJjbvuO3dW{r$1FT0j zR$c9$t~P50P+NhG^krLH%k}wsQ%mm+@#c;-c9>rYy;8#(jZ|KA8RrmnN2~>w0ciU7 zGiLC?Q^{^Ox-9F()RE^>Xq(MAbGaT0^6jc>M5^*&uc@YGt5Iw4i{6_z5}H$oO`arY z4BT(POK%DnxbH>P$A;OWPb@gYS96F7`jTn6JO@hdM za>_p!1mf?ULJZb1w-+HamqN__2CtI%VK`k^(++Ga0%z*z@k0wYJDqT^)~%|4O299; zh1_iRtc7you(kOK8?Q$R7v-@Qk4+i=8GD2_zI0%{Ra`_prF{+UPW^m5MCA&4ZUpZb z2*!)KA8b--Upp~U%f+rsmCmV~!Y>Gzl#yVvZER2h;f&rkdx{r#9mc8DZMJaQXs?SL zCg3#>xR6ve8&YkP*`Z=lng|Ow+h@t*!Ial*XQg3P;VS8@E1C)VS`?L9N+rxlD7bxC z3@Ag)Vu?#ykY`ND+GvRYTUP&-KDMiqly$Z~uFXt^)4Jjk9RIs*&$?-UPM*d7&m${m zm12kaN3mV1J|c6f$>V+{lvHp~XVW3DU0;cBR>7|)4bo{xa1-ts-lYU-Q-b)_fVVl`EP5X}+J9EzT20x8XIv=m7witdu7!3Lh=KE#OyKpT1GWk{YAo^ny|fvZt<+jmsFs=l*%e& zmRkBt5ccv4O7!HAyv2~rsq*(FmMTm?@TX3&1`nu|7C^F{ad%GLuoX}Rl}6`)uHF_xlx^gVca+mGH4T8u8;q{S*x3=j;kelz^atO~)v!Q_BT z4H6%IA}bvfuk0_vweELeEl8N5w-Q1GF!@f{VKnbyYB2?}d&QvI-j}~RI_+9t9$tC2 z94m=3eLi=sQb^S5;fqP?3aaXc&`}`lq z&M8dOXvxx9Y1^u_ZQHhO+qP}nwkvJhwoz$Mp6Qcq^7M#eWm}!3U@s07hop` zW24|J{t$aB`W>uBTssEvYMyi$hkaOqWh+^(RV_1MYnE0XPgW?7sBDk=Cqs(;$qrPEflqa0ZE?A3cBfW%0RPA235Wb6@=R_d>Sez; z`spwa50bq?-zh+id~Q!T`AYn`$GHzs;jxIw(A1_Ql&f|qP}|bon#H;sjKmSDM!nyn z>bU8l%3DB3F+$}|J^da!!pN|DO!Ndc2J)wMk!+Rr1hes#V}5o(?(yQSphn|9_aU<- zn|nsDS{^x&tweP;Ft`2ur>Koo2IdXJDsr6IN)7vB41Yy-^Wbo9*2th2QA@C zE0-0Gk12YOO?d_Guu6b3&(PIL`d zh4{`k54hu9o%v1K3PGuccez-wdC<&2fp)>`qIIaf)R{5un7-vwm=>LD7ibnJ$|KyE zzw`X*tM0S|V(I3vf454PY{yA5lbE+36_<1kd=&0Xy4jfvUKZ0$Jq!AG4KS7DrE9rph;dK^6*#CIU9qu7 z?)6O`TN&MCWGmUVd1@E2ow2`vZ1A#nGo8_n!dmX77DCgAP1va*ILU+!a&$zdm6Pa6 z4#|*&3dM+r_RJb%!0}7X!An&T4a4@ejqNJ;=1YVQ{J6|oURuj8MBZ8i7l=zz%S4-; zL}=M^wU43lZVwNJgN|#xIfo$aZfY#odZ6~z?aNn=oR1@zDb=a(o3w`IGu&j>6lYxL z&MtqINe4Z>bdsHNkVIu$Dbq0wc#X-xev221e~L zbm8kJ(Xzij$gF4Ij0(yuR?H1hShSy@{WXsHyKtAedk4O!IdpR{E32Oqp{1TD{usJi zGG@{3A$x%R*pp8b$RQo4w&eDhN`&b~iZ2m3U>@9p1o5kXoEVmHX7I6Uw4dn((mFw` zilWrqFd=F5sH$&*(eJB52zaLwRe zz`sruIc=Ck75>v5P5kd>B2u=drvGPg6s&k5^W!%CDxtRO)V6_Y_QP{%7B>E~vyMLG zhrfn8kijyK&bX+rZsnSJ26!j$1x+V!Pyn|ph%sXWr9^f&lf|C;+I^Fi_4;`-LJI&F zr;5O@#4jZX=Yaw0`pUyfF4J8A9wE#7_9!X|_s8~YUzWu&#E^%4NxUA3*jK-F5R3LP2|msHBLmiMIzVpPAEX)2 zLKYjm3VI4r#7|nP^}-}rL+Q4?LqlmBnbL+R8P%8VmV{`wP0=~2)LptW_i682*sUR# z+EifOk_cWVKg-iWr^Qf4cs^3&@BFRC6n0vu{HqZzNqW1{m)3K@gi$i}O(hT`f#bT- z8PqCdSj~FncPNmMKl9i9QPH1OMhvd42zLL~qWVup#nIJRg_?7KQ-g3jGTt5ywN;Qx zwmz4dddJYIOsC8VqC2R%NQ>zm=PJH70kS|EsEB>2Otmtf-18`jUGA6kMZL3vEASDN zNX%?0+=vgsUz!dxZ@~)eU17m4pN3xGC0T;#a@b9Iu0g_v*a3|ck^s_DVA^%yH-wt= zm1)7&q6&Rq#)nc9PQ6DKD{NU=&ul10rTiIe!)x^PS~=K(wX9|?k&{Mv&S$iL9@H7= zG0w~UxKXLF003zJ-H%fGA4Db9{~#p&Bl7ki^SWwv2sfoAlrLMvza)uh;7Aa_@FL4b z4G>`j5Mn9e5JrrN#R$wiB(!6@lU@49(tawM&oma6lB$-^!Pmmo;&j57CDmKi)yesg~P;lJPy9D(!;n;^1ql)$5uYf~f z&GywSWx=ABov_%8pCx=g-gww_u26?5st=rdeExu?5dvj^C?ZZxDv@Si^nX~2qA&K= z2jr;{=L(x~9GLXrIGXs>dehU^D}_NMCMegdtNVWyx)8xHT6Qu!R>?%@RvADs9er;NMkweUBFNrBm1F5e0_>^%CwM6ui}K_MpRqLS0*@lAcj zB6TTCBv>w2qh)qU3*kN+6tPmMQx|5Z0A4n67U-nss90Ec_rDF}r)IR4PE{$8;BSt= zT%6|jyD^(w6a*A5>_|TkMqx~e$n@8{`q?|)Q&Y4UWcI!yP-8AwBQ#P`%M&ib;}pli z9KAPU_9txQ3zOM#(x}*lN8q$2(Tq1yT4RN0!t~|&RdQMXfm!81d0ZuyD}aG3r4+g` z8Aevs3E_ssRAMR+&*Q30M!J5&o%^(3$ZJ=PLZ9<@x^0nb>dm17;8EQJE>hLgR(Wc% zn_LXw|5=b$6%X zS~ClDAZ?wdQrtKcV9>_v1_IXqy)?<@cGGq#!H`DNOE1hb4*P_@tGbMy6r@iCN=NiA zL1jLwuMw&N-e9H(v7>HGwqegSgD{GSzZ@sZ?g5Y`fuZ^X2hL=qeFO(;u|QZl1|HmW zYv+kq#fq_Kzr_LaezT zqIkG6R+ve#k6!xy*}@Kz@jcRaG9g|~j5fAYegGOE0k8+qtF?EgI99h*W}Cw z7TP&T0tz4QxiW!r zF4?|!WiNo=$ZCyrom-ep7y}(MVWOWxL+9?AlhX<>p||=VzvX`lUX(EdR^e5m%Rp_q zim6JL6{>S%OKoX(0FS>c1zY|;&!%i-sSE>ybYX3&^>zb`NPj7?N^ydh=s=0fpyyz% zraFILQ17_9<ettJJt~I+sl=&CPHwz zC9dEb#QFQcY?bk11Y=tEl{t+2IG`QFmYS>ECl;kv=N6&_xJLQt>}ZQiFSf+!D*4Ar zGJ~LFB7e_2AQaxg*h{$!eJ6=smO(d2ZNmwzcy3OG@)kNymCWS44|>fP^7QkJHkE9JmLryhcxFASKb4GYkJ|u^Fj=VdF0%6kgKllkt zC|_ov2R4cJ2QjjYjT6jE#J1J<xaNC>Xm;0SX<`LuW*}*{yQ3c9{Zl=<9NP z^2g5rAdO!-b4XfeBrXa4f{M0&VDrq+ps&2C8FYl@S59?edhp~7ee>GR$zQI4r8ONi zP^OA+8zrTAxOMx5ZBS03RS@J_V`3{QsOxznx6Yt*$IuEd3%R|Ki&zZkjNvrxlPD$m z%K+rwM!`E&Z46ogXCu!3 z8use`FJJ?g_xi?~?MxZYXEu=F=XTC8P3{W*CbG3Wk)^31nD~W>*cJ@W4xg%Qqo7rq z`pUu8wL!6Cm~@niI*YmQ+NbldAlQRh?L!)upVZ)|1{2;0gh38FD&8h#V{7tR&&J}I zX1?;dBqK}5XVyv;l(%?@IVMYj3lL4r)Wx9$<99}{B92UthUfHW3DvGth^Q0-=kcJ1 z!*I9xYAc$5N$~rXV>_VzPVv`6CeX(A_j3*ZkeB~lor#8O-k+0OOYzTkri@PVRRpOP zmBV|NKlJT?y4Q82er)@lK&P%CeLbRw8f+ZC9R)twg5ayJ-Va!hbpPlhs?>297lC8 zvD*WtsmSS{t{}hMPS;JjNf)`_WzqoEt~Pd0T;+_0g*?p=dEQ0#Aemzg_czxPUspzI z^H5oelpi$Z{#zG$emQJ#$q#|K%a0_x5`|;7XGMuQ7lQB9zsnh6b75B9@>ZatHR_6c z0(k}`kfHic{V|@;ghTu>UOZ_jFClp>UT#piDniL(5ZNYXWeW0VRfBerxamg4su5<; z(}Ct2AhR@I-ro0}DdZLRtgI@dm+V`cRZjgV-H+aXm5|Mgz`aZX63i<|oHk-E)cABn z0$NR?(>fla7)Ong28FZSi9Yk0LtYl5lZw5wT!K5=fYT$avgkMKJWx~V#i@7~6_{dM zxDDPIW2l{O2Elv#i^cjYg~lGHRj(W*9gD`(FILKY$R`tL2qo&rtU*c;li!V`O$aV{ z!m|n!FAB2>MR_FVN*Ktv5+2dW4rr3YmfEheyD+48%USM#q6)w%#2}~=5yZE1LLcth zF%VtefH&#AcMx7)JNC$P>~OFuG6sK}F7V$D7m!{ixz&inpAVpFXiu^QruAw@Sc7Y2 z_A^V(2W_+KTGRp2aQSMAgyV#b3@{?5q@hPEP6oF3^}|@8GuD6iKbX;!LI!L=P#Za zL$Zuv#=x3fseRMZ()#SQcXv->xW`C|6quwqL1M&KByBj z2V`}(uL4JB-hUs6304@%QL~S6VF^6ZI=e-Nm9Tc^7gWLd*HM-^S&0d1NuObw-Y3e> zqSXR3>u^~aDQx>tHzn9x?XRk}+__h_LvS~3Fa`#+m*MB9qG(g(GY-^;wO|i#x^?CR zVsOitW{)5m7YV{kb&Z!eXmI}pxP_^kI{}#_ zgjaG)(y7RO*u`io)9E{kXo@kDHrbP;mO`v2Hei32u~HxyuS)acL!R(MUiOKsKCRtv z#H4&dEtrDz|MLy<&(dV!`Pr-J2RVuX1OUME@1%*GzLOchqoc94!9QF$QnrTrRzl`K zYz}h+XD4&p|5Pg33fh+ch;6#w*H5`@6xA;;S5)H>i$}ii2d*l_1qHxY`L3g=t? z!-H0J5>kDt$4DQ{@V3$htxCI;N+$d^K^ad8q~&)NCV6wa5(D${P!Y2w(XF!8d0GpJ zRa=xLRQ;=8`J2+A334};LOIhU`HQ*0v4Upn?w|sciL|{AJSrG_(%-(W9EZb%>EAGG zpDY?z1rQLps`nbCtzqJ#@wxU4}(j!ZQ{`g`g*SXlLah*W9 zyuh)UWoRCknQtd~Lk#BT_qjwj&Kw8U)w=owaJ;A5ae}3)y>{neYNS`|VHJdcSEBF# zBJ6a;T)u;^i#L~LVF-X7!E$SggILXMlsEy~v}K*DM2)f@U~g|Q6I-Pss@)`>fgFWx zsq&7pe!|VA-h;@=fBF{(mR1^{1>ukTYUdyF^#A+(|I_&nm{_xaKn3h4&yMyym2k-wMFg(s@ez=DPmuB%`| z6;e@HQKB(|!PU1sW)W6~x|=8m6rL~4dQ9LTk|RzL-_(_77B4I~ZG=q7K%qHiv!FD8 zmt;Vnhb{ymaydv2V;X-5p zTt2ln?kaB9&(dH_X70^@rrCfz)nwfa9LYTHXO(IPcTEf$QiEhTpl??L+`Eetyqof8 zzl=q)?KdYni!C_9b8Z3xm7r5<5ZG-0uA`u^7Dm7k4mAsQ(rkoWy*^DZJa~#y6+hNG zh?7{D9$a9LS`a@SvZ5?C{JUHovWU9KI}z8YV4pWftx21v*Q;MpU{+b@>Or(}pwO^fu0qA3_k_Bo2}lIxvmMhucG-o>O=+R6YxZ zjs!o%K1AA*q#&bs@~%YA@C;}?!7yIml1`%lT3Cvq4)%A)U0o1)7HM;mm4-ZZK2`Lj zLo?!Kq1G1y1lk>$U~_tOW=%XFoyIui^Cdk511&V}x#n4JeB7>bpQkYIkpGQRHxH$L z%tS=WHC~upIXSem>=TTv?BLsQ37AO88(X+L1bI<;Bt>eY!}wjYoBn#2RGEP49&ZH-Z_}R_JK_ z>o*_y!pOI6?Vf*{x-XT;^(_0}2twfk`*)_lLl0H-g|}BC?dm7CU|^-gNJ~rx z($>97WTKf71$?2|V$Ybpf~Aj@ZZOcb3#uRq51%4^ts-#RMrJhgm|K3QpCsPGW=2dZ zAr5-HYX!D*o#Q&2;jL%X?0{}yH}j*(JC4ck;u%=a_D6CrXyBIM&O#7QWgc?@7MCsY zfH6&xgQmG$U6Miu$iF(*6d8Mq3Z+en_Fi`6VFF=i6L8+;Hr6J zmT=k0A2T{9Ghh9@)|G5R-<3A|qe_a#ipsFs6Yd!}Lcdl8k)I22-)F^4O&GP&1ljl~ z!REpRoer@}YTSWM&mueNci|^H?GbJcfC_Y@?Y+e4Yw?Qoy@VLy_8u2d#0W~C6j(pe zyO6SqpGhB-;)%3lwMGseMkWH0EgErnd9a_pLaxbWJug8$meJoY@o-5kNv&A$MJZ=U z^fXPLqV6m3#x%4V*OYD zUPS&WHikdN<{#Yj|EFQ`UojD4`Zh*CZO4Cv`w^&*FfqBi`iXsWg%%a< zk@*c%j1+xib(4q^nHHO^y5d8iNkvczbqZ5;^ZVu%*PJ!O?X-CoNP*&tOU!5%bwUEw zQN?P*a=KKlu{`7GoA}DE=#nDibRgecw>-*da~7&wgow}|DyCJq!-Lp8a~(zR@tO1 zgu(4s4HptPGn(HmN2ayYs@g+yx1n`nU3KM{tQHhMHBw7f#gwru$=C()`aKZAl^dYc ze7fC)8EZEXOryk6AD&-4L+4cJ&M@3;;{R)mi4=`ti7IZByr^|_HNsjcNFu?mIE)jD za2j)FPwRY!R_YR-P?URm0Pti*e#5jmfK)6EvaKCT{h)kbJl{AGr1Ekt}pG?^e z*botRf-RsB8q10BTroj{ZP**)2zkXTF+{9<4@$aNDreO7%tttKkR3z`3ljd?heAJEe<0%4zYK?};Ur*!a>PbGYFFi(OF-%wyzbKeBdbkjv^i9mn@UocSS z4;J%-Q$l`zb&r*Pb`U;3@qkc=8QaPE9KwmlVwAf01sa*uI2*N`9U^3*1lLsM9dJ(4 zZBkU}os|5YT#Z;PD8xVv!yo$-n{-n4JM5ukjnTciniiT`(cZ6sD6~67e5_?8am%!w zeCLUxq~7x-!Xg#PgKV&caC@7mu<86am{WaXo(lAemt4~I$utSp(URWpYNo$RvU*$N z#%iiA+h`(E;BUg;=I!#EaxO89bUK3*v5Nc3GPmURC5TqzC|))DsFNtJICH6oBW6#q z+B(N{ey+^mk_{!@ z)VhAWXG=_0j|0f9iJ;c404PiIFqK)(AD05Xh`Fk`r$^b`v+>*g+_+h@r)e+ELJ45) z?20~u<}HQyQ5AsBz(teF9!!_GLXnm{5Z0e{Ki*@!=&3x4-RcjBn##DDzHJ|KSZ5(E z9=tFZ)p~-}x%9sCY27)2i>(E-^OiYT?_)a;yXAGR$y+E`myMd;xDA#_Q49t*E}&ql#H~|x z2J2R1_#2lt91NnF!uqW%_=HlbF?A{B{n>}9$g5QF!bh_a7LTU~Jyz}7>W5{_LAov{ zy2_dmGy)d)&7^bJyUjEw%3xj{cuG0Eo zwL*XQB*Oi=r&HIIecC1%lbE;Y-*5|cL955S+2@uR18JDL<0;;Uc2Q9JEyo1R!!sz_ z#BqnkGfbLP#oQJk3y}nwMd(3Tt^PVA#zXnYF7D0W1)#+`i?@cm}fBkKD z+Mpcuim53|v7;8Tv(KraEyOK`HvJq^;rlNzOjIbW&HJDFqW>doN&j7)`RDv#v|PQ+ z03WnB4Y4X@Fe-@%3;He*FjY1MFmkyv0>64Cp~FIDKQTwmFP~_CxZOf{8gPy}I<=JC zo%_bmue&$UU0|GG%%99eI!m#5Y1MD3AsJqG#gt3u{%sj5&tQ&xZpP%fcKdYPtr<3$ zAeqgZ=vdjA;Xi##r%!J+yhK)TDP3%C7Y#J|&N^))dRk&qJSU*b;1W%t1;j#2{l~#{ zo8QYEny2AY>N{z4S6|uBzYp>7nP_tqX#!DfgQfeY6CO7ZRJ10&$5Rc+BEPb{ns!Bi z`y;v{>LQheel`}&OniUiNtQv@;EQP5iR&MitbPCYvoZgL76Tqu#lruAI`#g9F#j!= z^FLRVg0?m$=BCaL`u{ZnNKV>N`O$SuDvY`AoyfIzL9~ zo|bs1ADoXMr{tRGL% zA#cLu%kuMrYQXJq8(&qS|UYUxdCla(;SJLYIdQp)1luCxniVg~duy zUTPo9%ev2~W}Vbm-*=!DKv$%TktO$2rF~7-W-{ODp{sL%yQY_tcupR@HlA0f#^1l8 zbi>MV~o zz)zl1a?sGv)E}kP$4v3CQgTjpSJo?s>_$e>s2i+M^D5EfrwjFAo(8E%(^ROV0vz0o z-cg0jIk24n!wxZainfH)+?MGu@kg$XgaMY-^H}z^vG~XC7z2;p2Kv`b^3S#b5ssMOJ7724v>S36dD zeypxJ<=E~sD4f5wX060RIF-AR0#{Z z=&y$r8A-e6q18lIF{@O9Mi%dYSYT6erw!@zrl=uj>o(3=M*Bg4E$#bLhNUPO+Mn}>+IVN-`>5gM7tT7jre|&*_t;Tpk%PJL z%$qScr*q7OJ6?p&;VjEZ&*A;wHv2GdJ+fE;d(Qj#pmf2WL5#s^ZrXYC8x7)>5vq_7 zMCL}T{jNMA5`}6P5#PaMJDB2~TVt;!yEP)WEDAoi9PUt89S2Cj?+E0V(=_sv4Vn6b z_kS6~X!G;PKK>vZF@gWpg8Zuh%YX^2UYPdCg7?EH#^gkdOWpy(%RnXyyrhmJT~UJw zAR;%Zgb6z(mS+o9MT|Sc6O({!i0pzk;s9?Dq)%tTW3*XdM3zhPn*`z45$Bg!P4xfy zD*{>30*JsSk?bQ-DgG62v>Vw-w`SA}{*Za7%N(d-mr@~xq5&OvPa*F2Q3Mqzzf%Oe z4N$`+<=;f5_$9nBd=PhPRU>9_2N8M`tT<-fcvc&!qkoAo4J{e3&;6(YoF8Wd&A+>; z|MSKXb~83~{=byCWHm57tRs{!AI<5papN(zKssb_p_WT@0kL0T0Z5#KLbz%zfk?f7 zR!vXBs36XaNcq5usS7<>skM_*P$e*^8y1ksiuokbsGFQ_{-8BAMfu!Z6G=88;>Fxt z|F-RU{=9i6obkTa0k~L#g;9ot8GCSxjAsyeN~1;^E=o5`m%u7dO1C*nn1gklHCBUw z;R(LgZ}sHld`c%&=S+Vx%;_I1*36P`WYx%&AboA1W@P;BvuFW+ng*wh?^aH4-b7So zG?9kFs_6ma85@wo!Z`L)B#zQAZz{Mc7S%d<*_4cKYaKRSY`#<{w?}4*Z>f2gvK`P1 zfT~v?LkvzaxnV|3^^P5UZa1I@u*4>TdXADYkent$d1q;jzE~%v?@rFYC~jB;IM5n_U0;r>5Xmdu{;2%zCwa&n>vnRC^&+dUZKy zt=@Lfsb$dsMP}Bn;3sb+u76jBKX(|0P-^P!&CUJ!;M?R?z7)$0DXkMG*ccBLj+xI) zYP=jIl88MY5Jyf@wKN--x@We~_^#kM2#Xg$0yD+2Tu^MZ1w%AIpCToT-qQbctHpc_ z>Z97ECB%ak;R<4hEt6bVqgYm(!~^Yx9?6_FUDqQQVk=HETyWpi!O^`EZ_5AoSv@VbUzsqusIZ;yX!4CsMiznO}S{4e>^0`c<)c~mC#*{90@+T@%EQ~>bovc8n_$bvqkOU7CrYe8uI5~{3O7EijeX`js z-$LNz4pJA7_V5~JA_Wl*uSrQYSh9Wm($%@jowv^fSPW<~kK&M*hAleywHd?7v{`;Y zBhL2+-O+7QK_)7XOJAbdTV-S`!I)t~GE8z+fV7y;wp#!wj75drv;R*UdSh(}u$%{VSd0gLeFp;h6FkiVz%g=EY3G#>RU;alRy;vQmk*| z@x-ba0XKE%IyL4OYw6IXzMiS(q^UDk=t(#XgkuF`{P?=k8k3r)rmhkv`vg@kiWd34 z-~t+1aV3SabTbG=nQYs>3~E<}{5@0g**LAWi*~SfRZhGcgP{e5T!0M7CU}`f@r8xI z0bx%sI!?5);-wG+Mx&S=NRfIi>V-wP(n&$X0Bhd)qI^ch%96s6&u7qpiK8ijA=X_R zk&|9f$GXf-;VgnrxV83Cp-Q!!sHH`5O^o~qZu!xny1t?(Au(EAn)D??v<1Uo;#m7-M@ovk|()C(`o>QMTp}F?> zakm3bHBKUjH-MHXDow7#Z|@wea1X9ePH;%YA)fCZ9-MD)p^(p!2E`aU9nmJlm;CXQ zkx~$WQ`Yq{1h5k>E>Ex{Z=P=)N*0b8_O({IeKg?vqQ)hk=JHe z5iqUKm!~mLP0fnRwkCO(xxTV@&p+o8wdSP$jZofYP}yEkvSc z5yD-^>04{zTP7X44q9Af&-wgt7k|XtncO&L@y-wFFR44RsPu57FRvIBaI^Pqy_*DV z@i13CsaR5@X@xH=NT3}T`_vsy!a02n80eQqya=-p7#YW`Jc0z!QglGg`1zeg6uXwI zsB~hlNMo)kFL(V3Q1<%8yoI6X7ncn-&&Uh3rL@S(6@wKAXt6Wr=a2ObI7}8$D-FoI z>AJA>WsBEMi5ba6JhJ%9EAi&ocd(ZsD|MsXwu@X;2h#|(bSWu@2{+c7soC`%uo{sMYq&Vyufb)?OI59ds)O+kyE8@G z@tlpNr0UO~}qd0HQve6njJ zda2+l$gdX7AvvGhxM6OToCuQ|Zw|9!g1)O+7>~{KNvASjp9#Cqce-or+y5xdzWL3gLWt2oa+T(I+{j(&bF1laUsJB{fOgE-B}qslaS>C z)TjzG8XecbS%a+?yT!0QmTex?E478;D|sL*oS4C-g0Tq(YoH|eyxJ#1j088C|U-w5id`%Sz7X_w#l+U9+)$|2no<}5J zRb_9@0esSr?n}HvVGbD5@$p$8k4?qOe-GNOk3-K^Mw>Xg+drCKi5@$GTeijpI;;IG ziD<&go`ptLC&^<0jw^l0aY?_pUUK+xp#0Bk66iQ29vpR)VBE{JOJ&OL^gKsN<&t<| zCMLTYMSDG5Ie9O>6Dl#T{@cscz%)}?tC#?rj>iwQ0!YUk~R z$rB-k=fa9x&631Z9Mfqj_GRoS1MzqSMEdaZ2!isP19Sr>qG8!yL(WWF)_&{F)r>KnJGSciSp!P0fqHr+G=fGO02Q#9gHK zpwz+yhpC4w*<9JO@#(MdkZcWbdCO5B!H`Z|nV?UtcBo96$BgX+7VYMwp@b-%;BrJu zMd*K!{1txv{kHKPDs9?WZrz_^o1Tq2P=+=|E=Oy4#WE{>9}*9(apqhmE`&AeBzQgQ zELFLCmb~q|6y0FCt|B}*uI*ayZ#6=$BpGtF{Jfye#Q>FZ?BPnk)*Qmd?rNG^tvFUU z_b&antYsZnUR6Q9tQUy81r$&ovT#fy;(Db4F&M*C=KxQgHDrRcVR#d+ z0(D|*9#u`w_%2o3faI{?dNd9$#5nj1PROHNq z7HJ(;7B1ThyM>a@Fo^lJb2ls2lD`}ocREH|5pKN;$>gFyM6k)kZG;lA;@kSJIqUhf zX%dhcN(Jtomz4(rNng&1br3Xx33EvCWz%o8s;SpRiKEUFd+KJ+u|gn|J85dZ)Exc&=V|Ns8Xs#P>qv6PX&VAJXJ(ILZO!WJd0 z`+|f5HrEj~isRN7?dBHotcPI7;6W48*%J(9 zftl1Tr`bKH*WNdFx+h;BZ+`p!qKl~|Zt5izh}#pU9FQKE97#$@*pf38Hr8A+`N+50U3$6h%^!4fBN zjh^cl#8qW5OZbvxCfYzKHuyeKLF4z^@~+oqlz9(Hx8vypIiUlt!(vs}_t#4@nh$s; z>FYERg*KD#Xs+W4q-V-IBQK!)M1)Aa+h+V+is)z!_=gEn&^ci7<DEEmYcoSh?WdXUsP7O4)&lQXA(BVM5jI8s6;mO}94AC0gG(`>|T)yuV1l~i-ejCCt zoejDhX0nrZDP|x9u4zp%S2UeDzV`o#pBGu1tZ-$<9TIbN=ALwhQ0=9S{8#}Uu8n-~ z5~xIvUhLSz@c@0|me$CdZCpZl(vQw@a0Y4^{T0w_>pOkwI^x4KkBf3qGmm)nG|Ps5 z_XTY~^b^mL&_*yjl~RRIi&eS(>y?y}O4-)nWyTEPpQAb#Xz8SnnfIL+nAcNL9nqV9 zRL|eyF)RKI5-kJO6}>Q89XmgY@b1&!JI>g3ryZ@jN2v3vm7O`AL!BTWNouJzV+$+Y zYY}u%i>K6=IYU2O$2TAyVjGt?wgF9xCj;?EK(8fWu!!~48`3u^W$eUlCh*91PLxu1 zRY(F7Q3s7h$Q-p&L$ucN}it*-9KR z_<wHu?!dav0$P+PI3{J8?{+l|n&2YMLV2 z+hRta$A5WpCXl1RNbYBsX8IGX{2v>U|8_I-JD56K|GexW>}F_e_g_1r?08v8Kz{V$ zT=6aGMk>ibvRO@Yrc@ezaD0%ydHkXGHrR{7>q~~tO7ChJflwa4-xL|@#YIJejC5VT zInU4CjQ9V0+lClQY=vh^s4MadwQmk7li{54Y;Ht}gkZOIh9(vfK?3kXLoD72!lHD# zwI-Jg|IhT=Y#s|tso1PWp;|aJ2}M?Y{ETyYG<86woO_b+WVRh<9eJu#i5jxKu(s~3 z4mz+@3=aNl^xt{E2_xewFIsHJfCzEkqQ0<7e|{vT>{;WlICA|DW4c@^A*osWudRAP zJut4A^wh@}XW4*&iFq|rOUqg*x%1F+hu3U6Am;CLXMF&({;q0uEWG2w2lZtg)prt` z=5@!oRH~lpncz1yO4+)?>NkO4NEgP4U~VPmfw~CEWo`!#AeTySp3qOE#{oUW>FwHkZ3rBaFeISHfiVSB7%}M) z=10EZ1Ec&l;4 zG98m5sU!pVqojGEFh8P{2|!ReQ&hfDEH2dmTVkrS;$dN~G2v-qnxn^A2VeHqY@;P} zudZD5vHtVvB*loIDF1M7AEEvS&h0;X`u}!1vj6S-NmdbeL=r{*T2J6^VA7F`S`CDd zY|=AA6|9Tu8>ND6fQhfK4;L3vAdJPBA}d6YOyKP&ZVi%z6{lbkE|VyB*p1_julR^k zqBwjkqmFK=u&e8MfArjW-(Ei8{rWso1vt5NhUdN|zpXqK{ylJ8@}wq-nV~L4bIjtt zt$&(1FTIs+aw}{&0SO4*sa0H2h&7g}VN5uYjfed5h7eGp$2Wu*@m9WIr0kxOc}fX9eOWh zFKfV>+SD$@kESKYm{F*J90XQjr$!<~v(J%&RMuQM+6CkmnYZDGlOUdq}%)VA& zl#acS%XE2KuX~7IamK`og@C`21~*cEEc#PZM6HT*Veb_l&Ej~j0zL7p0Eo`mMu(=X zJ$v;&Lya75I4C^saKROgfi(fdP0C$GM3WyZn%mm3yEI>|S&O(u{{S<}ihUp#`X&_z zmQBma;82#`C;dR5Sx09e07FvtJLhZ{9R~|$FCdU6TDNUwTc9kNct?8e@o2MpQDrkg zN?G+aYtTjiUPA=RX5o{4RYu}6;)ET>TcgL^VpfIpluJ|lQR(_)>6k%L^FZmoK-Wm- zR5qy0P)hm8yvqOL>>Z;k4U}!s?%1~7v7K~m+gh=0c9Ip_9UC3nwr$%^I>yU6`;2kV z-uJ%y-afzA7;BC7jc-=XnpHK+Kf*tcOS>f5ab2&J&5hIOfXzs=&cz|Qmrpu6Z);`R z0%3^dioK5x?o7t~SK7u5m{dyUZ#QUPqBHYn@jETeG>VU=ieZuJ;mm^j>dZM7))cw?a`w8R z%3M0R=kdOt^W^$Kq5Z%aJ(a$(*qFpy^W}Ij$h+Jnmc9eaP(vB@{@8t zz=RQ$x4XYC#enS$fxh@;cSZ|D%7ug;0z{C8I8h{KocN-cyv3UG_nk99UNS4ki^OFkYea`q`rs zG@qdMI;4ogcd5Tr`di1JBg4I*6CFvCID_2SN5&)DZG&wXW{|c+BdQ4)G9_{YGA@A* zaf}o^hQFJCFtzt&*ua~%3NylCjLtqWTfmA-@zw;@*?d&RE3O8G&d;AVC|rZrU}jx# zC-9SF`9;CbQ(?07o8Q9E12vi)EP@tOIYKEKnO@-o!ggkC)^#L-c40iZtb4Y-cS>$I zTn~+>rn*Ts>*y*z^b3-fAlne+M-*%ecrI^rmKAVv23cB`aWD?JDJ5NIafRvRr*~~C z)99Afs`BPK!5BFT)b_^8GyH*{22}yDq;be`GnPl=vW+ITnaqzl(uYOHhXi}S!P+QZ z4SwfEPuu&z4t#?6Zaw}bvN{;|80DfxCTuOdz-}iY%AO}SBj1nx1(*F%3A-zdxU0aj z`zzw9-l?C(2H7rtBA*_)*rea>G?SnBgv#L)17oe57KFyDgzE36&tlDunHKKW$?}ta ztJc>6h<^^#x1@iTYrc}__pe0yf1OnQmoTjWaCG`#Cbdb?g5kXaXd-7;tfx?>Y-gI| zt7_K}yT5WM-2?bD-}ym*?~sZ{FgkQ9tXFSF zls=QGy?fZ=+(@M>P3Y>@O{f44yU^fP>zNzIQ0(&O$JCd_!p?2;} zI6E1j@`DxzgJvqcE@zgapQ?tophO14`=14DUZ*#@%rRi``pi0lkNgidSsHGjXK8gO{drQoNqR&tRjM4>^DtW`)fiRFO4LE=Z+nCBS~|B3gZsh`Y?-$g z@8@Z$D7C!L9l=SWoE;(+*YirPLWvBd$5Ztn3J3EaGM+#pW#@{3%yksGqy(2Bt5PVE zf*fICtPp77%}5j#0G8<=v=)LR>-a3dxja8cy3m$=MZ2#$8mbLvxE%NptMd+L?mG`v zF1cANFv17DqP^P5)AYHDQWHk*s~HFq6OaJ3h#BUqUOMkh)~!(ptZ2WP!_$TBV}!@>Ta#eQS_{ffgpfiRbyw1f)X4S z_iU`lNuTy86;%!sF3yh?$5zjW4F?6E9Ts-TnA zDyx5p1h$Z3IsHv7b*Q{5(bkPc{f`2Wfxg*Z#IvQ;W_q9|GqXGj<@abo)FyPtzI~i25&o zC!cJR%0!}lLf^L2eAfZg7Z69wp{J?D6UhXr%vvAn?%)7Ngct4Hrs@LZqD9qFHYAWy z4l=2LI?ER&$He2n`RiG&nsfLv?8$Cl)&d8a-~-N`I|&EPa@Y=v@>0Gl?jlt>AUY;H z`**5bpS#VGhdp4pKbf3iEF*>-eXg_$bqt5Dc%q0+)R50>zd^l7sN5R5Z)Ut+oz-8_ zJ`Z9HE9(=wRTD)T=%GZTEi9K5naPzlfE$|3GYGLRCLsnqLi8Sc6y&iskqA&Z$#7Ng z7Q@C0)6k;J$TlQ+VKZ5)-Ff_BNoIMm+~!@Cv1yAUI-U!R)LHc@+nSUzo$GlRb+8W< zYPG%NFfr;!(RlnvBbN~~EpT6Xj5*^Z&73tdIQ$LZu`vkfzdTKa5|JJtQ_rm4g$9LO zKtgYVdW=b<2WGM3I_j|Rd8gZ3j;)S#AT(aP^d>9wrtQS_+K>pZDX^?mN!Z>f^jP@1 zlJ;i79_MgOAJa`%S9EdVn>ip{d!k6c5%zizdIoB9Nr!n`*X#%6xP1?vHKc6*6+vKx zmEt|f^02)S_u_wlW_<`7uLQU%{wdH0iojOf_=}2=(krE<*!~kn%==#0Zz`?8v@4gP zPB=-O-W=OO3tD19%eX>PZj3YfrCt0sEjgTd#b$buAgBri#)wW14x7QcHf2Cneuizz z368r7`zpf`YltXY9|2V{stf8VCHgKXVGjv$m!hdDf0gi`(Q!(Pyg~FO28Vr#!BYP| zI)qG2?Ho=1Us9dTml}-ZOR?g5Vk)f+r=dbCN*N1=qNfG>UCLeA8pd3Ub-pRx1b3FA zEn`CIMf`2Mt3>>#3RkE19o}aMzi^C`+Z>8iIPHSdTdmjCdJBtNmd9o0^LrJc9|U9c zD~=FUnSyghk7jScMWT|SHkP(&DK$Z=n&lGm+FDTpGxfoIyKV)H6^nY~INQ#=OtIT! zyB*J=(#oHf=S)MNOncW->!c0r0H#=2QzobO&f@x&Y8sYi-)Ld;83zO$9@nPPhD}yt z{P`*fT@Z(?YAmF{1)C;o?G@dfd2$c+=Av*|;P@Yz1KnclB-Z-fJQ-=+T*g>0B7!g# zQH{dHt_%wj=wlmT&m59)TQ~xK)gB6f^EY$=1zcbGf~Q>p_PzDCHR6lndGmqPY2)&w z$Th^K%1v@KeY-5DpLr4zeJcHqB`HqX0A$e)AIm(Y(hNQk5uqovcuch0v=`DU5YC3y z-5i&?5@i$icVgS3@YrU<+aBw+WUaTr5Ya9$)S>!<@Q?5PsQIz560=q4wGE3Ycs*vK z8@ys>cpbG8Ff74#oVzfy)S@LK27V5-0h|;_~=j1TTZ9_1LrbBUHb?)F4fc)&F7hX1v160!vJc!aRI>vp*bYK=CB(Qbtw7 zDr2O^J%%#zHa7M5hGBh#8(2IBAk}zdhAk$`=QYe^0P6Bb+j5X)Grmi$ z6YH?*kx9hX>KCI04iaM_wzSVD+%EWS)@DR&nWsSBc2VIZ>C(jX((ZiV0=cp}rtTO&|GMvbmE4FpBF5Rd z6ZG=>X&>N3?ZN2^11pXEP4L?XUo`qrwxgQm4X~RCttXmZAhnhu4KDK=VkKq?@@Q_Z za`*xyHrsAEsR zV(7)2+|h)%EHHLD3>Qg{>G|ns_%5g5aSzA#z91R zMDKNuIt@|t?PkPsjCxUy&fu^At*yUYdBV!R_KOyVb?DO&z$GLJh9~b|3ELsysL7U6 zp24`RH+;%C(!bWHtX&*bF!l-jEXsR_|K~XL+9c+$`<11IzZ4>se?JZh1Ds60y#7sW zoh+O!Tuqd}w)1VxzL>W?;A=$xf1Os={m;|NbvBxm+JC@H^Fj$J=?t2XqL|2KWl$3+ zz$K+#_-KW(t)MEg6zBSF8XqU$IUhHj+&VwsZqd7) ztjz$#CZrccfmFdi_1$#&wl~A*RisBaBy~)w|txu1QrvR1?)2mb&m2N$C(5MS%hSX)VJnb@ZGXB5^%(<#1L@ zL^>fBd+dEe`&hxXM<0A9tviIs^BDkByJdc~mtTYr!%F7Q1XnK2$%h$Ob30*hSP$Bt zDd#w{2Z%x^Wpv8!)hm>6u01mY!xmPgwZ#Q0148)SxJc3Udt!-&}eRO^LN ze26pQB!Jhg&Z>#FD>`C`sU44><=v>O>tJdLs!HPpV#AM32^J@Za-9J(CQjKxpzXao zQfRkWP%g9P8XV21MmoHfx{DICLSc*t4qVeQL9t}&Pz0rM}YTba@XsD=XMW@FxFM{QYQJHvM(JsUSa3mcTUl9^qcVA zBveO--fqw%{#QGR1vy;x88+qMcgzmcYc#8U`CPPt6bl?uj%w_`b~9JliftnOa|ziW z|6(q&STs_*0{KNa(Z79@{`X&JY1^+;Xa69b|Dd7D&H!hVf6&hh4NZ5v0pt&DEsMpo zMr0ak4U%PP5+e(ja@sKj)2IONU+B`cVR&53WbXAm5=K>~>@0Qh7kK*=iU^KaC~-ir zYFQA7@!SSrZyYEp95i%GCj*1WgtDId*icG=rKu~O#ZtEB2^+&4+s_Tv1;2OIjh~pG zcfHczxNp>;OeocnVoL-HyKU!i!v0vWF_jJs&O1zm%4%40S7_FVNX1;R4h^c1u9V@f z`YzP6l>w>%a#*jk(Y82xQ@`@L(*zD&H>NY`iH(iyEU5R$qwTKC5jm4>BikQGHp^)u z-RQ`UCa70hJaYQeA=HtU1;fyxkcB2oY&q&->r-G9pis)t$`508$?eDDueFdW=n5hJ z08lH$dKN$y#OEE@k{#|<%GYY=_c~fHfC@pD54KSP9{Ek@T47ez$;m$}iwR}3?)hbkwS$@p2iVH0IM$lB*XYA+#}-re|UNzCE)SOYwy z=Y!fkG4&I%3J(_H#UsV#SjHulRIVcpJ`utDTY{k&6?#fzt~@Om=L(vs6cxAJxkIWI z@H7)f2h%9!jl@C!lm+X4uu;TT6o0pd7 zteFQ(ND@djf#o2kTkjcgT=dHs7ukmP0&l8{f;o3JuHGd2Op*?p7?Ct=jA*tIg{MZk z$2Lsc0e8Tdcwrjx|_Ok?9uB3Il|^2FF%X#ck}WoIvrzQXN%kT$9NI{79Wm~gZ3`8I+O`)`n30feZ( zDO-fl6IG3c^8S;Y_M-)+^CmM0tT^g0?H#>H8!oC8W%oU!~3|DJ?)~LT9*&GAQG13zOGq6gs*={cu|(V7{R$y@{-iV*9q@AD(#Ktb}J&3&k|5Djs$)9WM7!6#EaJ_ilvbfUvyh8c?-{n zfuFrC0u6}UJZ7aj@(cNG_(CKgjQQTA-UK@-MVmick zot}6F%@jhq(*}!rVFp5d6?dg|G}M*moyLriI!PQDI;E1L1eOa6>F9E6&mdLD>^0jJ z09l?1PptuV65gm=)VYiv<5?*<+MH~*G|$~9Z3XEy@B1-M(}o&*Fr9Sv6NYAP#`h{p zbwbUE3xeJ;vD}QMqECN)!yvDHRwb7c1s6IRmW!094`?Fm!l~45w)0X`Hg+6Y0-xf# zSMemBdE)Q=e^58HR{kWrL5-H0X6pDu%o{0=#!KxGp0A;6{N5kI+EoY_eTE%2q|rwm zekNeLY-R?htk!YP2|@dbd8TWG4#G)=bXlE{^ZTb^Q$}Er zz)Fp)ul24tBtQFIegdI37`K$VR3tVdi<(fIsu{#QMx=$&CK9M8oN%3Mk;>ZPd-;Q- zn|sSKSnc-S0yrw#TlA$+p{J~u=u98s>IoL@cNLOxH=+1m?;t1bR$vR=M$US&Z8DO3 z_&zhQuId1$wVNsS=X?&s(ecIi#00o{kuPs6kpYkL$jMyGW8U7mlCVaZeEL=HsIxqm zFRLxWin8B>!Dc#9Z#t0RNQiR-@5J+=;tC7|1D*~rxcwHa5iIVD@99cCFE@BukUC-S z^iJdt?dwU)kH2VY9?|zVShMbZctzFRz5Q4tiXa^>@U%jDYq}$rSyc#p2wXr}mc0qq z^lT>$y)N(Qg0dwmEwTopneoU(y)>Mj+f{iHM0o|>ZtCg-itPj4addYz??aE)Rp&hk z_SI)%XeSf=SjZq18h!Cc>Xy&EynnxdHQ){(x@g|ZA%`3LU^KzX02c5N;F#tEk1)7v z(|V9tO3>?^X|kQ*rRBf4>mWW2$-Lx})|M7z125&VHcxsCqB!<$l1F$zCrJ+nm0f3Z z%Hq^=SKpHyV2@Y*Cu2x>fXC0SscnR*($zEB{KOniJcpn@e`PMH*_Q6*0Z^8RNCEvZ z+UU9!927p9YZ&g=bnUvQUZcdisyn;-4;ACXOe-Xor9K8Qbp{ldE17+G@VQT+9ZJQ*9dZoXfU2ue|mMhrrZk2R7&~YjFW4`BTq45UwVc6JORKU)wBCTanITh0GD}s$`C5pb(9{b9 znwee6j%?-UV)_7opOioCf5@C?@w^@g& z&68+oMmV;5JW@TT63&CSDrfYL2$L)pVseDtAwPwleEM3F^-Ufn3PpfxFmx6o zQ`Wq9x#d$e`VKn5LOXNsrqhGao7~|s(u~drPrZ+;aP!C%z4NskZstCbAibD}O%8Ij zb~C(taxco~WzJLxhL1T}3ctXMbV6}_z=IZN9L0|SxLSe`$X`<)BhM`$1&&)e_}fCh z=idVL<+u6Vn{&ksP*ZLlMo$fC`dtzF_?~L?4Rril2G4%v5^7sUa^&8aMtMX&mtapl zD(dW|cisM3fqMaB`8?QbkyiUl2g>hMB5EoS&IB8TdoC~)b$nT=`%GgU`k-)+8}`)F*~I~DXMaTP%kZftx11~?iALs5J+&Rom#p%Y z>dH}-euH4u=_V3hc6^*2WMtL!9%yRTJ93p}@aV0zdY*?xchFI>m+UivV=;aMFp0P~ zwB8P)wvV6D-GL?6hJ#g7Hy7=2i^&Od#S=j!;Rc_yjO!*4aN7{vqzg2t-R|Dav%_NDk z`H_FVlSi==(~f-#65VmQ{EE92x<03lwo5p)s=ZJ^L7PlS>132Whr zR6v~t(#I+(`usYLCoO;Rt8j&b^5g_xgs*98Gp|N}b>-`HtVm)MscD)71y?(K6DRCZV26RsHPHKk)EKKZA%C99t3$t^B0-k5@?E>A-YMbFe?>ms?J?_guHHNU(;id*>xH zTrtam+Aq?n@-y@uY@A?hy?1qX^eLu_RaH4Ave?A8NapgQF=C%XI7wlcCf4<6BRo_% zBXxxc*A6-3CruF?3i8HOdbc%>N=-iiOF+9HX|ht6SCkz;A^am&qi_I&qk1B(x<=(m z>QG)nswCOLl_1{SZ@_eE#m^qb6#6DoMsB*)`17ui+XvF%(}|J4G$z2G*;E!1ERnAH z@q%=#uV6kBddqy4=g>!VTV)9*1=i{wJ}Ep!I*?)uJdA(LwE?(!?;}_u=^M2NShWC_ z*7l4aBJ=!QVU2-iehgb`$vOI8zkm{W%QO~?xOD;NgI;Iqa3#^$^U5D&McReLe&qs# zR<^@QpR4#W~Laz+QBsPt@3L#KF`Yr8}jgHe;5(cfpQ=;Zjtbt;c%y^#-m=hqOT z;KAYakW+$w0&F}>K10&SiPcD9SrDOuczj@U#W})5jGU-_htU`U6Q%wdy((%?J}y+$ z=$4jw1N nJo)qTxG{D(`3*#8tY|67hJRF;)r6F|#I`Ar6I0aafRa=kr-Z0I^}9xf^u;G5iEQCbpv3b#S#%H|HYHsQaHK$! zU#3Fpz8*^pK%RRmX<_09eIVziB0jOgPgFnI-*QcwEBtBiO#v!>{W1cLNXyw3D9M|A z*oGy(u8BkDA1c;MsXmpK^-~pl=We^RYnhZ4bz*)Q)C2G+E3tgx9PzU0T>c|1ilS!T zyE=bz`=wskDiOi!@!l?Y))#%{FM`}7r~X)i1)1*c6_2Q!_1{)fp%cS|YF+Q-CB%d< z=zYus`Vt@Mx*a7V)=mpLS$-5viaKgNB=+zN657qy0qR94!cTtX-Z%KBCg4OKw7b=t zr=`7q5Ox=lJ%!G5WIyNQC1xpqYU0{!I$hyrk!6%De$gp<_*Gc?ES(OwY8U^)Kjgc{ zSlhpXDb|;{+y9`u{EuMz54rlky2~p6xX2>MV6BZ&k`$q%q7v(xYps2wr9e8^4<;CB zc)eAT~B^rjzO6<4BDDH;il6 zFsM8jL+agQ;zazW(uiQjM%fPf2N~_p{cy29XP11_lQFpt`t#9nlk}>fv((FZt-dBa zuMIc4HmPHW04n0TTG9ug9;&OV9euL$Ib|+M7}}L~z4e%%%b|r~6OQj(S2d7XfYn#xp8;KQ55UYu#gY*De5j6Cc z#R%?rqwpy7I1(kpU7B*Pq=etXeYUn04jg%ZPjYqQNa$==yTG=6KX+=;i2Xg+kjV2T*Gc!(ef z`Q4fR*TA=M5-}z+s%YO+!K{k}S**ic&>o4_Tmv$EQTOp7F6TXPCj-UTXy?OQ=%*y62Qajk{rXbR%jMCOFMiVE3KekQa4xR}B%=iPtd8BXo~q$OX_ zSp910{Ew;m|GATsq_XiJ3w@s(jrj^NDtr(Dp!`Ve!Oq?|EJ9=vY2>IfrV{rT%(jiY zi}W@jA2iqd=?q>s;3%?@oi7~Ndo3Ge-2!zX58j(w&zVlPuXm3rcHb7O0RsM|!Ys(b zh(=*&Aywo3vuJoWZnU!u2_4bNkDTc&&bCYc%T zM~~xYxS#3KXFzQ@OXdc%9QDOxqiTd_> zT;(DX9{5dIuC4pO_xy+3{Ov)1I7j!Z)6&nHUvTRP>VU5dm#849icG)cvl0QOPkCIzG^lOp4#UcNr`VhBp(Ha%8@KPlvT*5u!v_$b#b~%sn3K{mu zaxeD%Q~{;Lw03ZAq(Pc-IVj>n*h3l2{sqioCMGatQY0kx zi`1(WWDQ=;gmLSGptEQ%UFC)th@|71<8eiRtX&Mx@#1q#nMF_BMfQdS>!!Qkx2o}= zuqRi?`UOX5P3fP%M+71Q$ctH4Av}bXED#fQ`KR4!b~60nsAv^*M7c-x`|~B}XIuq% zlqIJOf>WvlhQ@Uw$du|14)tZ?; zPNZ|xZSwp1y+d4sut8E4*l2JWR|~o0A9vD-?zC-w zDc@=wE1YKb*OMSi_Kx}&w;#h3>sHp|8^hnA3w?-WK)X?@Z2dgV7`9Cupf-B2RE4x^ zwlw+~!V9C^tyb`J;m2}ksD`w}G9`yu(^--{SQ+wt^Fu4Li~Fft!3QO`upSkAU?o;# z(1Q%GUVWbbkTK-M=T+ULkk3s6Dc9`G4CO6|=&-S&D+rbJQ$`Y-xL~ol;kc(l)VbU>{&>bV+*?ua;$bnDc29RW+Ig16)Vf6=L|fMR_P2b7>6}0 zdlB#-gj|j*C~M=F^2=K*k~=tl6YM3SXXi&K-`EvEXnWz&4D-^hQRBJI3gKKDj^6|> z*WhHSim1qAffNt60Mve9lfw^+&0bx-AM0%j>QP3%W=S@(l=(nrJ678mRQ(#+sI@d{ zdb#5fo#T;hK7xJ=M58wZf|?DHwD%!OZ3JrTGV5#{cfQwuiMvz%!CQ}CubJ7`z?@rSF<+KHNV2goc)a6hP0oHB@3LLKSH2w{um&J*z1Ka2 zLIR>lvOvh>Oxe%?3A@v<_T|}${zf_&@C~^FCo#jB(W9VLO?DX{)n(BQ0(V0`mI|9Y z#U3WwxixJkU_NTvA>5q(A@r2dnEXJp#6B=pww$XGU}~1~c``UKqQb=^*2P|4Dq*_! zhY^i61Sy%T5$Td0O6^C>h(xVvT!}Y##WeT8+s+Uuz=7)~V$>!zU;%d>H)rm*6^IrsCma%|cifwDLk_ z!^W2voQ)D;I$=v2E>iSaBw!d7aD+|LWl2iD!cBw`Q5p1~fk_xGiPi8e^mY&#viTAk zmaKL8m;JQ4bY(n6uBZt02z#noMMxTfF-RzjKre-c+@B)#J3pN-Zv7F}JtAwNk3j?OkpVCL6W1)Q$FLAj zGI!tX;g`O{%pt=0|q54Jyj##w*4e*|_;Us2Tn?!#^R(>u}|FAw1G_ z#wQsagnj9$TAC`2B_XgB$wNq~Sxgl?#0+QWWcB{G`c6~&SosbtRt}Tukw`TQ!oG1= zYyL(y<;Wh+H24>=E}Gs=Hs2%fg;&Qdvr74{E!R?Bd zIRQ?{{xkLJ_44P@y3^#(Be%(pk%$liKbUUo76wSoVfJmt9iTKL3z{uW6L&?jYg>EY zsx{kRiW@q%<$VZvbS(TKKTO4{Ad6l^IeY(F^3}=mX9|FZmQ`~RErNxlBPl3ast}W$T4V?SW=6kIGn@-^`qJv| zZXwhK4Kl1a4E}nLI`rdOi?^pd6;LZ-|8G&INHgOeC5q{_#s+SXb0r(;5ryHFsoTJD zx$VtNDh=-Tx3t!NTlk=hgAaSM)#U}e>_-Ex(|JoX*hWmBPPdTIa-2(BIOUJ|Iddy| zwY*J%z%W$}*;uSoB!BIJB6N6UhQUIQE_yz_qzI>J^KBi}BY>=s6i!&Tc@qiz!=i?7 zxiX$U`wY+pL|g$eMs`>($`tgd_(wYg79#sL4Fo+aAXig?OQz2#X0Qak(8U8^&8==C z#-0^IygzQfJG4SWwS5vko2aaOJn*kM+f1-)aG{T43VJAgxdP(fJ4&U{XR90*#a)G8+clOwdF?hJ?D) zmxu>0>M|g_QRHe_7G|q6o`C>9x4xd$Gl7lAuR~+FtNid=%DRsnf}YI*yOToWO%xnP zY*1G5yDnTGv{{xg5FhWU65q3-|-(+-rJ2WCeSJn(7Az>ej4Jp9+l-GyZ_| zJ8}>iA4g|}q1AhEEv#uWR&$g&Uyht?fVU(qk(j?^D`))s>oG08pow!f>P1u71P%oL2)UC4GeS87&G?{)NE;D=my1Q9{~;y zJULE=bG6jXE28Y11YmoZoo945`MM*`v%5b=_02*0cwzDve#3(4M}NPt`)?SCa|7*q z-94ks(R6WH-l9fE4m4}10WSu&O`|;ZCIT%vL$_pbABY!}s33@~gIvZ0H4co|=_-T$ zF#lC7r`89_+RL9wYN=E3YwR?2{$^ki(KKd>smX(Wh*^VmQh|Ob5$n_%N{!{9xP~LJO0^=V?BK8AbCEFBhDd$^yih$>U z(o{RReCU{#zHSEavFNdc8Yt<%N9pd1flD{ZVSWQu*ea1t#$J5f6*6;tCx=&;EIN^S}*3s%=M#)`~=nz!&Q0&{EP|9nzWyS<#!QxP;!E8&3D}?QKh^ zqGum|+;xu9QE=F#fe2ws5+y1Igr&l`fLyLKry=1}(W+2W`waeOR`ZXlW1B{|;4sE3 zn^ZVlR11hiV~p<~TaSen8I~ay#7Ql=-_|U@$8yjZsZ=Vi+^`JV2+kn+oiSUi%omO_+7}saXnJ9 z5ETilbag(g#jZPopCgJu+n@(i7g}3EK2@N zd64$77H5a`i%b%a^iRjMaprwzWz(`=7E6QY)o)gek7H)yZ-BLw^6FAoHwTj9nJtWc ztKaytMlWGLg29W{?gr|rx&snb@XyvR_}x3fmC>d=-nQp5ab3*whTw}DfUcKlMDDx` z-%?ek^*|Kqooy#>2lfklZ|jN4X$&n6f)RNNPl(+0S>t(8xSeOGj~X0CGRrWmm(WXT z))DDW_t&y$D#2`9<-+JT0x1==26*gpWPV~IF=rePVF%e-I&y$@5eo~A+>yZ&z6&7> z*INESfBHGNegTWga&d@;n;FSCGyW?}e_Qw#GTLHo*fWxuuG@I~5VA!A1pOdRTiPA~ z^AGe(yo=9bwLJD}@oDf$d+34~=(vIuPtOKiP}obDc|?@hY}J*@V|UynBeAkYa?S{@ z_f$U=K+>deTAi&=a*xv>Ruyw$UsTWY=Yn=xjf;s)6NQu>_niQ_idmzIwuL`Scf)f= zyzK?D5a5)^D@H&qN%F6Zd0JeXX*Knbe~VLe^gi|?JK67&mB4jrapV-$`hCQT;C{%T z*pjxB+Y|~LD9bmMN%Iq}S$F$x1yWU7@GcR91V8h;!O2I5MN_rq*gRx(k8T!1WSDTp zr9eJO4$~H94aG^6k5p8k=kFJ>4lnY0q_Bsa$@vTRW6uY?slH|Qt)Yu6Yun&pfJ zBi!h;6x?FDs&79#PT*HSCEUsKws#s%TFy*=2PAfb`>gEPBn+D-WdfXA?MkB=<8kb_ z1+4D11mdHG0EcAyg4dneLtfJ8)RyHQl@6hWJNe(d_EjyCHf7%Xsd)S4A-4COz{G@% z5xQ!P>AS@H@;4Ws)N91)3A6PleMe2<& z!(zv#%Uc?N`(Xmm)OJPYt)BM`nRjoWA&P0Yxl@c9Y02zlPH1J5l$nhPrMwu=atkz4 z)a-1+OEL;d@ctx=s<<+3Sv1VYy0RYmiji|#hy$66#`5;u~BkH4^$EGZ-Y4xyZ=%3KuaeLYKAUr$xMtIh_5mga> zPz<#G0mQ7IxEw-yO}BueN}RaFlg$RwCDB)vLF$wDu%qZyLYsPKdcbHD23$qn9i#JFqIo#OK?u7db2-$GatzO!On87%}Br};~#}n zziVB;qf_4(K$u>Qyz$ln_kBGS!CD-t4Y}9oxL@7@Sx*?NOAzdeINUD>Hl#*V%pfA; zSA`==YatS*G*crJ3`3ll4)vKss&)UtY#7ZxiVoG%9(4<%`WWcjX2jV(^g7Yhj+h5J z$5=?S=tuCyEt74^6jo@6y|@~N>&cVfFNtaRl=)Gm!vR;Bc$3-;ySCI$%kdmjQ|si` z{$q_YCe6vjy6re9jGN|`43D``)1PODtz0)vhV4XV36nVpOnMx2uM%qZ<3TtcI%>BQ zf0(J`{JqPPJxw>k#&nIvoZ5e9Sno)B2r+E0G} z@&M|zf4E0Q$O*NBR2I;?i7N} z@2^Su#`%qeX}m3cbSojiLk#84kvW1fICNPS`OyT0SpUoA0(s^2m~J<^eKE!dhJx_N zG_T}0&(<*an>oF=@?6?55g&IxSgY3?7|@pmDRE6gJyJNPH6un~%0hZ@?h=hI6O$b^ z)29#<4$E)cE-5IFbRpk9JVrw$$966UDyw;Iym4OY4Fc!&s1ZH4BJ1-$9<)Zt1c)N- zU^&9hsk6z?3%<9kGKHW|6~k;&cghtWz`oz`_YjVuvy;B;T67=L2c6=8`7WyTBv*QH zNv*bo1#KOk{O&)@&pkd*?v+kcJ8tM>AGx$~WMhH{L40_N=bkrVg+^p!H)IqXCQf2_ z0fPig=8CEo>p4vE(nc^DKbZ|9_Xo}$i4zJ`jVh95; z5%aNP3@``=EJ=Vt9U`y+$YtX;%OPzgZ_3+;+mh{p#W&y4-%%Bf`LhOy-*kB0qnB^m z_nBTz_b?-`F$*ymByshU>D)za2g`0j^ioo;A#QeL@x3@|+_!=YXA5f6Xg(Ack&WOg zJ<2i|Fd6OmyH!@YSMVxb;=M)ZDhBt)4`5T*>cUXWPG#%@$&*>K&u3#|`fm2mj*FKVf?du{xZ}WKWETTFhq6_fO$PS5(ItF=3~pFp~*j z!ys1<4EL1)#{`mz@gW|t-FpPkd%pK)n_Rb)F;z7cQ6dym_>YI3&e!=!m006oS3Mjq{q ze%hNzW=G0jpfl2K(x`CDuZCsJV*hm9T~%5n7R_g}VFpk`G((D^MWVMAmRp--T{`P; zwMgD<;e`fm`g3|fPns|6qnd{|FCHY*YAguXH(?%sx%4+Gu|Y)_8mk4EljxmP+MP`* z`SUbI{TCIN2OV+$y#g->Jqv#$wL;}4xJmah#$0`v^ughM_XjTA$B}ux)JZuY5-GW4 zKy440I+w=ZtE-_i+0xImq}vyzD68?8;94-5L~_O6Ty>X3itdA-x?6P(c4jkr+f!H( zUDeqiG>3bn^Sf8(`_YwqPeJ9&-@OCQZm4X{FfRMeBtN4E9Ca@;GVpU*L>lVb;@=PH zTQvTr?^jKyCKh&ZVOI*<y%T*Aw(XCPrFC=39*y$A`FSzxBiQ#W+uW10d8&gYp4{teh;^p@anft+z$5!Hv&@h0X-@xJG>hbTCxjDwMiWK@1b%8wYL6BrV zT41m}tX8g-`P@vj4T!Mlk8F0S!MA`^J=SCy9-jdwDe^hVDa`WwyI^H@ryt=F5y6>b zT8&iI6&j8edAfX^ycgWbnMZQ26Q~`LmdEScKC8|~$Jgyw(>18NAQ$9AwCRmri!96L zp^)b0P2CR-9S%cG$#rU}MXnx21T#031o>2VrDs@sa-FpjfvgLPW>Q&LHUoNOtmkt# zoDZ=5OGp{^vO~=p29^`aXd8K?(+f-bW`N$U;-o;%f?RcR!k02Nod2h^^8ly%Z67#E zC3|IOuj~^YBO=Fklo@3mvd6I{Z*&FZ>iq* zxh|JuJoo2$p8MJ3zO@dQ;%1#~Mrm48 zB0053{1bDi_a@jo<4!@!`w4}B(&Qb`~IeSBh zu+_yIYl2Wgk+?x4pCmAM>x_SqBPUj#c`C`k>_fp@qPlAAwD$!zOxRkL7;=|nu(#ut zyF^;&hm-D_;ji{d6rOloACu5*NkF4IC3@rifMG(|^Skv$H&^YnYL*rpw=UCi;JOuz zN*NX(7wZXS4tF@6PIWAs%*j!$RoL*3sh)}iry%thDvN5AUM888q_(>|Tzt|Yea3AyMYBgm$H_`F^v2%)bux)3s znFIEBDK;-JS5SH|;1?afJb<*=c5puu=w%tv#ihn*R!^Hd$KWAp4$#`joJ*)$kNtZ z2Al6h>Z>(u?3tmzA4^d+jLKx{97!Pb4;CX&u;M||**7zXI7hO6nrdMx*Xa=|-`#1^ zBQ?Ha&7cd7hN=%y4yUp?zl8~Lo;%mQrDe8!ce-W_K94FFMN*g(w8q-_K5S+c0{o29X&PzpV;UJE^!xnFc%b@>kvW4m#xiOj-L*DadC&2N#0Us z;<-(m1WB7$=j6hjcPC6JB)D3T2#IC`ibu#yi!uK7W2!j|Z>~RaJ*&XXy#ytIk2DIp z5?Qd^s90_?ILjU#>ZWk5HXts}grg_!Gmgm!d?eLGR7xEP zvTCrslV~94ym5_i<5oqy(@@?wN}lIdtiY8=?|Ng!XeYnly`@9wCGx2S$3x|0x8T2h zz7A85Vb2>s44rKpI_4Y7_Pnd2^mYj2%^jM|Du>u4`^Psda^JIP%*DK6bo`Vf&f{!% zDTYCwF5Nhi=)QhU2$@eQv&ZzxsX+Hl+gP6kW|e!n9IU2>Vh~cioI{>4WvR}t*4Hpz z%5z?HjLGoka}Q3AbX9AkY|Yjf^M(>@tBAI9JO5pDCQu0R3Nns>)LC#vB2p96C*?K? zvX$un$sBDx$1=+NNj*@Oa@u*b@O*XBr_sg@8sCUq-|LK!MUmC)epklrv}5O_^<{NP zX16|c$9Wtbks3y7geI^tF5oRZJu;v zwkW8j+8Ccxo9stEDOT_Go&j%$KCgVO7pm+^%PKEPBZqbMw%s@732XS{cX+wCSjH1s z5)bc=g**<^NNsroY` z?}fHHlgu^B?2r{^^gQ&j zbF~T((>|Yg&C5WKL8DCnl1}Z3!YHFW2S1|;Xr0`Uz-;=FxEwYc4QpeAtnm7^f~uzX zl;xA!?>MLR?tL80Iudm;mi{!ewL91KhG7Hsa-XepKi<2mc6%zf0GwtbfJ1Zf-<@Xu z#|XWDzv|04t)&9Id!UxAAkN{t5qC%%8-WV3i;3duS19%m2||Y{!3pR1=g|zQYAMqc zff)_2nj-O4wfxy;UNM?|Uieo!^J$A*uDe>@V(NKH;KS;Y_dtE8${p>RdcrW;=2*fj4~d?OG0l-(g?ik}vz} z)5-wDppVts>K-=|@{=!53?=8)Jw#RGpS_FWpbwtn}{v!JEJ$q-sr7F6&OPBuI# zuVNFMPte79XgEu!P&qRq8u4J>r%$l-IQ00Lin90(_KtC)aR_de zxN=pY2<1b29_^AG2WJIGmmX4rv3$!`l15{e(H!1^+x9voZ6;882YAE12q7+lgy+>) zj|s0CyzI9=Mo!R}&LXB`&DYpZ7c?0r(&KNV+~TULd0y^e;G{KVR4nL0KvU9mr8&$^ zxrM-9P8zE`J?aZ(iB~Rz<{vvnk2HaZU#K$aVFfYnbAXVUOLU#As5JvS%+26 zi$sNuPY}dLGUS$0g&;oBqhzv2dY`l3@6Na403M!Sh${B|7(y|_cONa;6BrtUe@ZzV z7SThtHT8k?Rwc)(Z}@BP#H@JJHz&GR&M=E@P9KJ89yQKmRh&I~%vbL1L-K3E>7>CH z)Y!=jXVb1iPrAoAZZ3}3wU*5~nrV!ZjL5zqJ<@NwjHCZC>68Cc<{&E_#S;E*jOdjtg?uKN|l`P8sjz&Qf7a^z9 z;{3-8T+H4y99_zc;JYIvs!sk$G}` z??mt*Mm9Z@glCZb!X?!xXD-21sFDPEpZOK{sbQseQ$%6~b;n+*z0hRoR}0Pe>B|#t z$XrVcXv8M|q*Z8MY&r9J0A=d^1bHpjrUXu)qEj~$%%=gZp`^~%O*lzxUquG^p6;n; z^(3HL+hx4gRP?4N*b2p9!^|2~rcw3!9nQj$vmZusbXYz_x^AVc`3qBFm(jS9ueU5h z^AnNnbswfQ2Jq=W=T+p-V|nQco@bOAH$pLQZ+BKH8E$iM>IDz z3|wc?QP`yI=X5YTlp8h}%p6{Deq?S0QD$Ug>ih1SdPZg237Rl{S~=Ha4~-ckMoIWMn+X@@`V6 z#HHZj>MQbt$Qqp*9T(cjc^lxZ7UO(>PwzF-qEr(wo`vaulxdall|KP`7p4gd`23&Jy=#sAes*0diLB(U$Nx46VQvP)8idSs8^zaV91xw*O-JMH=)FoJshRob|_)O)ojtfP))WHCr(;*2;VMQ75^ zfN@a^f#o<|*9X;3IcGodLUz-3i~FAu+zI4c5h+nW^h_!^)b*B_xw-l4O$TB(ixaqW ziMoa%i=BeS<-F45kMO;Tw|FWa`G2c!SuOA3CbowPhF6csf1|&qqugUrj;UgGHm| z;j^yoH?MZhR;AYOW_XW2Lg2j%%ejL)B@*bUMD`g<#Z${1+fa57r7X82 zcqY-cfPnK%Y^3@szRner zt)bBToYCph6Jv*W+&t?&9FG4(Iu2w46 z4B#AcFy_^J@f*6<{>CN}Sj969*DYV*e7<61U>GoN{tz!Do90+jApFueVY_IW(MQF; zl?4yA_(MvMwN&pWKVyg{3uU_+y6RMdot2vu%mC?st=N0pf-~JZXE?3JFf)j<{1xsU z`2ephz)#HzsWEP!inHm2hI(V(~@W zY7gGU-lO52cHD&SY)>QHgy$=>^X%u0TQZfCizro!*weMyvZC=;MWOawdAx~`3C*W` z%^#^$uRP;gyqEE0<(i8xcQY$oc+6mY#z{-XFxsO1(cN8Y)>p;^q9|5bk`Z*p|c!?(rErw#y;yT(%@c7trQBv6cj)$3>pI z>tz+;IB?D=aQV=s(n)o63*yn8dX1m7#Z4G{%fF@K2o5n3jxR~mU?nzMi#;}8e#(>{ zy{Z4!AI)jZ8TY;nq1aq}tq;~=zzoTv)er06oeX3;9{uP{LWR*2%9cmE%S^`~!BW>X zn3PZFTf3g*dG68~^1*q@#^Ge(_8puPEFLD8OS|0b2a{5e=N4S%;~f3tC>F6UxK#v9 z)N-#Mv8=ePCh1KsUKD1A8jF_%$MPf|_yCN9oy%*@um6D{w*2|4GY zb}gafrSC+f=b*W{)!a!fqwZ9)K>fk=i4qf!4M?0v{CMNTo2A9}mQzV=%3UT&i{3{W z>ulG#M!K7%jPf6Mjff9BMslgQq3zIogY);Cv3v;&b#;^=sh#(Bn%W)H*bHNaLwdpq z85%fUTUJJNjYO_426T2TBj0D{6t zw&S_HZ|C?pI_2q(9Fas&@uJs6nVX;P*5K#6p|#)_(8PM-{L(;2wl`ma{ZAd5gA)?y z>0GSLoK<*FwW+G8@-M3vcffg7I(qm7lzF)n`Q9iCvp*mn7=|CjlpG{x z&r0n}XLWZ!>=lynUr7D`6n`7a_ZgT< zm!i;&?Fb0Q2QmqmCHfZ7ex=_tU~(7b)L?RIvPyEAU=gLIZ-VTAA~WR00yKyTXg^(G zqWLZJs!FnQYMOH3*fN&Tn(IKMLf{Ki?pRo8zZJ6YVyj)y0^)-sR}2-)%mI(Aw2AgT zbbp1T{qB(OSNJd0cVBH^tI>HR(q+#*lmi@LWe*rZz&M2h1L_=50uZ1e*n#E*`6?aw zj`ka&JpceRGe@}Ey1)Q~O}0qHRg4K_u>4e1arvJ7Q9!=t5AuzG`n=a-f0}{+lnCE#zu$`oVn44eS&T?N*wz~t~E&oQDBrB_MSg z_yVrQehWbD0xHX|v-hpselAu;O7s;P*!uAT`dr~}Lie=tknaGoiU?;*8Cwgala-65 zosOB4mATbdXJFujzgA4?UkCKE093A1KM?W&Pw>A?IACqg1z~IZYkdP70EeCfjii(n z3k%ax?4|rY(87N&_vhsyVK1zp@uils|B%`(V4e3%sj5f|i(eIhiSg-fHK1Pb0-mS^ zeh?WA7#{hhNci5e;?n*iVy|)iJiR>|8{TN3!=VBC2dN)~^ISSW_(g<^rHr$)nVrdA z39BMa5wl5q+5F@)4b%5-> zA^-P20l_e^S2PTa&HE2wf3jf)#)2ITVXzndeuMpPo8}kphQKhegB%QO+yBpDpgkcl z1nlPp14#+^bIA7__h16pMFECzKJ3p4`;Rf$gnr%{!5#oG42AH&X8hV8061%4W91ku z`OW_hyI+uBOqYXkVC&BqoKWmv;|{O|4d#Nay<)gkxBr^^N48(VDF7Sj#H1i3>9138 zkhxAU7;M)I18&d!Yw!V9zQA0tp(G4<8U5GX{YoYCQ?p56FxcD-2FwO5fqyx@__=$L zeK6Sg3>XQv)qz1?zW-k$_j`-)tf+yRU_%fXrenc>$^70d1Q-W?T#vy;6#Y-Q-<2)+ z5iTl6MA7j9m&oBhRXTKr*$3gec z3E;zX457RGZwUvD$l&8e42Qb^cbq>zYy@ive8`2N9vk=#6+AQlZZ7qk=?(ap1q0n0 z{B9Fte-{Gi-Tvax1)M+d1}Fyg@9X~sh1m|hsDcZuYOnxriBPN;z)q3<=-yBN2iM6V A?*IS* literal 0 HcmV?d00001 diff --git a/spring/.mvn/wrapper/maven-wrapper.properties b/spring/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..5f0536eb --- /dev/null +++ b/spring/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/spring/Dockerfile b/spring/Dockerfile new file mode 100644 index 00000000..75e1da14 --- /dev/null +++ b/spring/Dockerfile @@ -0,0 +1,14 @@ +# syntax=docker/dockerfile:1 + +FROM --platform=linux/amd64 eclipse-temurin:17-jdk-jammy + +WORKDIR /app + +COPY .mvn/ .mvn +COPY mvnw pom.xml ./ +COPY libs ./libs +RUN ./mvnw dependency:resolve + +COPY src ./src + +CMD ["./mvnw", "-Denvironment=localDev", "spring-boot:run"] \ No newline at end of file diff --git a/spring/mvnw b/spring/mvnw new file mode 100755 index 00000000..66df2854 --- /dev/null +++ b/spring/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/spring/pom.xml b/spring/pom.xml new file mode 100644 index 00000000..d184d99d --- /dev/null +++ b/spring/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.2.1 + + + ca.bcgov + spring-vinpower + 1.0 + vinpower + + + + 17 + UTF-8 + UTF-8 + + 2023-05-10T07:42:50Z + + + + + org.springframework.boot + spring-boot-starter-web + + + espdata + vinpower + 1.0 + system + ${basedir}/libs/vp4jo_bttm_msrp_gvw_50011.jar + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + + devtools + + + environment + localDev + + + + + org.springframework.boot + spring-boot-devtools + + + + + + \ No newline at end of file diff --git a/spring/src/main/java/com/vinpower/VinpowerApplication.java b/spring/src/main/java/com/vinpower/VinpowerApplication.java new file mode 100644 index 00000000..12ffb280 --- /dev/null +++ b/spring/src/main/java/com/vinpower/VinpowerApplication.java @@ -0,0 +1,11 @@ +package com.vinpower; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class VinpowerApplication { + public static void main(String[] args) { + SpringApplication.run(VinpowerApplication.class, args); + } +} \ No newline at end of file diff --git a/spring/src/main/java/com/vinpower/controller/MainController.java b/spring/src/main/java/com/vinpower/controller/MainController.java new file mode 100644 index 00000000..314474ad --- /dev/null +++ b/spring/src/main/java/com/vinpower/controller/MainController.java @@ -0,0 +1,31 @@ +package com.vinpower.controller; + +import java.util.Map; +import java.util.List; + +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.http.ResponseEntity; +import org.springframework.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; + +import com.vinpower.service.VinDecodeService; + +@RestController +public class MainController { + + @Autowired + VinDecodeService vinDecodeService; + + @GetMapping("/decode") + public ResponseEntity> decode(@RequestBody List data) { + try { + Map decodedVins = vinDecodeService.getDecodedVins(data); + return new ResponseEntity>(decodedVins, HttpStatus.OK); + } catch (Exception ex) { + ex.printStackTrace(); + } + return new ResponseEntity>(HttpStatus.INTERNAL_SERVER_ERROR); + } +} diff --git a/spring/src/main/java/com/vinpower/service/VinDecodeService.java b/spring/src/main/java/com/vinpower/service/VinDecodeService.java new file mode 100644 index 00000000..3e0c8db4 --- /dev/null +++ b/spring/src/main/java/com/vinpower/service/VinDecodeService.java @@ -0,0 +1,35 @@ +package com.vinpower.service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.stereotype.Service; + +import com.pki.vp4j.VinPower; + +@Service +public class VinDecodeService { + + // returns a decoded vin as an XML string + public String getDecodedVin(String vin) throws Exception { + VinPower vp = new VinPower(); + boolean rc = vp.decodeVIN(vin); + if (rc) { + return vp.getAsXML(); + } + return null; + } + + // returns decoded vins as a map of vins to XML strings + public Map getDecodedVins(List vins) throws Exception { + Map result = new HashMap<>(); + for (String vin : vins) { + String decodedVin = getDecodedVin(vin); + if (decodedVin != null) { + result.put(vin, decodedVin); + } + } + return result; + } +} From a69a0bac07a0f44c2585c8fbcaa9b22cd51a02d0 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:18:30 -0700 Subject: [PATCH 05/67] feat: 310 - ICBC file processing refinements (#335) * feat: 310 - ICBC file processing refinements * remove commented out code --- .../api/migrations/0028_auto_20240611_0251.py | 27 ++++ .../0029_alter_uploadedvinrecord_timestamp.py | 18 +++ django/api/models/uploaded_vin_record.py | 4 +- django/api/models/uploaded_vins_file.py | 2 - django/api/services/uploaded_vin_record.py | 119 ++++++++++++++---- django/workers/decorators/tasks.py | 31 +++++ django/workers/scheduled_jobs.py | 4 +- django/workers/tasks.py | 11 +- 8 files changed, 177 insertions(+), 39 deletions(-) create mode 100644 django/api/migrations/0028_auto_20240611_0251.py create mode 100644 django/api/migrations/0029_alter_uploadedvinrecord_timestamp.py create mode 100644 django/workers/decorators/tasks.py diff --git a/django/api/migrations/0028_auto_20240611_0251.py b/django/api/migrations/0028_auto_20240611_0251.py new file mode 100644 index 00000000..3ffde2ad --- /dev/null +++ b/django/api/migrations/0028_auto_20240611_0251.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-06-11 02:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0027_goelectricrebates_rebate_adjustment'), + ] + + operations = [ + migrations.RemoveField( + model_name='uploadedvinsfile', + name='chunks_per_run', + ), + migrations.AddField( + model_name='uploadedvinrecord', + name='timestamp', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='uploadedvinrecord', + name='postal_code', + field=models.CharField(max_length=7), + ), + ] diff --git a/django/api/migrations/0029_alter_uploadedvinrecord_timestamp.py b/django/api/migrations/0029_alter_uploadedvinrecord_timestamp.py new file mode 100644 index 00000000..d59c32a0 --- /dev/null +++ b/django/api/migrations/0029_alter_uploadedvinrecord_timestamp.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-11 02:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0028_auto_20240611_0251'), + ] + + operations = [ + migrations.AlterField( + model_name='uploadedvinrecord', + name='timestamp', + field=models.DateTimeField(), + ), + ] diff --git a/django/api/models/uploaded_vin_record.py b/django/api/models/uploaded_vin_record.py index 371b6b38..1d6d306b 100644 --- a/django/api/models/uploaded_vin_record.py +++ b/django/api/models/uploaded_vin_record.py @@ -5,7 +5,9 @@ class UploadedVinRecord(Auditable): vin = models.CharField(max_length=17) - postal_code = models.CharField(max_length=7, null=True, blank=True) + postal_code = models.CharField(max_length=7) + + timestamp = models.DateTimeField() data = models.JSONField() diff --git a/django/api/models/uploaded_vins_file.py b/django/api/models/uploaded_vins_file.py index a958dd28..853397f7 100644 --- a/django/api/models/uploaded_vins_file.py +++ b/django/api/models/uploaded_vins_file.py @@ -7,8 +7,6 @@ class UploadedVinsFile(Auditable): chunk_size = models.IntegerField(default=5000) - chunks_per_run = models.IntegerField(default=4) - start_index = models.IntegerField(default=0) processed = models.BooleanField(default=False) diff --git a/django/api/services/uploaded_vin_record.py b/django/api/services/uploaded_vin_record.py index ae1bea36..da5e2ff2 100644 --- a/django/api/services/uploaded_vin_record.py +++ b/django/api/services/uploaded_vin_record.py @@ -1,53 +1,120 @@ +from datetime import datetime import pandas as pd +from django.utils import timezone from api.models.uploaded_vin_record import UploadedVinRecord from api.decoder_constants import get_service def parse_and_save(uploaded_vins_file, file_response): processed = True - number_of_chunks_processed = 0 - number_of_chunks_to_process = uploaded_vins_file.chunks_per_run - chunksize = uploaded_vins_file.chunk_size start_index = uploaded_vins_file.start_index - chunks = pd.read_csv(file_response, sep="|", chunksize=chunksize) - - for idx, chunk in enumerate(chunks): - if ( - idx >= start_index - and number_of_chunks_processed < number_of_chunks_to_process - ): - vin_records_to_insert = get_vin_records_to_insert(chunk) - UploadedVinRecord.objects.bulk_create( - vin_records_to_insert, - ignore_conflicts=True, + chunks = pd.read_csv( + file_response, sep="|", chunksize=uploaded_vins_file.chunk_size + ) + + for idx, df in enumerate(chunks): + if idx == start_index: + df.fillna("", inplace=True) + vins = [] + for _, row in df.iterrows(): + if row["vin"] != "": + vins.append(row["vin"]) + df_records_map = get_df_records_map(df) + existing_records_map = get_existing_records_map(vins) + records_to_insert = get_records_to_insert( + df_records_map, existing_records_map + ) + UploadedVinRecord.objects.bulk_create(records_to_insert) + records_to_update = get_records_to_update( + df_records_map, existing_records_map ) - number_of_chunks_processed = number_of_chunks_processed + 1 - elif idx >= start_index + number_of_chunks_processed: + UploadedVinRecord.objects.bulk_update( + records_to_update, ["data", "timestamp", "update_timestamp"] + ) + elif idx > start_index: processed = False break - new_start_index = start_index + number_of_chunks_processed uploaded_vins_file.processed = processed - uploaded_vins_file.start_index = new_start_index + uploaded_vins_file.start_index = start_index + 1 uploaded_vins_file.save() -def get_vin_records_to_insert(df): - result = [] - df.fillna("", inplace=True) +# returns a dict of (vin, postal_code) -> {timestamp, data} +def get_df_records_map(df): + result = {} for _, row in df.iterrows(): - if row["vin"] != "": - vin = row["vin"] - postal_code = row["postal_code"] - data = row.to_dict() + vin = row["vin"] + postal_code = row["postal_code"] + df_timestamp = row["snapshot_date"] + if vin and postal_code and df_timestamp: + key = (vin, postal_code) + timestamp = timezone.make_aware( + datetime.strptime(df_timestamp, "%Y-%m-%d %H:%M:%S.%f") + ) + df_data = row.to_dict() + data = df_data if df_data else {} del data["vin"] del data["postal_code"] + del data["snapshot_date"] + if key in result: + most_recent_ts = result[key]["timestamp"] + if most_recent_ts < timestamp: + result[key] = {"timestamp": timestamp, "data": data} + else: + result[key] = {"timestamp": timestamp, "data": data} + return result + + +# returns a dict of (vin, postal_code) -> {id, timestamp} +def get_existing_records_map(vins): + result = {} + records = UploadedVinRecord.objects.only( + "id", "vin", "postal_code", "timestamp" + ).filter(vin__in=vins) + for record in records: + key = (record.vin, record.postal_code) + result[key] = {"id": record.id, "timestamp": record.timestamp} + return result + + +# df_records_map should be dict of (vin, postal_code) -> {timestamp, data} +# existing_records_map should be dict of (vin, postal_code) -> {id, timestamp} +def get_records_to_insert(df_records_map, existing_records_map): + result = [] + for key, value in df_records_map.items(): + if key not in existing_records_map: result.append( - UploadedVinRecord(vin=vin, postal_code=postal_code, data=data) + UploadedVinRecord( + vin=key[0], + postal_code=key[1], + timestamp=value["timestamp"], + data=value["data"], + ) ) return result +# df_records_map should be dict of (vin, postal_code) -> {timestamp, data} +# existing_records_map should be dict of (vin, postal_code) -> {id, timestamp} +def get_records_to_update(df_records_map, existing_records_map): + result = [] + for key, value in df_records_map.items(): + if key in existing_records_map: + existing_record = existing_records_map[key] + timestamp = value["timestamp"] + if existing_record["timestamp"] < timestamp: + result.append( + UploadedVinRecord( + id=existing_record["id"], + timestamp=timestamp, + data=value["data"], + update_timestamp=timezone.now(), + ) + ) + return result + + def get_decode_successful(service_name, uploaded_record): service = get_service(service_name) if service: diff --git a/django/workers/decorators/tasks.py b/django/workers/decorators/tasks.py new file mode 100644 index 00000000..e367defe --- /dev/null +++ b/django/workers/decorators/tasks.py @@ -0,0 +1,31 @@ +import ctypes +import threading + + +class TaskTimeoutException(Exception): + pass + + +def timeout(time): + def wrapper(func): + def wrapped(*args, **kwargs): + current_thread_id = threading.current_thread().ident + + def throw_timeout(): + ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_long(current_thread_id), + ctypes.py_object(TaskTimeoutException), + ) + + t = threading.Timer(time, throw_timeout) + t.start() + try: + func(*args, **kwargs) + t.cancel() + except Exception as ex: + t.cancel() + raise ex + + return wrapped + + return wrapper diff --git a/django/workers/scheduled_jobs.py b/django/workers/scheduled_jobs.py index 9d280832..ea1ce1d2 100644 --- a/django/workers/scheduled_jobs.py +++ b/django/workers/scheduled_jobs.py @@ -20,8 +20,8 @@ def schedule_read_uploaded_vins_file(): "workers.tasks.read_uploaded_vins_file", name="read_uploaded_vins_file", schedule_type="C", - cron="*/15 * * * *", - q_options={"timeout": 660, "ack_failure": True}, + cron="*/3 * * * *", + q_options={"timeout": 165, "ack_failure": True}, ) except IntegrityError: pass diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 3f13d1dd..0061554a 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -1,6 +1,5 @@ from django.conf import settings from api.services.minio import get_minio_client, get_minio_object -from func_timeout import func_timeout, FunctionTimedOut from api.models.uploaded_vins_file import UploadedVinsFile from api.models.uploaded_vin_record import UploadedVinRecord from api.decoder_constants import get_service @@ -8,6 +7,7 @@ from api.services.decoded_vin_record import save_decoded_data from api.services.uploaded_vin_record import parse_and_save from django.db import transaction +from workers.decorators.tasks import timeout def create_minio_bucket(): @@ -19,14 +19,8 @@ def create_minio_bucket(): @transaction.atomic +@timeout(150) def read_uploaded_vins_file(): - # TODO: this job will probably have to become more involved; it currently just uploads whatever is in the file while skipping records - # that encounter uniqueness conflicts. - # we'll probably have to do an initial, chunked read from the - # file in order to build a map of (vin, postal_code) -> (record chunk index, record index within chunk) of unique records (based on snapshot_date?), - # then we'll have to compare the (vin, postal_code) keys to existing records in the database, and - # determine which ones need to get bulk-inserted, and which ones bulk-updated. - # also have to keep in mind the memory used by any data structures we use vins_file = ( UploadedVinsFile.objects.filter(processed=False) .order_by("create_timestamp") @@ -43,6 +37,7 @@ def read_uploaded_vins_file(): pass +@timeout(45) def batch_decode_vins(service_name, batch_size=50): max_decode_attempts = settings.MAX_DECODE_ATTEMPTS service = get_service(service_name) From 3bea6c1754187f54197f5727c73dde62b2985404 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:32:53 -0700 Subject: [PATCH 06/67] feat: CTHUB 316 - GER/SUVI Consistent prepositions and articles (#336) * feat: adds preposition capitalization logic * chore: fix line length --- .../api/services/spreadsheet_uploader_prep.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 1f502723..dfaecc53 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -96,6 +96,7 @@ def prepare_go_electric_rebates(df): df[format_dict[key]] = df[format_dict[key]].apply(format_case, case = key) make_names_consistent(df) + make_prepositions_consistent(df) return df @@ -149,3 +150,22 @@ def make_names_consistent(df): df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer']].replace( consistent_name_dict, regex=True) + +def make_prepositions_consistent(df): + df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer']].replace( + dict.fromkeys( + ['(?i)\\bbc(?=\\W)', # Matches word "bc" regardless of case + '(?i)\\bb\\.c\\.(?=\\W)'], 'BC'), # Matches word "b.c." regardless of case + regex=True + ).replace( + {'BC Ltd.': 'B.C. Ltd.', + '\\bOf(?=\\W)': 'of', + '\\bAnd(?=\\W)': 'and', # Matches word "And" + '\\bThe(?=\\W)': 'the', + '\\bA(?=\\W)': 'a', + '\\bAn(?=\\W)': 'an'}, + regex=True + ) + ##The first letter should be capitalized + df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer'] + ].applymap(lambda x: x[0].upper() + x[1:]) From 8a4deb1c4baf8081c9fca91a6f8a070605249174 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:46:35 -0700 Subject: [PATCH 07/67] feat: adds notes column to GER, changes tab name (#338) --- django/api/constants.py | 5 ++++- .../migrations/0030_goelectricrebates_notes.py | 18 ++++++++++++++++++ django/api/models/go_electric_rebates.py | 2 +- .../api/services/spreadsheet_uploader_prep.py | 7 +++++-- 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 django/api/migrations/0030_goelectricrebates_notes.py diff --git a/django/api/constants.py b/django/api/constants.py index 2b9bc345..a830b694 100644 --- a/django/api/constants.py +++ b/django/api/constants.py @@ -382,6 +382,7 @@ class GoElectricRebatesColumns(Enum): VIN = "VIN Number" VEHICLE_CLASS = "Class" REBATE_ADJUSTMENT = "Rebate adjustment (discount)" + NOTES = "Notes" class GoElectricRebatesColumnMapping(Enum): @@ -402,6 +403,7 @@ class GoElectricRebatesColumnMapping(Enum): vin = "VIN Number" vehicle_class = "Class" rebate_adjustment = "Rebate adjustment (discount)" + notes = "Notes" @@ -579,6 +581,7 @@ class GoElectricRebatesColumnMapping(Enum): "vin": str, "vehicle_class": str, "rebate_adjustment": str, + "notes": str, }, } @@ -644,7 +647,7 @@ class GoElectricRebatesColumnMapping(Enum): "model": GoElectricRebates, "columns": GoElectricRebatesColumns, "column_mapping": GoElectricRebatesColumnMapping, - "sheet_name": "Main list", + "sheet_name": "Distribution List - Master", "preparation_functions": [prepare_go_electric_rebates], }, } diff --git a/django/api/migrations/0030_goelectricrebates_notes.py b/django/api/migrations/0030_goelectricrebates_notes.py new file mode 100644 index 00000000..dbb27341 --- /dev/null +++ b/django/api/migrations/0030_goelectricrebates_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-06-18 20:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0029_alter_uploadedvinrecord_timestamp'), + ] + + operations = [ + migrations.AddField( + model_name='goelectricrebates', + name='notes', + field=models.CharField(blank=True, max_length=250, null=True), + ), + ] diff --git a/django/api/models/go_electric_rebates.py b/django/api/models/go_electric_rebates.py index 1bd5b870..759b9f68 100644 --- a/django/api/models/go_electric_rebates.py +++ b/django/api/models/go_electric_rebates.py @@ -30,6 +30,6 @@ class GoElectricRebates(Auditable): vin = models.CharField(blank=True, max_length=100, null=True) vehicle_class = models.CharField(blank=True, null=True, max_length=50) rebate_adjustment = models.CharField(blank=True, null=True, max_length=50) - + notes = models.CharField(blank=True, null=True, max_length=250) class Meta: db_table = "go_electric_rebates" diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index dfaecc53..e327791d 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -90,7 +90,8 @@ def prepare_go_electric_rebates(df): 'Manufacturer', 'City'], 'upper': ['Model', 'Postal code', 'VIN Number'], 'lower': ['Email'], - 'skip': ['Phone Number'] + 'skip': ['Phone Number'], + 'sentence': ['Notes'], } for key in format_dict: df[format_dict[key]] = df[format_dict[key]].apply(format_case, case = key) @@ -106,13 +107,15 @@ def format_case(s, case = 'skip', ignore_list = []): .astype(str) # Convert to string .str.strip() # Strip white spaces (this dataset suffers from extra tabs, lines, etc.) ) - if case == 'title': s = s.str.title() elif case == 'upper': s = s.str.upper() elif case == 'lower': s = s.str.lower() + elif case == 'sentence': + ##filter out the temporary null records before changing to sentence case + s = s[s != 'TEMP_NULL'].str.capitalize() elif case == 'skip': pass From c44bd97b8904c58250923367bc5e5df9ba420061 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:48:49 -0700 Subject: [PATCH 08/67] Task: Adjust Manufacturer Names #317 (#339) * Adding prep function to alter manufacturer names to have more readable casing * Cleanup --- .../api/services/spreadsheet_uploader_prep.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index e327791d..7c47b5b2 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -98,6 +98,7 @@ def prepare_go_electric_rebates(df): make_names_consistent(df) make_prepositions_consistent(df) + adjust_ger_manufacturer_names(df) return df @@ -172,3 +173,21 @@ def make_prepositions_consistent(df): ##The first letter should be capitalized df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer'] ].applymap(lambda x: x[0].upper() + x[1:]) + +def adjust_ger_manufacturer_names(df): + """"" + This function is currently GER specific updating the manufacturer names to have casing that makes more sense + since currently all manufacturer column entries are set to sentence casing. + + """"" + + name_replacements = { + 'International Ic Bus': 'International IC Bus', + 'Lightning Emotors': 'Lightning eMotors', + 'Avro Gse': 'Avro GSE', + 'Bmw': 'BMW', + 'Ego': 'EGO', + 'Sc Carts': 'SC Carts' + } + + df[['Manufacturer']] = df[['Manufacturer']].replace(name_replacements, regex=False) \ No newline at end of file From 840bf8c84d1d4ee73eb6cf45b1be95cf717aa6c0 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:54:15 -0700 Subject: [PATCH 09/67] chore: set numpy version (#340) --- django/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/django/requirements.txt b/django/requirements.txt index 2e78859d..5ff73ec4 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -16,6 +16,7 @@ whitenoise==5.2.0 dj-database-url==0.5.0 django-heroku==0.3.1 gunicorn==20.1.0 +numpy==1.26.4 python-keycloak==0.26.1 python-dotenv==0.19.0 pandas==1.3.4 From 796525299894a6aa0bd87e7c465427c56ffd3590 Mon Sep 17 00:00:00 2001 From: Kuan Fan <31664961+kuanfandevops@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:30:46 -0700 Subject: [PATCH 10/67] Add postgrest (#341) --- openshift/templates/postgrest/Dockerfile | 12 ++++ .../templates/postgrest/postgrest-bc.yaml | 68 +++++++++++++++++++ openshift/templates/postgrest/readme.txt | 12 ++++ 3 files changed, 92 insertions(+) create mode 100644 openshift/templates/postgrest/Dockerfile create mode 100644 openshift/templates/postgrest/postgrest-bc.yaml create mode 100644 openshift/templates/postgrest/readme.txt diff --git a/openshift/templates/postgrest/Dockerfile b/openshift/templates/postgrest/Dockerfile new file mode 100644 index 00000000..c130f244 --- /dev/null +++ b/openshift/templates/postgrest/Dockerfile @@ -0,0 +1,12 @@ +FROM bitnami/postgrest:11.2.2 + +USER 0 + +RUN apt-get -y update \ + && apt-get -y install curl + +RUN mkdir /var/run/postgresql \ + && chgrp -R root /var/run/postgresql \ + && chmod -R g+w /var/run/postgresql + +USER 1001 \ No newline at end of file diff --git a/openshift/templates/postgrest/postgrest-bc.yaml b/openshift/templates/postgrest/postgrest-bc.yaml new file mode 100644 index 00000000..6548c2f7 --- /dev/null +++ b/openshift/templates/postgrest/postgrest-bc.yaml @@ -0,0 +1,68 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + creationTimestamp: null + name: postgrest-bc +parameters: + - name: GIT_URL + displayName: + description: cthub repo + required: false + - name: GIT_REF + displayName: + description: cthub branch name of the pr + required: false +objects: +- apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + annotations: + description: postgrest image + labels: + shared: "true" + creationTimestamp: null + name: cthub-postgrest + spec: + lookupPolicy: + local: false + status: + dockerImageRepository: "" +- apiVersion: build.openshift.io/v1 + kind: BuildConfig + metadata: + creationTimestamp: null + name: cthub-postgrest + spec: + failedBuildsHistoryLimit: 5 + nodeSelector: null + output: + to: + kind: ImageStreamTag + name: cthub-postgrest:11.2.2 + postCommit: {} + resources: + limits: + cpu: 2000m + memory: 4Gi + requests: + cpu: 500m + memory: 2Gi + runPolicy: Serial + source: + git: + ref: postgrest-0.3.0 + uri: https://github.com/bcgov/cthub.git + type: Git + contextDir: openshift/templates/postgrest + strategy: + dockerStrategy: + noCache: true + forcePull: true + type: Docker + successfulBuildsHistoryLimit: 5 + triggers: + - imageChange: {} + type: ImageChange + - type: ConfigChange + status: + lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/postgrest/readme.txt b/openshift/templates/postgrest/readme.txt new file mode 100644 index 00000000..eb6fee71 --- /dev/null +++ b/openshift/templates/postgrest/readme.txt @@ -0,0 +1,12 @@ +== Postgrest research on test env +create role web_anon nologin; + # GRANT CONNECT ON DATABASE cthub TO web_anon; +GRANT USAGE ON SCHEMA public TO web_anon; +GRANT SELECT ON public.vehicle TO web_anon; + # ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO web_anon; +create role authenticator noinherit login password 'xxxxx'; +GRANT web_anon to authenticator; + +select * from information_schema.role_table_grants where grantee='web_anon'; + +openssl rand -hex 64 From e60e80fa30993327c7fc36eab5232fe62ff5e165 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:40:28 -0700 Subject: [PATCH 11/67] chore: django-q config change (#342) --- django/api/settings.py | 2 +- django/workers/tasks.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django/api/settings.py b/django/api/settings.py index d5671c9e..ec5cb484 100644 --- a/django/api/settings.py +++ b/django/api/settings.py @@ -188,7 +188,7 @@ "name": "CTHUB", "workers": 4, "timeout": 90, - "retry": 120, + "retry": 1800, "queue_limit": 50, "bulk": 10, "orm": "default", diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 0061554a..d6d2c5a6 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -37,6 +37,7 @@ def read_uploaded_vins_file(): pass +@transaction.atomic @timeout(45) def batch_decode_vins(service_name, batch_size=50): max_decode_attempts = settings.MAX_DECODE_ATTEMPTS From 96aa5679815a747fd8ec81b77ad32417cbaeddd9 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:43:04 -0700 Subject: [PATCH 12/67] feat: 328 - GER Typo checker for Applicant Name (#345) * feat: adds typo checking to backend and returns warnings to frontend * feat: popup showing warnings for similar sounding names * chore: code cleanup --- django/api/services/spreadsheet_uploader.py | 20 +++- .../api/services/spreadsheet_uploader_prep.py | 53 ++++++++++- django/api/viewsets/upload.py | 6 +- frontend/src/uploads/UploadContainer.js | 92 +++++++++++++++---- frontend/src/uploads/components/UploadPage.js | 2 +- 5 files changed, 147 insertions(+), 26 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 30dbaedb..5a1a315a 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -1,8 +1,10 @@ from decimal import Decimal, ROUND_HALF_UP +from rest_framework.response import Response +from rest_framework import status import pandas as pd import traceback from django.db import transaction - +from api.services.spreadsheet_uploader_prep import typo_checker def get_field_default(model, field): field = model._meta.get_field(field) @@ -57,7 +59,6 @@ def transform_data( for validate in validation_functions: df = validate(df) - column_mapping = {col.name: col.value for col in column_mapping_enum} # Need to use the inverse (keys) for mapping the columns to what the database expects in order to use enums inverse_column_mapping = {v: k for k, v in column_mapping.items()} @@ -169,6 +170,7 @@ def import_from_xls( user, preparation_functions=[], validation_functions=[], + check_for_warnings=False, ): try: df = extract_data(excel_file, sheet_name, header_row) @@ -179,6 +181,20 @@ def import_from_xls( preparation_functions, validation_functions, ) + + if check_for_warnings: + ## do the error checking + typo_warnings = typo_checker(df, df['applicant_name'].dropna(), .8) + if typo_warnings: + return { + "success": True, + "message": "We encountered some potential typos in your data. Please choose whether to ignore them and continue inserting data or cancel upload and make edits to the data before reuploading", + "warning": True, + "warnings": typo_warnings, + } + else: + print('no warnings') + result = load_data(df, model, field_types, replace_data, user) total_rows = result["row_count"] diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 7c47b5b2..a6ca8603 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -1,7 +1,7 @@ from decimal import Decimal import numpy as np import pandas as pd - +import difflib as dl def prepare_arc_project_tracking(df): df["Publicly Announced"] = df["Publicly Announced"].replace( @@ -99,7 +99,7 @@ def prepare_go_electric_rebates(df): make_names_consistent(df) make_prepositions_consistent(df) adjust_ger_manufacturer_names(df) - + return df def format_case(s, case = 'skip', ignore_list = []): @@ -190,4 +190,51 @@ def adjust_ger_manufacturer_names(df): 'Sc Carts': 'SC Carts' } - df[['Manufacturer']] = df[['Manufacturer']].replace(name_replacements, regex=False) \ No newline at end of file + df[['Manufacturer']] = df[['Manufacturer']].replace(name_replacements, regex=False) + + +def typo_checker(df, s, c=0.7): + """ + Check for similar words in a single Pandas Series. + + Parameters + ---------- + s : Panda Series + c : Similarity cutoff, higher is more similar + + Returns + ------- + dict + A dictionary with similar words + + """ + if isinstance(s, pd.Series) is False: + raise Exception('Function argument "s" has to be Pandas Series type') + + if s.unique().shape[0] == 1: + raise Exception('Function argument "s" contains only one unique value, there is nothing to compare') + elif s.shape[0] == 0: + raise Exception('Function argument "s" is empty, there is nothing to compare') + + unique_vals = list(set(s)) # Get all unique values from the series + unique_vals.sort(reverse=True) # Sort them to check for duplicates later + + match_dict = {} + for value in unique_vals: + cutoff = c + matches = dl.get_close_matches( + value, # Value to compare + unique_vals[:unique_vals.index(value)] + unique_vals[unique_vals.index(value)+1:], # All other values to compare value to + cutoff = cutoff # Similarity cutoff score, higher values mean more similar + ) + + if (len(matches) > 0) & (value not in sum(match_dict.values(), [])): + match_dict[value] = matches # Add value to the dictionary if it has matches and if it is not yet in the dictionary + else: + pass + + if bool(match_dict) == True: + # If the dictionary is not empty, return it + return match_dict + else: + print('No issues') diff --git a/django/api/viewsets/upload.py b/django/api/viewsets/upload.py index 13704861..206bea59 100644 --- a/django/api/viewsets/upload.py +++ b/django/api/viewsets/upload.py @@ -47,11 +47,14 @@ def datasets_list(self, request): @action(detail=False, methods=["post"]) @method_decorator(check_upload_permission()) def import_data(self, request): - filename = request.data.get("filename") dataset_selected = request.data.get("datasetSelected") replace_data = request.data.get("replace", False) filepath = request.data.get("filepath") + check_for_warnings = request.data.get("checkForWarnings") + #boolean, if true show warnings before inserting data + #after displaying warnings, code can be rerun with show_warnings = false + #if warnings have been ignore if dataset_selected == "ICBC Vins": file_extension = pathlib.Path(filepath).suffix @@ -95,6 +98,7 @@ def import_data(self, request): field_types=constants.FIELD_TYPES.get(dataset_selected), replace_data=replace_data, user=request.user, + check_for_warnings=check_for_warnings ) if not result["success"]: diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index 22528fc0..c51181bd 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -24,7 +24,14 @@ const UploadContainer = () => { const [adminUser, setAdminUser] = useState(false); const axios = useAxios(); const axiosDefault = useAxios(true); - + const [dataWarning, setDataWarning] = useState({}) + const [alertDialogText, setAlertDialogText] = useState({ + title: "", + content: "", + confirmText: "", + confirmAction: ()=>{}, + cancelAction: ()=>{}, + }) const refreshList = () => { setRefresh(true); axios.get(ROUTES_UPLOAD.LIST).then((response) => { @@ -57,28 +64,27 @@ const UploadContainer = () => { setAlert(true); }; - const doUpload = () => + const doUpload = (checkForWarnings) => uploadFiles.forEach((file) => { let filepath = file.path; - setLoading(true); + setLoading(true); + if (datasetSelected !== 'Go Electric Rebates Program'){ + checkForWarnings = false + } const uploadPromises = uploadFiles.map((file) => { return axios.get(ROUTES_UPLOAD.MINIO_URL).then((response) => { const { url: uploadUrl, minio_object_name: filename } = response.data; return axiosDefault.put(uploadUrl, file).then(() => { - let replace = false; - if (replaceData === true) { - replace = true; - } return axios.post(ROUTES_UPLOAD.UPLOAD, { filename, datasetSelected, - replace, + replaceData, filepath, + checkForWarnings }); }); }); }); - Promise.all(uploadPromises) .then((responses) => { const errorCheck = responses.some( @@ -86,16 +92,49 @@ const UploadContainer = () => { ); setAlertSeverity(errorCheck ? "success" : "error"); - const message = responses + .map( + (response) => + `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, + ) + .join("\n"); + setAlert(true); + setAlertContent(message); + const warnings = responses .map( (response) => - `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, + response.data.warnings ? response.data.warnings: "" ) - .join("\n"); - setAlertContent(message); - setAlert(true); + + if (warnings && checkForWarnings == true) { // ie it is the first attempt to upload (when upload is called from the dialog its set to false) + setOpenDialog(true) + setAlertDialogText({ + title: "Warning: There are similar names in the data to review", + content:( + <> +
+

+ Click continue to insert these records as is, or click cancel + to exit out and no records will be inserted: +

+ {warnings.map((warningItem, index) => ( +
+ {Object.keys(warningItem).map(company => ( +
+ {company}: {warningItem[company].join(', ')} +
+ ))} +
+ ))} +
+ + ), + confirmText: "Continue (all records will be inserted as is)", + confirmAction: handleConfirmDataInsert, + cancelAction: handleReplaceDataCancel, + })} + setUploadFiles([]); }) .catch((error) => { @@ -135,11 +174,24 @@ const UploadContainer = () => { const choice = event.target.value; if (choice === "replace") { setOpenDialog(true); + setAlertDialogText({ + title: "Replace existing data?", + content: "Selecting replace will delete all previously uploaded records for this dataset", + confirmText: "Replace existing data", + confirmAction: handleReplaceDataConfirm, + cancelAction: handleReplaceDataCancel, + }) } else { setReplaceData(false); } }; + const handleConfirmDataInsert = () => { + setOpenDialog(false); + showError(false); + setAlertContent("") + doUpload(false); //upload with the checkForWarnings flag set to false! + } const handleReplaceDataConfirm = () => { setReplaceData(true); setOpenDialog(false); @@ -157,6 +209,7 @@ const UploadContainer = () => { return ; } + const alertElement = alert && alertContent && alertSeverity ? ( @@ -169,20 +222,21 @@ const UploadContainer = () => { ) : null; + return (
<> diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index 6228e42b..d3c5ec7d 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -105,7 +105,7 @@ const UploadPage = (props) => {
), diff --git a/frontend/src/uploads/components/WarningsList.js b/frontend/src/uploads/components/WarningsList.js new file mode 100644 index 00000000..1f9bd9e7 --- /dev/null +++ b/frontend/src/uploads/components/WarningsList.js @@ -0,0 +1,26 @@ +import React from "react"; + +const WarningsList = ({ warnings = {} }) => { + const warningElements = []; + for (const [filename, value] of Object.entries(warnings)) { + for (const [warningType, warning] of Object.entries(value)) { + for (const [column, indices] of Object.entries(warning)) { + const indicesString = indices.join(", "); + // todo: remove inline styling once we know more about the UI + const warningElement = ( +
+ {`The file "${filename}" contains warnings of type "${warningType}" located under column "${column}", and along row(s): `}{" "} + {indicesString} +
+ ); + warningElements.push(warningElement); + } + } + } + if (Object.keys(warningElements).length > 0) { + return
{warningElements}
; + } + return null; +}; + +export default WarningsList; From 136ca0c237256ca4063cbdc51c67c6c4a0aee047 Mon Sep 17 00:00:00 2001 From: Kuan Fan <31664961+kuanfandevops@users.noreply.github.com> Date: Tue, 9 Jul 2024 15:43:10 -0700 Subject: [PATCH 22/67] Spring build 0.3.0 (#360) --- .github/workflows/dev-ci.yaml | 44 +++++---- .../vinpower/artifactory-secret.yaml | 15 ++++ openshift/templates/vinpower/vinpower-bc.yaml | 89 +++++++++++++++++++ spring/Dockerfile-Openshift | 29 ++++++ 4 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 openshift/templates/vinpower/artifactory-secret.yaml create mode 100644 openshift/templates/vinpower/vinpower-bc.yaml create mode 100644 spring/Dockerfile-Openshift diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml index 59968be8..194d4536 100644 --- a/.github/workflows/dev-ci.yaml +++ b/.github/workflows/dev-ci.yaml @@ -4,10 +4,10 @@ name: CTHUB 0.3.0 Dev CI on: push: - branches: [ release-0.3.0 ] - paths: - - frontend/** - - backend/** + branches: [ 0.3.0 ] + # paths: + # - frontend/** + # - backend/** workflow_dispatch: env: @@ -81,6 +81,14 @@ jobs: oc -n ${{ env.TOOLS_NAMESPACE }} wait --for=condition=Complete --timeout=900s build/cthub-task-queue-${{ env.VERSION }}-${{ env.PRE_RELEASE }}-1 oc tag ${{ env.TOOLS_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.DEV_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} + - name: Build CTHUB Vinpower + run: | + cd openshift/templates/vinpower + oc process -f ./vinpower-bc.yaml NAME=cthub SUFFIX=-${{ env.VERSION }}-${{ env.PRE_RELEASE }} VERSION=${{ env.VERSION }}-${{ env.PRE_RELEASE }} GIT_URL=${{ env.GIT_URL }} GIT_REF=release-${{ env.VERSION }} | oc apply --wait=true -f - -n ${{ env.TOOLS_NAMESPACE }} + sleep 5s + oc -n ${{ env.TOOLS_NAMESPACE }} wait --for=condition=Complete --timeout=900s build/cthub-vinpower-${{ env.VERSION }}-${{ env.PRE_RELEASE }}-1 + oc tag ${{ env.TOOLS_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.DEV_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} + deploy: name: Deploy CTHUB on Dev @@ -100,20 +108,24 @@ jobs: ref: main ssh-key: ${{ secrets.MANIFEST_REPO_DEPLOY_KEY }} - - name: Update frontend tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml - - - name: Update backend tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml - - - name: Update task-queue tag + - name: Update tags uses: mikefarah/yq@v4.40.5 with: - cmd: yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml + cmd: | + yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml + yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml] + yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml + yq -i '.vinpower.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml + + # - name: Update backend tag + # uses: mikefarah/yq@v4.40.5 + # with: + # cmd: yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml + + # - name: Update task-queue tag + # uses: mikefarah/yq@v4.40.5 + # with: + # cmd: yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml - name: GitHub Commit & Push run: | diff --git a/openshift/templates/vinpower/artifactory-secret.yaml b/openshift/templates/vinpower/artifactory-secret.yaml new file mode 100644 index 00000000..5df3408f --- /dev/null +++ b/openshift/templates/vinpower/artifactory-secret.yaml @@ -0,0 +1,15 @@ +apiVersion: template.openshift.io/v1 +kind: Template +parameters: +- name: ARTIFACTORY_URL + description: + required: true +objects: +- apiVersion: v1 + kind: Secret + metadata: + annotations: null + name: cthub-artifactory + stringData: + ARTIFACTORY_URL: ${ARTIFACTORY_URL} + diff --git a/openshift/templates/vinpower/vinpower-bc.yaml b/openshift/templates/vinpower/vinpower-bc.yaml new file mode 100644 index 00000000..68c2e94d --- /dev/null +++ b/openshift/templates/vinpower/vinpower-bc.yaml @@ -0,0 +1,89 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + creationTimestamp: null + name: vinpower +parameters: + - name: NAME + displayName: + description: the module name entered when run yo bcdk:pipeline, which is zeva + required: true + - name: SUFFIX + displayName: + description: sample is -pr-0 + required: true + - name: VERSION + displayName: + description: image tag name for output + required: true + - name: GIT_URL + displayName: + description: cthub repo + required: true + - name: GIT_REF + displayName: + description: cthub branch name of the pr + required: true +objects: + - apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + annotations: + description: Keeps track of changes in the client / front end image + labels: + shared: "true" + creationTimestamp: null + name: ${NAME}-vinpower + spec: + lookupPolicy: + local: false + status: + dockerImageRepository: "" + - apiVersion: build.openshift.io/v1 + kind: BuildConfig + metadata: + name: ${NAME}-vinpower${SUFFIX} + creationTimestamp: + annotations: + description: Defines how to build the vinpower image in docker + spec: + output: + to: + kind: ImageStreamTag + name: ${NAME}-vinpower:${VERSION} + resources: + limits: + cpu: 1500m + memory: 1300Mi + requests: + cpu: 750m + memory: 650Mi + source: + contextDir: spring + git: + uri: ${GIT_URL} + ref: ${GIT_REF} + type: Git + strategy: + dockerStrategy: + dockerfilePath: ./Dockerfile-Openshift + env: + - name: ARTIFACTORY_USER + valueFrom: + secretKeyRef: + name: artifacts-default-idxprm + key: username + - name: ARTIFACTORY_PASSWORD + valueFrom: + secretKeyRef: + name: artifacts-default-idxprm + key: password + - name: ARTIFACTORY_URL + valueFrom: + secretKeyRef: + name: cthub-artifactory + key: ARTIFACTORY_URL + type: Docker + triggers: + - type: ConfigChange + - type: ImageChange diff --git a/spring/Dockerfile-Openshift b/spring/Dockerfile-Openshift new file mode 100644 index 00000000..9c41618a --- /dev/null +++ b/spring/Dockerfile-Openshift @@ -0,0 +1,29 @@ +FROM --platform=linux/amd64 artifacts.developer.gov.bc.ca/docker-remote/eclipse-temurin:17-jdk-jammy + +ARG ARTIFACTORY_URL +ARG ARTIFACTORY_USER +ARG ARTIFACTORY_PASSWORD + +RUN apt-get update && apt-get install -y curl + +RUN mkdir libs +WORKDIR /app/libs + +# Download the JAR file from Artifactory +RUN curl -u ${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -O ${ARTIFACTORY_URL} + +WORKDIR /app + +COPY .mvn/ .mvn +COPY mvnw pom.xml ./ +RUN ls -l ./libs && \ + mkdir /.m2 && \ + chgrp -R root /.m2 && \ + chmod -R g+rw /.m2 && \ + chgrp -R root /app && \ + chmod -R g+rw /app +RUN ./mvnw dependency:resolve + +COPY src ./src + +CMD ["./mvnw", "-Denvironment=localDev", "spring-boot:run"] From 3425b0b679fb0c5469cc26c082a5e8d0642c3bf7 Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Tue, 9 Jul 2024 15:45:39 -0700 Subject: [PATCH 23/67] update branches in dev-ci.yaml --- .github/workflows/dev-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml index 194d4536..cadddd45 100644 --- a/.github/workflows/dev-ci.yaml +++ b/.github/workflows/dev-ci.yaml @@ -4,7 +4,7 @@ name: CTHUB 0.3.0 Dev CI on: push: - branches: [ 0.3.0 ] + branches: [ release-0.3.0 ] # paths: # - frontend/** # - backend/** From 28b65e28fd83fd11d6e1570aab598ad7ebd433ae Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:51:42 -0700 Subject: [PATCH 24/67] feat/fix - 337 and 324 - vinpower job and use google dns resolver for email validation (#362) --- django/api/constants/constants.py | 3 ++- django/api/services/resolvers.py | 7 +++++++ .../api/services/spreadsheet_uploader_prep.py | 18 ++++++++++++++---- django/requirements.txt | 1 + django/workers/apps.py | 6 ++++-- django/workers/scheduled_jobs.py | 19 +++++++++++++++++-- docker-compose.yml | 8 ++++++++ spring/.gitignore | 3 ++- spring/Dockerfile-Openshift | 2 +- 9 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 django/api/services/resolvers.py diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index f026bea6..8d94c852 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -24,6 +24,7 @@ location_checker, email_validator ) +from api.services.resolvers import get_google_resolver class ARCProjectTrackingColumns(Enum): @@ -655,7 +656,7 @@ class GoElectricRebatesColumnMapping(Enum): {"error_type": "Phone Error", "function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, {"error_type": "Potential Typo", "function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, {"error_type": "Location Not Found", "function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, - {"error_type": "Invalid Email", "function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2}} + {"error_type": "Invalid Email", "function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}} ] }, } diff --git a/django/api/services/resolvers.py b/django/api/services/resolvers.py new file mode 100644 index 00000000..510a8ee0 --- /dev/null +++ b/django/api/services/resolvers.py @@ -0,0 +1,7 @@ +from dns.resolver import Resolver + + +def get_google_resolver(): + resolver = Resolver() + resolver.nameservers = ["8.8.8.8"] + return resolver diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index d3ca2802..b467a398 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -211,8 +211,13 @@ def typo_checker(df, *columns, **kwargs): cutoff = kwargs["cutoff"] ) if matches: - indices_to_add = map_of_values_to_indices[value] - indices.extend(indices_to_add) + value_indices = map_of_values_to_indices[value] + indices.extend(value_indices) + # it appears that difflib's "is similar" predicate S is not symmetric (i.e. aSb does not imply bSa) + # so we have to do: + for match in matches: + match_indices = map_of_values_to_indices[match] + indices.extend(match_indices) if indices: result[column] = sorted(list(set(indices))) return result @@ -260,18 +265,23 @@ def location_checker(df, *columns, **kwargs): indices_to_add = map_of_values_to_indices[name] indices.extend(indices_to_add) if indices: - result[column] = sorted(list(set(indices))) + indices.sort() + result[column] = indices return result def email_validator(df, *columns, **kwargs): + resolver = None + get_resolver = kwargs.get("get_resolver") + if get_resolver is not None: + resolver = get_resolver() result = {} for column in columns: indices = [] series = df[column] for index, value in series.items(): try: - validate_email(value) + validate_email(value, dns_resolver=resolver) except EmailNotValidError: indices.append(index + kwargs.get("indices_offset", 0)) if indices: diff --git a/django/requirements.txt b/django/requirements.txt index ccf90a28..1ece7e27 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -24,4 +24,5 @@ openpyxl==3.0.9 minio==7.1.1 xlsxwriter==3.2.0 xmltodict==0.13.0 +dnspython==2.6.1 email-validator==2.2.0 diff --git a/django/workers/apps.py b/django/workers/apps.py index c05d62f6..81e58fbb 100644 --- a/django/workers/apps.py +++ b/django/workers/apps.py @@ -9,10 +9,12 @@ def ready(self): from workers.scheduled_jobs import ( schedule_create_minio_bucket, schedule_read_uploaded_vins_file, - schedule_batch_decode_vins, + schedule_batch_decode_vins_vpic, + schedule_batch_decode_vins_vinpower, ) if "qcluster" in sys.argv: schedule_create_minio_bucket() schedule_read_uploaded_vins_file() - schedule_batch_decode_vins() + schedule_batch_decode_vins_vpic() + schedule_batch_decode_vins_vinpower() diff --git a/django/workers/scheduled_jobs.py b/django/workers/scheduled_jobs.py index 66f23b59..e10758a0 100644 --- a/django/workers/scheduled_jobs.py +++ b/django/workers/scheduled_jobs.py @@ -27,13 +27,28 @@ def schedule_read_uploaded_vins_file(): pass -def schedule_batch_decode_vins(): +def schedule_batch_decode_vins_vpic(): try: schedule( "workers.tasks.batch_decode_vins", "vpic", 50, - name="batch_decode_vins", + name="vpic_batch_decode_vins", + schedule_type="C", + cron="*/2 * * * *", + q_options={"timeout": 105, "ack_failure": True}, + ) + except IntegrityError: + pass + + +def schedule_batch_decode_vins_vinpower(): + try: + schedule( + "workers.tasks.batch_decode_vins", + "vinpower", + 2000, + name="vinpower_batch_decode_vins", schedule_type="C", cron="*/2 * * * *", q_options={"timeout": 105, "ack_failure": True}, diff --git a/docker-compose.yml b/docker-compose.yml index 52add316..117c935d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,6 +103,14 @@ services: - /web/node_modules ports: - 3000:3000 + spring: + build: ./spring + ports: + - "8080:8080" + environment: + - SERVER_PORT=8080 + volumes: + - ./spring:/app volumes: superset_home: diff --git a/spring/.gitignore b/spring/.gitignore index a2e40053..74ff94e7 100644 --- a/spring/.gitignore +++ b/spring/.gitignore @@ -15,4 +15,5 @@ build/* .sts4-cache/ .vscode _site/ -*.css \ No newline at end of file +*.css +libs/* \ No newline at end of file diff --git a/spring/Dockerfile-Openshift b/spring/Dockerfile-Openshift index 9c41618a..87e36f1a 100644 --- a/spring/Dockerfile-Openshift +++ b/spring/Dockerfile-Openshift @@ -26,4 +26,4 @@ RUN ./mvnw dependency:resolve COPY src ./src -CMD ["./mvnw", "-Denvironment=localDev", "spring-boot:run"] +CMD ["./mvnw", "-Denvironment=production", "spring-boot:run"] From 20b31cab892d85eaae13b9a499875e723b063f4d Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:19:21 -0700 Subject: [PATCH 25/67] vin decoding config changes (#363) --- django/workers/scheduled_jobs.py | 2 +- django/workers/tasks.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/django/workers/scheduled_jobs.py b/django/workers/scheduled_jobs.py index e10758a0..5726177c 100644 --- a/django/workers/scheduled_jobs.py +++ b/django/workers/scheduled_jobs.py @@ -47,7 +47,7 @@ def schedule_batch_decode_vins_vinpower(): schedule( "workers.tasks.batch_decode_vins", "vinpower", - 2000, + 1000, name="vinpower_batch_decode_vins", schedule_type="C", cron="*/2 * * * *", diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 636fa8cc..8fd67cbe 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -52,9 +52,11 @@ def batch_decode_vins(service_name, batch_size=50): service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value, "create_timestamp", ] - uploaded_vin_records = UploadedVinRecord.objects.filter(**filters).order_by( - *order_by - )[:batch_size] + uploaded_vin_records = ( + UploadedVinRecord.objects.defer("data") + .filter(**filters) + .order_by(*order_by)[:batch_size] + ) uploaded_vins = set() for uploaded_record in uploaded_vin_records: uploaded_vins.add(uploaded_record.vin) From 1168db327dc0b645ba78b2f423df05772b3abb82 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:31:33 -0700 Subject: [PATCH 26/67] cthub - reduce vinpower batch further (#364) --- django/workers/scheduled_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/workers/scheduled_jobs.py b/django/workers/scheduled_jobs.py index 5726177c..09775486 100644 --- a/django/workers/scheduled_jobs.py +++ b/django/workers/scheduled_jobs.py @@ -47,7 +47,7 @@ def schedule_batch_decode_vins_vinpower(): schedule( "workers.tasks.batch_decode_vins", "vinpower", - 1000, + 500, name="vinpower_batch_decode_vins", schedule_type="C", cron="*/2 * * * *", From b4a3e748d7d8732c4691f19e964f68a88d1e31c4 Mon Sep 17 00:00:00 2001 From: Kuan Fan <31664961+kuanfandevops@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:40:53 -0700 Subject: [PATCH 27/67] add vinpower to test and prod (#367) --- .github/workflows/dev-ci.yaml | 2 +- .github/workflows/prod-ci.yaml | 17 ++++++----------- .github/workflows/test-ci.yaml | 17 ++++++----------- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml index cadddd45..0a8285b6 100644 --- a/.github/workflows/dev-ci.yaml +++ b/.github/workflows/dev-ci.yaml @@ -113,7 +113,7 @@ jobs: with: cmd: | yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml - yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml] + yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml yq -i '.vinpower.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-dev.yaml diff --git a/.github/workflows/prod-ci.yaml b/.github/workflows/prod-ci.yaml index c0a881ba..d73e41a8 100644 --- a/.github/workflows/prod-ci.yaml +++ b/.github/workflows/prod-ci.yaml @@ -70,6 +70,7 @@ jobs: oc tag ${{ env.TEST_NAMESPACE }}/cthub-backend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.PROD_NAMESPACE }}/cthub-backend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} oc tag ${{ env.TEST_NAMESPACE }}/cthub-frontend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.PROD_NAMESPACE }}/cthub-frontend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} oc tag ${{ env.TEST_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.PROD_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} + oc tag ${{ env.TEST_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.PROD_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} - name: Checkout Manifest repository uses: actions/checkout@v4.1.1 @@ -81,17 +82,11 @@ jobs: - name: Update frontend tag uses: mikefarah/yq@v4.40.5 with: - cmd: yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml - - - name: Update backend tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml - - - name: Update backend tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml + cmd: | + yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml + yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml + yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml + yq -i '.vinpower.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-prod.yaml - name: GitHub Commit & Push run: | diff --git a/.github/workflows/test-ci.yaml b/.github/workflows/test-ci.yaml index 3217deb1..29bdad84 100644 --- a/.github/workflows/test-ci.yaml +++ b/.github/workflows/test-ci.yaml @@ -70,6 +70,7 @@ jobs: oc tag ${{ env.DEV_NAMESPACE }}/cthub-backend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.TEST_NAMESPACE }}/cthub-backend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} oc tag ${{ env.DEV_NAMESPACE }}/cthub-frontend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.TEST_NAMESPACE }}/cthub-frontend:${{ env.VERSION }}-${{ env.PRE_RELEASE }} oc tag ${{ env.DEV_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.TEST_NAMESPACE }}/cthub-task-queue:${{ env.VERSION }}-${{ env.PRE_RELEASE }} + oc tag ${{ env.DEV_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} ${{ env.TEST_NAMESPACE }}/cthub-vinpower:${{ env.VERSION }}-${{ env.PRE_RELEASE }} - name: Checkout Manifest repository uses: actions/checkout@v4.1.1 @@ -81,17 +82,11 @@ jobs: - name: Update frontend tag uses: mikefarah/yq@v4.40.5 with: - cmd: yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml - - - name: Update backend tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml - - - name: Update task-queue tag - uses: mikefarah/yq@v4.40.5 - with: - cmd: yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml + cmd: | + yq -i '.frontend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml + yq -i '.backend.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml + yq -i '.task-queue.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml + yq -i '.vinpower.image.tag = "${{ env.VERSION }}-${{ env.PRE_RELEASE }}"' cthub/values-test.yaml - name: GitHub Commit & Push run: | From 9c94c1273432a640a7270c3206d74f5f2b4ce4e2 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:58:39 -0700 Subject: [PATCH 28/67] fix: 366 - (vin decoding jobs)/resolver fixes/changes (#368) --- django/api/services/decoded_vin_record.py | 12 +++++++----- django/api/services/resolvers.py | 3 ++- django/api/utilities/generic.py | 9 +++++++++ django/workers/external_apis/vinpower.py | 2 +- django/workers/external_apis/vpic.py | 2 +- django/workers/tasks.py | 14 +++++++++----- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/django/api/services/decoded_vin_record.py b/django/api/services/decoded_vin_record.py index 9fd2304a..5ec6108e 100644 --- a/django/api/services/decoded_vin_record.py +++ b/django/api/services/decoded_vin_record.py @@ -13,7 +13,7 @@ def save_decoded_data( uploaded_vin_records, vins_to_insert, - decoded_records_to_update_map, + vins_to_decoded_record_ids_map, service_name, decoded_data, ): @@ -34,10 +34,12 @@ def save_decoded_data( decoded_records_to_insert.append( decoded_vin_model(vin=vin, data=decoded_datum) ) - elif vin in decoded_records_to_update_map: - decoded_record_to_update = decoded_records_to_update_map.get(vin) - decoded_record_to_update.update_timestamp = timezone.now() - decoded_record_to_update.data = decoded_datum + elif vin in vins_to_decoded_record_ids_map: + decoded_record_to_update = decoded_vin_model( + id=vins_to_decoded_record_ids_map[vin], + update_timestamp=timezone.now(), + data=decoded_datum, + ) decoded_records_to_update.append(decoded_record_to_update) elif vin in failed_vins: set_decode_successful(service_name, uploaded_record, False) diff --git a/django/api/services/resolvers.py b/django/api/services/resolvers.py index 510a8ee0..65b6785a 100644 --- a/django/api/services/resolvers.py +++ b/django/api/services/resolvers.py @@ -1,7 +1,8 @@ from dns.resolver import Resolver +from email_validator import caching_resolver def get_google_resolver(): resolver = Resolver() resolver.nameservers = ["8.8.8.8"] - return resolver + return caching_resolver(dns_resolver=resolver) diff --git a/django/api/utilities/generic.py b/django/api/utilities/generic.py index 20d5a8d7..98307551 100644 --- a/django/api/utilities/generic.py +++ b/django/api/utilities/generic.py @@ -4,3 +4,12 @@ def get_map(key_name, objects): key = getattr(object, key_name) result[key] = object return result + + +def get_unified_map(key_name, value_name, maps): + result = {} + for map in maps: + key = map.get(key_name) + value = map.get(value_name) + result[key] = value + return result diff --git a/django/workers/external_apis/vinpower.py b/django/workers/external_apis/vinpower.py index 98cb69e7..6c1adf86 100644 --- a/django/workers/external_apis/vinpower.py +++ b/django/workers/external_apis/vinpower.py @@ -14,7 +14,7 @@ def batch_decode(uploaded_vin_records): vins.append(record.vin) headers = {"content-type": "application/json"} response = requests.get(url, data=json.dumps(vins), headers=headers) - response.raise_for_status + response.raise_for_status() data = response.json() for vin in vins: diff --git a/django/workers/external_apis/vpic.py b/django/workers/external_apis/vpic.py index fa941a56..706df439 100644 --- a/django/workers/external_apis/vpic.py +++ b/django/workers/external_apis/vpic.py @@ -17,7 +17,7 @@ def batch_decode(uploaded_vin_records): body = {"format": "json", "data": request_data} response = requests.post(url, data=body) - response.raise_for_status + response.raise_for_status() data = response.json()["Results"] decoded_vins_map = {} for record in data: diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 8fd67cbe..a14b451e 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -3,7 +3,7 @@ from api.models.uploaded_vins_file import UploadedVinsFile from api.models.uploaded_vin_record import UploadedVinRecord from api.constants.decoder import get_service -from api.utilities.generic import get_map +from api.utilities.generic import get_unified_map from api.services.decoded_vin_record import save_decoded_data from api.services.uploaded_vin_record import parse_and_save from django.db import transaction @@ -61,10 +61,14 @@ def batch_decode_vins(service_name, batch_size=50): for uploaded_record in uploaded_vin_records: uploaded_vins.add(uploaded_record.vin) vins_to_update = set() - decoded_records_to_update_map = get_map( - "vin", decoded_vin_model.objects.filter(vin__in=uploaded_vins) + vins_to_decoded_record_ids_map = get_unified_map( + "vin", + "id", + decoded_vin_model.objects.only("id", "vin") + .filter(vin__in=uploaded_vins) + .values(), ) - for decoded_vin in decoded_records_to_update_map: + for decoded_vin in vins_to_decoded_record_ids_map: vins_to_update.add(decoded_vin) vins_to_insert = uploaded_vins.difference(vins_to_update) @@ -74,7 +78,7 @@ def batch_decode_vins(service_name, batch_size=50): save_decoded_data( uploaded_vin_records, vins_to_insert, - decoded_records_to_update_map, + vins_to_decoded_record_ids_map, service_name, decoded_data, ) From fd15645aa2671f2479085c7d128d6ae57d3117b1 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:51:23 -0700 Subject: [PATCH 29/67] Adding function to check if values in fields are valid (#369) --- django/api/constants/constants.py | 7 +++++-- django/api/constants/misc.py | 12 +++++++++++- .../api/services/spreadsheet_uploader_prep.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 8d94c852..0e8ca78c 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -22,9 +22,11 @@ validate_phone_numbers, typo_checker, location_checker, - email_validator + email_validator, + validate_field_values ) from api.services.resolvers import get_google_resolver +from api.constants.misc import GER_VALID_FIELD_VALUES class ARCProjectTrackingColumns(Enum): @@ -656,7 +658,8 @@ class GoElectricRebatesColumnMapping(Enum): {"error_type": "Phone Error", "function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, {"error_type": "Potential Typo", "function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, {"error_type": "Location Not Found", "function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, - {"error_type": "Invalid Email", "function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}} + {"error_type": "Invalid Email", "function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, + {"error_type": "Invalid Value", "function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}} ] }, } diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index 683c9898..c89b2d0f 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -55,7 +55,6 @@ 867, ] - RELEVANT_FEATURES = [ "Canadian Forces Base", "Canadian Forces Station", @@ -77,3 +76,14 @@ "Village (1)", "Town", ] + +GER_VALID_FIELD_VALUES = { + 'Approvals': ['Approved', 'Approved Fraudulent'], + 'Category': [ + 'Forklift', 'Low Speed', 'Motorcycle', 'Medium & Heavy Duty', + 'Airport & Port Specialty Vehicle', 'Cargo E-Bike', 'Utility Vehicle' + ], + 'Fleet/Individuals': ['Fleet', 'Individual'], + 'Rebate adjustment (discount)': ['Yes'], + 'Class': ['2B', '3', '4', '5', '6', '7', '8'] + } \ No newline at end of file diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index b467a398..b530f42a 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -286,4 +286,21 @@ def email_validator(df, *columns, **kwargs): indices.append(index + kwargs.get("indices_offset", 0)) if indices: result[column] = indices + return result + +def validate_field_values(df, *columns, **kwargs): + + allowed_values = kwargs.get("fields_and_values") + + result = {} + for column in df.columns: + if column in allowed_values: + indices = [] + series = df[column] + for index, value in series.items(): + if str(value) not in allowed_values[column]: + indices.append(index + kwargs.get("indices_offset", 0)) + if indices: + result[column] = indices + return result \ No newline at end of file From 2b3b78a8888581a3bbd9e6da5d65983cbff7d55f Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:34:13 -0700 Subject: [PATCH 30/67] Adding checks for allowing or disallowing empty cells. Removing TEMP_NULL bug. (#371) --- .../api/migrations/0031_auto_20240712_2036.py | 28 +++++++++++++++++++ django/api/models/go_electric_rebates.py | 8 +++--- django/api/services/spreadsheet_uploader.py | 22 ++++++++------- .../api/services/spreadsheet_uploader_prep.py | 2 +- 4 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 django/api/migrations/0031_auto_20240712_2036.py diff --git a/django/api/migrations/0031_auto_20240712_2036.py b/django/api/migrations/0031_auto_20240712_2036.py new file mode 100644 index 00000000..d186b5a0 --- /dev/null +++ b/django/api/migrations/0031_auto_20240712_2036.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.25 on 2024-07-12 20:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0030_goelectricrebates_notes'), + ] + + operations = [ + migrations.AlterField( + model_name='goelectricrebates', + name='email', + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AlterField( + model_name='goelectricrebates', + name='phone', + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AlterField( + model_name='goelectricrebates', + name='total_purchase_price', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/django/api/models/go_electric_rebates.py b/django/api/models/go_electric_rebates.py index 759b9f68..3b3a8e75 100644 --- a/django/api/models/go_electric_rebates.py +++ b/django/api/models/go_electric_rebates.py @@ -18,15 +18,15 @@ class GoElectricRebates(Auditable): blank=False, ) total_purchase_price = models.IntegerField( - null=False, - blank=False, + null=True, + blank=True, ) manufacturer = models.CharField(blank=False, max_length=250, null=False) model = models.CharField(blank=False, max_length=250, null=False) city = models.CharField(blank=False, max_length=250, null=False) postal_code = models.CharField(blank=True, max_length=250, null=True) - phone = models.CharField(blank=False, max_length=20, null=False) - email = models.CharField(blank=False, max_length=50, null=False) + phone = models.CharField(blank=True, max_length=20, null=True) + email = models.CharField(blank=True, max_length=50, null=True) vin = models.CharField(blank=True, max_length=100, null=True) vehicle_class = models.CharField(blank=True, null=True, max_length=50) rebate_adjustment = models.CharField(blank=True, null=True, max_length=50) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 436ed09e..24addff9 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -12,11 +12,11 @@ def get_field_default(model, field): def get_nullable_fields(model): - nullable_fields = {} + nullable_fields = [] for field in model._meta.get_fields(): if hasattr(field, "null") and field.null: - nullable_fields[field.name] = True + nullable_fields.append(field.name) return nullable_fields @@ -28,7 +28,6 @@ def trim_all_columns(df): def extract_data(excel_file, sheet_name, header_row): try: df = pd.read_excel(excel_file, sheet_name, header=header_row) - df = df.fillna('TEMP_NULL') df = trim_all_columns(df) return df except Exception as e: @@ -92,7 +91,15 @@ def load_data(df, model, field_types, replace_data, user, validation_errors): expected_type = field_types.get(column) is_nullable = column in nullable_fields - if expected_type in [int, float, Decimal] and value != 'TEMP_NULL': + if pd.isna(value) or value == "" or value is None: + if is_nullable: + row_dict[column] = None + else: + errors.append(f"Row {index + 1}: Has an empty cell where one is expected in '{column}'") + valid_row = False + continue + + if expected_type in [int, float, Decimal] and value != None and pd.notna(value): value = str(value).replace(',', '').strip() try: if expected_type == int: @@ -108,12 +115,6 @@ def load_data(df, model, field_types, replace_data, user, validation_errors): valid_row = False continue - if pd.isna(value) or value == "" or value == 'TEMP_NULL': - if is_nullable: - row_dict[column] = None - else: - row_dict[column] = get_field_default(model, column) - elif not isinstance(row_dict[column], expected_type) and value != "": errors.append( f"Row {index + 1}: Incorrect type for '{column}'. Expected {expected_type.__name__}, got {type(row_dict[column]).__name__}." @@ -200,6 +201,7 @@ def import_from_xls( "errors": result["errors"], "rows_processed": result["row_count"], } + else: return { "success": True, diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index b530f42a..638d5b53 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -298,7 +298,7 @@ def validate_field_values(df, *columns, **kwargs): indices = [] series = df[column] for index, value in series.items(): - if str(value) not in allowed_values[column]: + if str(value) not in allowed_values[column] and value != '' and value != None and not pd.isna(value): indices.append(index + kwargs.get("indices_offset", 0)) if indices: result[column] = indices From c32f45b9b5cc830c88efa196333d12d97613ac3e Mon Sep 17 00:00:00 2001 From: Samuel Chukwuma Arawu <75506215+ArawuSamuel1@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:16:56 -0700 Subject: [PATCH 31/67] Update user-story.md (#373) --- .github/ISSUE_TEMPLATE/user-story.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/user-story.md b/.github/ISSUE_TEMPLATE/user-story.md index 98ef72cc..91938c94 100644 --- a/.github/ISSUE_TEMPLATE/user-story.md +++ b/.github/ISSUE_TEMPLATE/user-story.md @@ -10,6 +10,7 @@ assignees: '' **Title:** **Description:** +- [ ] "As a [type of user]" "I want [an action or feature]" "So that [benefit or value]" - **Wireframe:** From 6424ca9261e4c06c0650311b59f1777076a34ac8 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:38:44 -0700 Subject: [PATCH 32/67] feat: adds components for displaying issues, modifies popup (#370) * feat: (partial) adds components for displaying issues, modifies popup * -feat: new issues component working as expected with errors and warnings split up * -adds issue type to detail ie 'warning name' or 'error name' --- frontend/src/app/components/AlertDialog.js | 36 ++- frontend/src/app/styles/App.scss | 16 +- frontend/src/app/styles/FileUpload.scss | 6 + frontend/src/uploads/UploadContainer.js | 243 ++++++++++++++---- .../src/uploads/components/UploadIssues.js | 122 +++++++++ .../uploads/components/UploadIssuesDetail.js | 85 ++++++ 6 files changed, 436 insertions(+), 72 deletions(-) create mode 100644 frontend/src/uploads/components/UploadIssues.js create mode 100644 frontend/src/uploads/components/UploadIssuesDetail.js diff --git a/frontend/src/app/components/AlertDialog.js b/frontend/src/app/components/AlertDialog.js index fe9977b0..c526b27a 100644 --- a/frontend/src/app/components/AlertDialog.js +++ b/frontend/src/app/components/AlertDialog.js @@ -6,6 +6,7 @@ import DialogActions from "@mui/material/DialogActions"; import DialogContent from "@mui/material/DialogContent"; import DialogContentText from "@mui/material/DialogContentText"; import DialogTitle from "@mui/material/DialogTitle"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; const AlertDialog = (props) => { const { @@ -31,11 +32,13 @@ const AlertDialog = (props) => { aria-labelledby="alert-dialog-title" aria-describedby="alert-dialog-description" > - {title} + + {title} + - + {dialogue} - + - + {confirmText && ( + + )}
@@ -62,14 +67,17 @@ const AlertDialog = (props) => { AlertDialog.defaultProps = { dialogue: "", title: "", + cancelText: "cancel", + confirmText: "", }; AlertDialog.propTypes = { open: PropTypes.bool.isRequired, title: PropTypes.string, - dialogue: PropTypes.string, - cancelText: PropTypes.string.isRequired, + dialogue: PropTypes.oneOfType([PropTypes.string, PropTypes.object]) + .isRequired, + cancelText: PropTypes.string, handleCancel: PropTypes.func.isRequired, - confirmText: PropTypes.string.isRequired, + confirmText: PropTypes.string, handleConfirm: PropTypes.func.isRequired, }; diff --git a/frontend/src/app/styles/App.scss b/frontend/src/app/styles/App.scss index 86664c1e..9e130984 100644 --- a/frontend/src/app/styles/App.scss +++ b/frontend/src/app/styles/App.scss @@ -14,6 +14,9 @@ $default-link-blue: #568dba; $default-background-grey: #f2f2f2; $md: 991px; $button-background-blue: #003366; +$error-red: #ce3e39; +$warning-yellow: #fcba19; +$default-blue: #003366; .App { background-color: $default-background-grey; @@ -63,7 +66,7 @@ h2, h3, h4 { font-family: "Roboto", "Open Sans", sans-serif; - color: #003366; + color: $default-blue; font-weight: 500; } @@ -110,3 +113,14 @@ h4 { .page-content { flex-grow: 1; } + +.error { + color: $error-red; +} +.warning { + color: $warning-yellow; +} +.showMore { + color: $default-link-blue; + text-decoration: underline; +} \ No newline at end of file diff --git a/frontend/src/app/styles/FileUpload.scss b/frontend/src/app/styles/FileUpload.scss index ee6608c5..1a26f7b5 100644 --- a/frontend/src/app/styles/FileUpload.scss +++ b/frontend/src/app/styles/FileUpload.scss @@ -43,3 +43,9 @@ background-color: $default-background-grey; } } +.cancel-button { + color: $default-blue !important; +} +.confirm-button { + background-color: $default-blue !important; +} diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index b52b355b..8b185767 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -9,6 +9,7 @@ import UsersContainer from "../users/UsersContainer"; import Loading from "../app/components/Loading"; import useAxios from "../app/utilities/useAxios"; import WarningsList from "./components/WarningsList"; +import UploadIssues from "./components/UploadIssues"; const UploadContainer = () => { const [uploadFiles, setUploadFiles] = useState([]); // array of objects for files to be uploaded @@ -23,16 +24,21 @@ const UploadContainer = () => { const [alertSeverity, setAlertSeverity] = useState(""); const [openDialog, setOpenDialog] = useState(false); const [adminUser, setAdminUser] = useState(false); - const axios = useAxios(); - const axiosDefault = useAxios(true); - const [dataWarning, setDataWarning] = useState({}) + const [totalIssueCount, setTotalIssueCount] = useState({}); + const [groupedErrors, setGroupedErrors] = useState({}); + const [groupedWarnings, setGroupedWarnings] = useState({}); const [alertDialogText, setAlertDialogText] = useState({ title: "", content: "", confirmText: "", - confirmAction: ()=>{}, - cancelAction: ()=>{}, - }) + confirmAction: () => {}, + cancelAction: () => {}, + cancelText: "cancel", + }); + + const axios = useAxios(); + const axiosDefault = useAxios(true); + const refreshList = () => { setRefresh(true); axios.get(ROUTES_UPLOAD.LIST).then((response) => { @@ -51,26 +57,80 @@ const UploadContainer = () => { }); }; + const groupAndCountRows = (issueArray) => { + const groupedErrors = {}; + const groupedWarnings = {}; + const totalIssueCount = { + errors: 0, + warnings: 0, + }; + + issueArray.forEach((issue) => { + const column = Object.keys(issue)[0]; + const errorDetails = issue[column]; + + Object.keys(errorDetails).forEach((errorType) => { + const severity = errorDetails[errorType].Severity; + const expectedType = errorDetails[errorType]["Expected Type"]; + const expectedFormat = errorDetails[errorType]["Expected Format"]; + const rows = errorDetails[errorType].Rows; + const rowCount = rows.length; + + if (severity === "Error") { + totalIssueCount.errors += rowCount; + if (!groupedErrors[column]) { + groupedErrors[column] = {}; + } + if (!groupedErrors[column][errorType]) { + groupedErrors[column][errorType] = { + ExpectedType: expectedType, + Rows: rows, + }; + } + } else if (severity === "Warning") { + totalIssueCount.warnings += rowCount; + if (!groupedWarnings[column]) { + groupedWarnings[column] = {}; + } + if (!groupedWarnings[column][errorType]) { + groupedWarnings[column][errorType] = { + ExpectedFormat: expectedFormat, + Rows: rows, + }; + } + } + }); + }); + + return { groupedErrors, groupedWarnings, totalIssueCount }; + }; + const showError = (error) => { const { response: errorResponse } = error; - setAlertContent("There was an issue uploading the file.") + setAlertContent("There was an issue uploading the file."); if (errorResponse && errorResponse.data && errorResponse.data.message) { setAlertContent( `${errorResponse.data.message}\n${errorResponse.data.errors ? "Errors: " + errorResponse.data.errors.join("\n") : ""}`, - ) - } else if (errorResponse && errorResponse.data && errorResponse.status === 403) { - setAlertContent("There was an error. Please refresh page and ensure you are logged in.") + ); + } else if ( + errorResponse && + errorResponse.data && + errorResponse.status === 403 + ) { + setAlertContent( + "There was an error. Please refresh page and ensure you are logged in.", + ); } setAlertSeverity("error"); setAlert(true); }; - const doUpload = (checkForWarnings) => + const doUpload = (checkForWarnings) => { uploadFiles.forEach((file) => { let filepath = file.path; - setLoading(true); - if (datasetSelected !== 'Go Electric Rebates Program'){ - checkForWarnings = false + setLoading(true); + if (datasetSelected !== "Go Electric Rebates Program") { + checkForWarnings = false; } const uploadPromises = uploadFiles.map((file) => { return axios.get(ROUTES_UPLOAD.MINIO_URL).then((response) => { @@ -81,7 +141,7 @@ const UploadContainer = () => { datasetSelected, replaceData, filepath, - checkForWarnings + checkForWarnings, }); }); }); @@ -94,44 +154,98 @@ const UploadContainer = () => { setAlertSeverity(errorCheck ? "success" : "error"); const message = responses - .map( - (response) => - `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, - ) - .join("\n"); + .map( + (response) => + `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, + ) + .join("\n"); setAlert(true); setAlertContent(message); - const warnings = {} + const warnings = {}; for (const [index, response] of responses.entries()) { - const filename = uploadFiles[index].name - const responseWarnings = response.data.warnings + const filename = uploadFiles[index].name; + const responseWarnings = response.data.warnings; if (responseWarnings) { - warnings[filename] = responseWarnings + warnings[filename] = responseWarnings; } } setAlertContent(message); - if (Object.keys(warnings).length > 0 && checkForWarnings == true) { // ie it is the first attempt to upload (when upload is called from the dialog its set to false) - setOpenDialog(true) + if (Object.keys(warnings).length > 0 && checkForWarnings === true) { + // ie it is the first attempt to upload (when upload is called from the dialog its set to false) + const fakeResponse = [ + { + // 'Applicant Name': { + // "blank": { + // "Expected Type": "must not be blank", + // Severity: "Error", + // Rows: [ + // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + // 18, 19, 20, 21, 22, 23, 24, + // ], + // }, + // }, + 'Phone': { + "phone number not formatted correctly": { + "Expected Type": "213-1234-1231", + Severity: "Warning", + Rows: [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, + ], + }, + }, + }, + { + "Company Name": { + "contains null values": { + "Expected Format": "Smith, John", + Severity: "Warning", + Rows: [ + 9, 12, 13, 14, 15, 16, 17, 28, 27, 43, 23, 2323, 24, 25, + 65, 342, 23, 7, 56, 53, 56, 67, 78, 89, 45, 3, 2, 1, 54, + 56, 76, 78, 79, 90, 34, 23, 22, 21, 255, 26, 27, 27, 28, + ], + }, + }, + }, + ]; + // Call groupAndCountRows to get data and pass to state + const { groupedErrors, groupedWarnings, totalIssueCount } = + groupAndCountRows(fakeResponse); + setGroupedErrors(groupedErrors); + setGroupedWarnings(groupedWarnings); + setTotalIssueCount(totalIssueCount); + //popup for showing issues setAlertDialogText({ - title: "Warning: There are errors in the data to review", - content:( + title: + "Your file has been processed and contains the following errors and warnings!", + content: ( <> -
-

- Click continue to insert these records as is, or click cancel - to exit out and no records will be inserted: -

- -
+ {totalIssueCount.errors >= 1 && ( +
+ + {totalIssueCount.errors} Errors + {" "} + - Must fix before uploading +
+ )} + {totalIssueCount.warnings >= 1 && ( +
+ + {totalIssueCount.warnings} Warnings + {" "} + - Can upload without fixing +
+ )} - ), - confirmText: "Continue (all records will be inserted as is)", - confirmAction: handleConfirmDataInsert, - cancelAction: handleReplaceDataCancel, - })} - - setUploadFiles([]); + ), + cancelAction: () => setOpenDialog(false), + confirmText: "View Details", + confirmAction: () => setOpenDialog(false), + }); + setOpenDialog(true); + } }) .catch((error) => { showError(error); @@ -140,6 +254,7 @@ const UploadContainer = () => { setLoading(false); }); }); + }; const downloadSpreadsheet = () => { axios @@ -170,24 +285,31 @@ const UploadContainer = () => { const choice = event.target.value; if (choice === "replace") { setOpenDialog(true); + //popup for replacing data setAlertDialogText({ title: "Replace existing data?", - content: "Selecting replace will delete all previously uploaded records for this dataset", - confirmText: "Replace existing data", + content: + "Selecting replace will delete all previously uploaded records for this dataset", + confirmText: "Replace existing data", confirmAction: handleReplaceDataConfirm, cancelAction: handleReplaceDataCancel, - }) + }); } else { setReplaceData(false); } }; + const handleConfirmDataInsert = () => { + setGroupedWarnings({}) + setGroupedErrors({}) + setTotalIssueCount({}) setOpenDialog(false); - showError(false); - setAlertContent("") - doUpload(false); //upload with the checkForWarnings flag set to false! + setAlert(false); + setAlertContent(""); + doUpload(false); // Upload with the checkForWarnings flag set to false! + setUploadFiles([]) + }; - } const handleReplaceDataConfirm = () => { setReplaceData(true); setOpenDialog(false); @@ -198,14 +320,13 @@ const UploadContainer = () => { }; useEffect(() => { - refreshList(true); + refreshList(); }, []); if (refresh) { return ; } - const alertElement = alert && alertContent && alertSeverity ? ( @@ -218,7 +339,6 @@ const UploadContainer = () => { ) : null; - return (
@@ -226,15 +346,23 @@ const UploadContainer = () => { + {(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( + + + + )} { /> {adminUser && ( - + )} @@ -263,4 +391,5 @@ const UploadContainer = () => {
); }; + export default withRouter(UploadContainer); diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js new file mode 100644 index 00000000..edcdc007 --- /dev/null +++ b/frontend/src/uploads/components/UploadIssues.js @@ -0,0 +1,122 @@ +import React, { useState } from "react"; +import { + Box, + Typography, + AccordionSummary, + AccordionDetails, + Accordion, + Button, +} from "@mui/material"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import UploadIssuesDetail from "./UploadIssuesDetail"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; + +const UploadIssues = ({ + confirmUpload, + groupedErrors, + groupedWarnings, + totalIssueCount, +}) => { + const [showAllIssues, setShowAllIssues] = useState(false); + + const toggleShowAllIssues = () => { + setShowAllIssues(!showAllIssues); + }; + + const errorMsg = "Must fix before uploading"; + const warningMsg = "Can upload without fixing"; + + return ( + <> + +

+ + Your file upload results +

+ + Your file has been processed and contains the following errors and + warnings. Please review them below: + + {totalIssueCount.errors >= 1 && ( + + + {totalIssueCount.errors} Errors   + + - {errorMsg} + + )} + {totalIssueCount.warnings >= 1 && ( + + + {totalIssueCount.warnings} Warnings   + + - {warningMsg} + + )} + + + + {showAllIssues ? "Show less" : "Show more"} + + + + + {totalIssueCount.errors >= 1 && ( + + )} + {totalIssueCount.warnings >= 1 && ( + + )} + + + {totalIssueCount.warnings >= 1 && totalIssueCount.errors === 0 && ( + +

Do you want to upload the file regardless of the warnings?

+ + + + + + )} + + + ); +}; + +export default UploadIssues; diff --git a/frontend/src/uploads/components/UploadIssuesDetail.js b/frontend/src/uploads/components/UploadIssuesDetail.js new file mode 100644 index 00000000..2a447862 --- /dev/null +++ b/frontend/src/uploads/components/UploadIssuesDetail.js @@ -0,0 +1,85 @@ +import PropTypes from "prop-types"; +import React, { useState } from "react"; +import { Box, Button } from "@mui/material"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; + +const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { + const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue + const classname = type === "error" ? "error" : "warning"; + const toggleShowAllRows = (column, errorType) => { + const key = `${column}_${errorType}`; + setShowAllRowsMap((prevState) => ({ + ...prevState, + [key]: !prevState[key], + })); + }; + + return ( + + + + + {totalIssueCount} {type}  + + + ({msg}) + {Object.keys(issues).map((column) => ( + + Column: {column} + {Object.keys(issues[column]).map((errorType, index) => ( +
+
{type.charAt(0).toUpperCase() + type.slice(1)} Name: {errorType}
+
+ Expected value:{" "} + {issues[column][errorType].ExpectedType || + issues[column][errorType].ExpectedFormat} +
+
+ Rows with {type}:{" "} + + {issues[column][errorType].Rows.slice( + 0, + showAllRowsMap[`${column}_${errorType}`] ? undefined : 15, + ).join(", ")} + {issues[column][errorType].Rows.length > 15 && + !showAllRowsMap[`${column}_${errorType}`] && + "..."} + +
+ {issues[column][errorType].Rows.length > 15 && ( + + )} +
+ ))} + + ))} + + ); +}; + +UploadIssuesDetail.propTypes = { + type: PropTypes.string.isRequired, + issues: PropTypes.object.isRequired, + totalIssueCount: PropTypes.number.isRequired, + msg: PropTypes.string.isRequired, +}; + +export default UploadIssuesDetail; From ed70a41377ff3c2dc34d84960eca416abc95b0f6 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:26:03 -0700 Subject: [PATCH 33/67] Feat: Data Uploader Refactor #365 (#374) * feat: (partial) adds components for displaying issues, modifies popup * -feat: new issues component working as expected with errors and warnings split up * -adds issue type to detail ie 'warning name' or 'error name' * First version of backend refactor * Cleanup --------- Co-authored-by: emi-hi Co-authored-by: Emily <44536222+emi-hi@users.noreply.github.com> --- django/api/constants/constants.py | 10 +- django/api/services/spreadsheet_uploader.py | 181 ++++----- .../api/services/spreadsheet_uploader_prep.py | 58 ++- frontend/src/uploads/UploadContainer.js | 371 ++++++++---------- .../src/uploads/components/UploadIssues.js | 4 +- .../uploads/components/UploadIssuesDetail.js | 2 +- 6 files changed, 305 insertions(+), 321 deletions(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 0e8ca78c..b1570408 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -655,11 +655,11 @@ class GoElectricRebatesColumnMapping(Enum): "sheet_name": "Distribution List - Master", "preparation_functions": [prepare_go_electric_rebates], "validation_functions": [ - {"error_type": "Phone Error", "function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, - {"error_type": "Potential Typo", "function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, - {"error_type": "Location Not Found", "function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, - {"error_type": "Invalid Email", "function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, - {"error_type": "Invalid Value", "function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}} + {"function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, + {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, + {"function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, + {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, + {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}} ] }, } diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 24addff9..1c9cb552 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -39,6 +39,8 @@ def transform_data( df, dataset_columns, column_mapping_enum, + field_types, + model, preparation_functions=[], validation_functions=[], ): @@ -47,45 +49,17 @@ def transform_data( df = df[[col for col in df.columns if col in required_columns]] missing_columns = [col for col in required_columns if col not in df.columns] - if missing_columns: + if (missing_columns): raise ValueError(f"Missing columns: {', '.join(missing_columns)}") for prep_func in preparation_functions: df = prep_func(df) - validation_errors = {} - for x in validation_functions: - validate = x["function"] - columns = x["columns"] - kwargs = x["kwargs"] - key = x["error_type"] - errors = validate(df, *columns, **kwargs) - if errors: - validation_errors[key] = errors - - column_mapping = {col.name: col.value for col in column_mapping_enum} - # Need to use the inverse (keys) for mapping the columns to what the database expects in order to use enums - inverse_column_mapping = {v: k for k, v in column_mapping.items()} - df.rename(columns=inverse_column_mapping, inplace=True) - - return df, validation_errors - - -@transaction.atomic -def load_data(df, model, field_types, replace_data, user, validation_errors): - row_count = 0 - records_inserted = 0 - errors = [] nullable_fields = get_nullable_fields(model) - - # validation_error_rows = get_validation_error_rows(errors) This may be used going forward for validation errors that cannot be overwritten. - - if replace_data: - model.objects.all().delete() + errors_and_warnings = {} for index, row in df.iterrows(): row_dict = row.to_dict() - valid_row = True for column, value in row_dict.items(): expected_type = field_types.get(column) @@ -95,11 +69,17 @@ def load_data(df, model, field_types, replace_data, user, validation_errors): if is_nullable: row_dict[column] = None else: - errors.append(f"Row {index + 1}: Has an empty cell where one is expected in '{column}'") - valid_row = False - continue - - if expected_type in [int, float, Decimal] and value != None and pd.notna(value): + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Empty Value" not in errors_and_warnings[column]: + errors_and_warnings[column]["Empty Value"] = { + "Expected Type": "Expected value where there isn't one.", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) + + if expected_type in [int, float, Decimal] and value is not None and pd.notna(value): value = str(value).replace(',', '').strip() try: if expected_type == int: @@ -109,39 +89,73 @@ def load_data(df, model, field_types, replace_data, user, validation_errors): else: row_dict[column] = float(value) except ValueError: - errors.append( - f"Row {index + 1}: Unable to convert value to {expected_type.__name__} for '{column}'. Value was '{value}'." - ) - valid_row = False - continue - - elif not isinstance(row_dict[column], expected_type) and value != "": - errors.append( - f"Row {index + 1}: Incorrect type for '{column}'. Expected {expected_type.__name__}, got {type(row_dict[column]).__name__}." - ) - valid_row = False - continue - - # if index + 1 in validation_error_rows: - # valid_row = False - # continue - - if valid_row: - try: - row_dict["update_user"] = user - model_instance = model(**row_dict) - model_instance.full_clean() - model_instance.save() - records_inserted += 1 - except Exception as e: - errors.append(f"Row {index + 1}: {e}") - - row_count += 1 + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Incorrect Type" not in errors_and_warnings[column]: + errors_and_warnings[column]["Incorrect Type"] = { + "Expected Type": "The following rows contained incorrect value types for the " + column + " column", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) + + # Check if expected_type is valid before using isinstance + elif expected_type is not None and isinstance(expected_type, type) and not isinstance(row_dict[column], expected_type) and value != "": + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Incorrect Type" not in errors_and_warnings[column]: + errors_and_warnings[column]["Incorrect Type"] = { + "Expected Type": "The following rows contained incorrect value types for the " + column + " column", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) + + for x in validation_functions: + validate = x["function"] + columns = x["columns"] + kwargs = x["kwargs"] + warnings = validate(df, *columns, **kwargs) + + if warnings: + for column, issues in warnings.items(): + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + for issue, details in issues.items(): + if issue not in errors_and_warnings[column]: + errors_and_warnings[column][issue] = { + "Expected Type": details.get("Expected Type", "Unknown"), + "Rows": details.get("Rows", []), + "Severity": details.get("Severity", "Error") + } + else: + errors_and_warnings[column][issue]["Rows"].extend(details.get("Rows", [])) + + column_mapping = {col.name: col.value for col in column_mapping_enum} + inverse_column_mapping = {v: k for k, v in column_mapping.items()} + df.rename(columns=inverse_column_mapping, inplace=True) + + return df, errors_and_warnings + + +@transaction.atomic +def load_data(df, model, replace_data, user): + records_inserted = 0 + + if replace_data: + model.objects.all().delete() + + for index, row in df.iterrows(): + row_dict = row.to_dict() + row_dict["update_user"] = user + + model_instance = model(**row_dict) + model_instance.save() + records_inserted += 1 return { - "row_count": row_count, + "row_count": len(df), "records_inserted": records_inserted, - "errors": sorted(errors, key=lambda x: int(x.split()[1][:-1])), } @@ -157,14 +171,16 @@ def import_from_xls( user, preparation_functions=[], validation_functions=[], - check_for_warnings=False, + check_for_warnings=True, ): try: df = extract_data(excel_file, sheet_name, header_row) - df, validation_errors = transform_data( + df, errors_and_warnings = transform_data( df, dataset_columns, column_mapping_enum, + field_types, + model, preparation_functions, validation_functions, ) @@ -172,42 +188,27 @@ def import_from_xls( if check_for_warnings: ## do the error checking - if validation_errors: + if errors_and_warnings: return { "success": True, "message": "We encountered some potential errors in your data. Please choose whether to ignore them and continue inserting data or cancel upload and make edits to the data before reuploading", "warning": True, - "warnings": validation_errors, + "errors_and_warnings": errors_and_warnings, } else: print('no warnings') - result = load_data(df, model, field_types, replace_data, user, validation_errors) + result = load_data(df, model, replace_data, user, errors_and_warnings) total_rows = result["row_count"] inserted_rows = result["records_inserted"] - if result["errors"] and result["records_inserted"] > 0: - return { - "success": True, - "message": f"{inserted_rows} out of {total_rows} rows successfully inserted with some errors encountered.", - "errors": result["errors"], - "rows_processed": result["row_count"], - } - elif len(result["errors"]) > 0: - return { - "success": False, - "message": "Errors encountered with no successful insertions.", - "errors": result["errors"], - "rows_processed": result["row_count"], - } - - else: - return { - "success": True, - "message": f"All {inserted_rows} records successfully inserted out of {total_rows}.", - "rows_processed": result["row_count"], + return { + "success": True, + "message": f"All {inserted_rows} records successfully inserted out of {total_rows}.", + "rows_processed": result["row_count"], } + except Exception as error: traceback.print_exc() error_msg = f"Unexpected error: {str(error)}" diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 638d5b53..3753c823 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -208,7 +208,7 @@ def typo_checker(df, *columns, **kwargs): matches = dl.get_close_matches( value, unique_vals.difference(singleton), - cutoff = kwargs["cutoff"] + cutoff=kwargs["cutoff"] ) if matches: value_indices = map_of_values_to_indices[value] @@ -219,21 +219,16 @@ def typo_checker(df, *columns, **kwargs): match_indices = map_of_values_to_indices[match] indices.extend(match_indices) if indices: - result[column] = sorted(list(set(indices))) + result[column] = { + "Similar Values Detected": { + "Expected Type": "We detected applicant names that sound very similar. If these names refer to the same person/entity, please replace the applicant names in your dataset to the preferred spelling to ensure consistency", + "Rows": sorted(list(set(indices))), + "Severity": "Warning" + } + } return result -def get_validation_error_rows(errors): - row_numbers = set() - for error in errors: - try: - row_number = int(error.split()[1][:-1]) - row_numbers.add(row_number) - except (IndexError, ValueError): - continue - return row_numbers - - def validate_phone_numbers(df, *columns, **kwargs): result = {} for column in columns: @@ -244,7 +239,13 @@ def validate_phone_numbers(df, *columns, **kwargs): if formatted_number == '' or len(formatted_number) != 10 or int(formatted_number[:3]) not in AREA_CODES: indices.append(index + kwargs.get("indices_offset", 0)) if indices: - result[column] = indices + result[column] = { + "Phone Number Appears Incorrect": { + "Expected Type": "Ensure phone numbers match the Canadian format (XXX-XXX-XXXX)", + "Rows": indices, + "Severity": "Warning" + } + } return result @@ -265,8 +266,13 @@ def location_checker(df, *columns, **kwargs): indices_to_add = map_of_values_to_indices[name] indices.extend(indices_to_add) if indices: - indices.sort() - result[column] = indices + result[column] = { + "Unrecognized City Names": { + "Expected Type": "The following city names are not in the list of geographic names. Please double check that these places exist or have correct spelling and adjust your dataset accordingly.", + "Rows": sorted(list(set(indices))), + "Severity": "Warning" + } + } return result @@ -285,11 +291,17 @@ def email_validator(df, *columns, **kwargs): except EmailNotValidError: indices.append(index + kwargs.get("indices_offset", 0)) if indices: - result[column] = indices + result[column] = { + "Possible Errors in Email Addresses": { + "Expected Type": "Verify email addresses are valid", + "Rows": indices, + "Severity": "Warning" + } + } return result -def validate_field_values(df, *columns, **kwargs): +def validate_field_values(df, *columns, **kwargs): allowed_values = kwargs.get("fields_and_values") result = {} @@ -298,9 +310,15 @@ def validate_field_values(df, *columns, **kwargs): indices = [] series = df[column] for index, value in series.items(): - if str(value) not in allowed_values[column] and value != '' and value != None and not pd.isna(value): + if str(value) not in allowed_values[column] and value != '' and value is not None and not pd.isna(value): indices.append(index + kwargs.get("indices_offset", 0)) if indices: - result[column] = indices + result[column] = { + "Invalid Values": { + "Expected Type": "The following rows only allow specific values", + "Rows": indices, + "Severity": "Error" + } + } return result \ No newline at end of file diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index 8b185767..a30cc902 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -64,46 +64,53 @@ const UploadContainer = () => { errors: 0, warnings: 0, }; - + issueArray.forEach((issue) => { - const column = Object.keys(issue)[0]; - const errorDetails = issue[column]; - - Object.keys(errorDetails).forEach((errorType) => { - const severity = errorDetails[errorType].Severity; - const expectedType = errorDetails[errorType]["Expected Type"]; - const expectedFormat = errorDetails[errorType]["Expected Format"]; - const rows = errorDetails[errorType].Rows; - const rowCount = rows.length; - - if (severity === "Error") { - totalIssueCount.errors += rowCount; - if (!groupedErrors[column]) { - groupedErrors[column] = {}; - } - if (!groupedErrors[column][errorType]) { - groupedErrors[column][errorType] = { - ExpectedType: expectedType, - Rows: rows, - }; - } - } else if (severity === "Warning") { - totalIssueCount.warnings += rowCount; - if (!groupedWarnings[column]) { - groupedWarnings[column] = {}; - } - if (!groupedWarnings[column][errorType]) { - groupedWarnings[column][errorType] = { - ExpectedFormat: expectedFormat, - Rows: rows, - }; + + Object.keys(issue).forEach((column) => { + const errorDetails = issue[column]; + + Object.keys(errorDetails).forEach((errorType) => { + const severity = errorDetails[errorType].Severity; + const expectedType = errorDetails[errorType]["Expected Type"]; + const rows = errorDetails[errorType].Rows; + const rowCount = rows.length; + + if (severity === "Error") { + totalIssueCount.errors += rowCount; + if (!groupedErrors[column]) { + groupedErrors[column] = {}; + } + if (!groupedErrors[column][errorType]) { + groupedErrors[column][errorType] = { + ExpectedType: expectedType, + Rows: rows, + }; + } else { + groupedErrors[column][errorType].Rows.push(...rows); + } + } else if (severity === "Warning") { + totalIssueCount.warnings += rowCount; + if (!groupedWarnings[column]) { + groupedWarnings[column] = {}; + } + if (!groupedWarnings[column][errorType]) { + groupedWarnings[column][errorType] = { + ExpectedType: expectedType, + Rows: rows, + }; + } else { + groupedWarnings[column][errorType].Rows.push(...rows); + } } - } + }); }); }); - + return { groupedErrors, groupedWarnings, totalIssueCount }; }; + + const showError = (error) => { const { response: errorResponse } = error; @@ -126,134 +133,92 @@ const UploadContainer = () => { }; const doUpload = (checkForWarnings) => { - uploadFiles.forEach((file) => { + setLoading(true); + + const uploadPromises = uploadFiles.map((file) => { let filepath = file.path; - setLoading(true); - if (datasetSelected !== "Go Electric Rebates Program") { - checkForWarnings = false; - } - const uploadPromises = uploadFiles.map((file) => { - return axios.get(ROUTES_UPLOAD.MINIO_URL).then((response) => { - const { url: uploadUrl, minio_object_name: filename } = response.data; - return axiosDefault.put(uploadUrl, file).then(() => { - return axios.post(ROUTES_UPLOAD.UPLOAD, { - filename, - datasetSelected, - replaceData, - filepath, - checkForWarnings, - }); + return axios.get(ROUTES_UPLOAD.MINIO_URL).then((response) => { + const { url: uploadUrl, minio_object_name: filename } = response.data; + return axiosDefault.put(uploadUrl, file).then(() => { + return axios.post(ROUTES_UPLOAD.UPLOAD, { + filename, + datasetSelected, + replaceData, + filepath, + checkForWarnings, }); }); }); - Promise.all(uploadPromises) - .then((responses) => { - const errorCheck = responses.some( - (response) => response.data.success, - ); + }); - setAlertSeverity(errorCheck ? "success" : "error"); - const message = responses - .map( - (response) => - `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, - ) - .join("\n"); - setAlert(true); - setAlertContent(message); - const warnings = {}; - for (const [index, response] of responses.entries()) { - const filename = uploadFiles[index].name; - const responseWarnings = response.data.warnings; - if (responseWarnings) { - warnings[filename] = responseWarnings; - } - } - setAlertContent(message); + Promise.all(uploadPromises) + .then((responses) => { + const errorCheck = responses.some( + (response) => !response.data.success + ); + + setAlertSeverity(errorCheck ? "error" : "success"); + const message = responses + .map( + (response) => + `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}` + ) + .join("\n"); + setAlert(true); + setAlertContent(message); - if (Object.keys(warnings).length > 0 && checkForWarnings === true) { - // ie it is the first attempt to upload (when upload is called from the dialog its set to false) - const fakeResponse = [ - { - // 'Applicant Name': { - // "blank": { - // "Expected Type": "must not be blank", - // Severity: "Error", - // Rows: [ - // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - // 18, 19, 20, 21, 22, 23, 24, - // ], - // }, - // }, - 'Phone': { - "phone number not formatted correctly": { - "Expected Type": "213-1234-1231", - Severity: "Warning", - Rows: [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, - ], - }, - }, - }, - { - "Company Name": { - "contains null values": { - "Expected Format": "Smith, John", - Severity: "Warning", - Rows: [ - 9, 12, 13, 14, 15, 16, 17, 28, 27, 43, 23, 2323, 24, 25, - 65, 342, 23, 7, 56, 53, 56, 67, 78, 89, 45, 3, 2, 1, 54, - 56, 76, 78, 79, 90, 34, 23, 22, 21, 255, 26, 27, 27, 28, - ], - }, - }, - }, - ]; - // Call groupAndCountRows to get data and pass to state - const { groupedErrors, groupedWarnings, totalIssueCount } = - groupAndCountRows(fakeResponse); - setGroupedErrors(groupedErrors); - setGroupedWarnings(groupedWarnings); - setTotalIssueCount(totalIssueCount); - //popup for showing issues - setAlertDialogText({ - title: - "Your file has been processed and contains the following errors and warnings!", - content: ( - <> - {totalIssueCount.errors >= 1 && ( -
- - {totalIssueCount.errors} Errors - {" "} - - Must fix before uploading -
- )} - {totalIssueCount.warnings >= 1 && ( -
- - {totalIssueCount.warnings} Warnings - {" "} - - Can upload without fixing -
- )} - - ), - cancelAction: () => setOpenDialog(false), - confirmText: "View Details", - confirmAction: () => setOpenDialog(false), - }); - setOpenDialog(true); + const warnings = {}; + responses.forEach((response, index) => { + const responseWarnings = response.data.errors_and_warnings; + if (responseWarnings) { + warnings[uploadFiles[index].name] = responseWarnings; } - }) - .catch((error) => { - showError(error); - }) - .finally(() => { - setLoading(false); }); - }); + + if (Object.keys(warnings).length > 0 && checkForWarnings) { + const { groupedErrors, groupedWarnings, totalIssueCount } = + groupAndCountRows(Object.values(warnings)); + setGroupedErrors(groupedErrors); + setGroupedWarnings(groupedWarnings); + setTotalIssueCount(totalIssueCount); + + setAlertDialogText({ + title: + "Your file has been processed and contains the following errors and warnings!", + content: ( + <> + {totalIssueCount.errors >= 1 && ( +
+ + {totalIssueCount.errors} Errors + + - Must fix before uploading +
+ )} + {totalIssueCount.warnings >= 1 && ( +
+ + {totalIssueCount.warnings} Warnings + + - Can upload without fixing +
+ )} + + ), + cancelAction: () => setOpenDialog(false), + confirmText: "View Details", + confirmAction: () => setOpenDialog(false), + }); + setOpenDialog(true); + + } + }) + .catch((error) => { + showError(error); + }) + .finally(() => { + setLoading(false); + }); }; const downloadSpreadsheet = () => { @@ -339,57 +304,57 @@ const UploadContainer = () => { ) : null; - return ( -
-
- <> - - - {(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( - - +
+ <> + + + {(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( + + + + )} + + - )} - - - - {adminUser && ( - - - - )} - - + {adminUser && ( + + + + )} + + +
-
- ); -}; + ); + }; export default withRouter(UploadContainer); diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js index edcdc007..c379bb9c 100644 --- a/frontend/src/uploads/components/UploadIssues.js +++ b/frontend/src/uploads/components/UploadIssues.js @@ -74,7 +74,7 @@ const UploadIssues = ({ {totalIssueCount.errors >= 1 && ( = 1 && ( { From 5ab244a4223187fe69252f3ba443299fdaa1a7af Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 31 Jul 2024 11:39:55 -0700 Subject: [PATCH 34/67] Frontend fixes for the error notification refactor (#375) --- frontend/src/app/components/AlertDialog.js | 4 ++-- frontend/src/uploads/components/UploadIssues.js | 4 ++-- frontend/src/uploads/components/UploadIssuesDetail.js | 11 +++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/components/AlertDialog.js b/frontend/src/app/components/AlertDialog.js index c526b27a..8a2b4cb2 100644 --- a/frontend/src/app/components/AlertDialog.js +++ b/frontend/src/app/components/AlertDialog.js @@ -6,7 +6,7 @@ import DialogActions from "@mui/material/DialogActions"; import DialogContent from "@mui/material/DialogContent"; import DialogContentText from "@mui/material/DialogContentText"; import DialogTitle from "@mui/material/DialogTitle"; -import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; const AlertDialog = (props) => { const { @@ -33,7 +33,7 @@ const AlertDialog = (props) => { aria-describedby="alert-dialog-description" > - {title} + {title} diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js index c379bb9c..3ea05989 100644 --- a/frontend/src/uploads/components/UploadIssues.js +++ b/frontend/src/uploads/components/UploadIssues.js @@ -7,7 +7,7 @@ import { Accordion, Button, } from "@mui/material"; -import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; import UploadIssuesDetail from "./UploadIssuesDetail"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; @@ -30,7 +30,7 @@ const UploadIssues = ({ <>

- diff --git a/frontend/src/uploads/components/UploadIssuesDetail.js b/frontend/src/uploads/components/UploadIssuesDetail.js index f2d93ba3..6c504e0d 100644 --- a/frontend/src/uploads/components/UploadIssuesDetail.js +++ b/frontend/src/uploads/components/UploadIssuesDetail.js @@ -1,7 +1,7 @@ import PropTypes from "prop-types"; import React, { useState } from "react"; import { Box, Button } from "@mui/material"; -import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { @@ -23,13 +23,13 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { mb: "1rem", }} > - - {totalIssueCount} {type}  + {totalIssueCount} {type == 'error' ? 'Errors' : 'Warnings'}  ({msg}) @@ -38,7 +38,10 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { Column: {column} {Object.keys(issues[column]).map((errorType, index) => (
-
{type.charAt(0).toUpperCase() + type.slice(1)} Name: {errorType}
+
+ {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')} + {type.charAt(0).toUpperCase() + type.slice(1)} Name: {errorType} +
Expected value:{" "} {issues[column][errorType].ExpectedType || From 8913558f55e1dc572ed3cc246d9e94f0b3d003c6 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:50:39 -0700 Subject: [PATCH 35/67] Added import service for ARC data, Created regions table for future mapping functionality, created region checker function to validate region strings, updated backend to account for data structure changes (#376) --- django/api/constants/constants.py | 93 +++++++++------ django/api/constants/misc.py | 15 ++- .../api/migrations/0032_auto_20240726_2118.py | 112 ++++++++++++++++++ django/api/migrations/0033_regions.py | 27 +++++ django/api/models/__init__.py | 1 + django/api/models/arc_project_tracking.py | 56 +++++---- django/api/models/regions.py | 10 ++ django/api/services/spreadsheet_uploader.py | 42 ++++--- .../api/services/spreadsheet_uploader_prep.py | 25 ++++ django/api/viewsets/upload.py | 1 - 10 files changed, 308 insertions(+), 74 deletions(-) create mode 100644 django/api/migrations/0032_auto_20240726_2118.py create mode 100644 django/api/migrations/0033_regions.py create mode 100644 django/api/models/regions.py diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index b1570408..51357409 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -23,47 +23,60 @@ typo_checker, location_checker, email_validator, - validate_field_values + validate_field_values, + region_checker ) from api.services.resolvers import get_google_resolver -from api.constants.misc import GER_VALID_FIELD_VALUES +from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES +from enum import Enum + class ARCProjectTrackingColumns(Enum): - FUNDING_CALL = "Funding Call" + REFERENCE_NUMBER = "Ref #" PROPONENT = "Proponent" - REF_NUMBER = "Ref #" + STATUS = "Status" + FUNDING_CALL = "Funding Call" PROJECT_TITLE = "Project Title" + VEHICLE_CATEGORY = "Vehicle Category" + ZEV_SUB_SECTOR = "ZEV Sub-Section" + FUEL_TYPE = "Fuel Type" + RETROFIT = "Retrofit" PRIMARY_LOCATION = "Primary Location" - STATUS = "Status" - ARC_FUNDING = "ARC Funding" - FUNDS_ISSUED = "Funds Issued" + ECONOMIC_REGION = "Economic Region" + JOBS = "Jobs (FTEs)" + FUNDS_COMMITED = "Funds Committed" + FUNDS_DISBURSED = "Funds Disbursed" + REMAINING_DISBURSED = "Remaining To Disburse" + TOTAL_PROJECT_VALUE = "Total Project Value" START_DATE = "Start Date" COMPLETION_DATE = "Completion Date" - TOTAL_PROJECT_VALUE = "Total Project Value" - ZEV_SUB_SECTOR = "ZEV Sub-Sector" - ON_ROAD_OFF_ROAD = "On-Road/Off-Road" - FUEL_TYPE = "Fuel Type" + COMPLETE_OR_TERMINATION_DATE = "Complete or Termination date" PUBLICLY_ANNOUNCED = "Publicly Announced" - + NOTES = "Notes" class ArcProjectTrackingColumnMapping(Enum): - funding_call = "Funding Call" - proponent = "Proponent" reference_number = "Ref #" + proponent = "Proponent" + status = "Status" + funding_call = "Funding Call" project_title = "Project Title" + vehicle_category = "Vehicle Category" + zev_sub_sector = "ZEV Sub-Section" + fuel_type = "Fuel Type" + retrofit = "Retrofit" primary_location = "Primary Location" - status = "Status" - arc_funding = "ARC Funding" - funds_issued = "Funds Issued" + economic_region = "Economic Region" + jobs = "Jobs (FTEs)" + funds_commited = "Funds Committed" + funds_disbursed = "Funds Disbursed" + remaining_disbursed = "Remaining To Disburse" + total_project_value = "Total Project Value" start_date = "Start Date" completion_date = "Completion Date" - total_project_value = "Total Project Value" - zev_sub_sector = "ZEV Sub-Sector" - on_road_off_road = "On-Road/Off-Road" - fuel_type = "Fuel Type" + complete_or_termination_date = "Complete or Termination date" publicly_announced = "Publicly Announced" - + notes = "Notes" class EVChargingRebatesColumns(Enum): ORGANIZATION = "Organization" @@ -414,21 +427,27 @@ class GoElectricRebatesColumnMapping(Enum): FIELD_TYPES = { "ARC Project Tracking": { - "funding_call": str, - "proponent": str, "reference_number": str, - "project_title": str, - "primary_location": str, + "proponent": str, "status": str, - "arc_funding": int, - "funds_issued": int, - "start_date": str, - "completion_date": str, - "total_project_value": int, + "funding_call": str, + "project_title": str, + "vehicle_category": str, "zev_sub_sector": str, - "on_road_off_road": str, "fuel_type": str, - "publicly_announced": bool, + "retrofit": str, + "primary_location": str, + "economic_region": str, + "jobs": int, + "funds_commited": int, + "funds_disbursed": int, + "remaining_disbursed": int, + "total_project_value": int, + "start_date": datetime.date, + "completion_date": datetime.date, + "complete_or_termination_date": datetime.date, + "publicly_announced": str, + "notes": str, }, "EV Charging Rebates": { "organization": str, @@ -596,8 +615,12 @@ class GoElectricRebatesColumnMapping(Enum): "model": ARCProjectTracking, "columns": ARCProjectTrackingColumns, "column_mapping": ArcProjectTrackingColumnMapping, - "sheet_name": "Project_Tracking", + "sheet_name": "ARC Data", "preparation_functions": [prepare_arc_project_tracking], + "validation_functions": [ + {'function': validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": ARC_VALID_FIELD_VALUES}}, + {"function": region_checker, "columns": ['Economic Region'], "kwargs": {"indices_offset":2}}, + ] }, "EV Charging Rebates": { "model": ChargerRebates, @@ -659,7 +682,7 @@ class GoElectricRebatesColumnMapping(Enum): {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, {"function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, - {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}} + {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, ] }, } diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index c89b2d0f..95de56bb 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -86,4 +86,17 @@ 'Fleet/Individuals': ['Fleet', 'Individual'], 'Rebate adjustment (discount)': ['Yes'], 'Class': ['2B', '3', '4', '5', '6', '7', '8'] - } \ No newline at end of file + } + +ARC_VALID_FIELD_VALUES = { + 'Funding Call': ['ARC-2018-1', 'ARC-2020-2'], + 'Status': ['Approved', 'Completed', 'Terminated'], + 'Vehicle Category': ['On-Road', 'On/Off Road', 'Marine', 'Aviation', 'Off-Road'], + 'Zev Sub-Section': [ + 'Testing and certification services', 'Battery components', + 'Vehicle components', 'Fuelling Infrastructure', 'Vehicles', + 'Transferable Technologies' + ], + 'Fuel Type': ['H2', 'Electric'], + 'Retrofit': ['BEV Retrofit', 'Hybrid Retrofit', 'H2 Retrofit', 'N/A'] +} \ No newline at end of file diff --git a/django/api/migrations/0032_auto_20240726_2118.py b/django/api/migrations/0032_auto_20240726_2118.py new file mode 100644 index 00000000..1331feee --- /dev/null +++ b/django/api/migrations/0032_auto_20240726_2118.py @@ -0,0 +1,112 @@ +# Generated by Django 3.2.25 on 2024-07-26 21:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0031_auto_20240712_2036'), + ] + + operations = [ + migrations.RenameField( + model_name='arcprojecttracking', + old_name='arc_funding', + new_name='funds_disbursed', + ), + migrations.RenameField( + model_name='arcprojecttracking', + old_name='funds_issued', + new_name='jobs', + ), + migrations.RenameField( + model_name='arcprojecttracking', + old_name='on_road_off_road', + new_name='notes', + ), + migrations.AddField( + model_name='arcprojecttracking', + name='complete_or_termination_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AddField( + model_name='arcprojecttracking', + name='economic_region', + field=models.CharField(default='Across BC', max_length=250), + preserve_default=False, + ), + migrations.AddField( + model_name='arcprojecttracking', + name='funds_commited', + field=models.IntegerField(default=0), + preserve_default=False, + ), + migrations.AddField( + model_name='arcprojecttracking', + name='remaining_disbursed', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='arcprojecttracking', + name='retrofit', + field=models.CharField(blank=True, max_length=250, null=True), + ), + migrations.AddField( + model_name='arcprojecttracking', + name='vehicle_category', + field=models.CharField(default='A', max_length=250), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='completion_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='funding_call', + field=models.CharField(default='a', max_length=50), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='primary_location', + field=models.CharField(default='BC', max_length=250), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='project_title', + field=models.CharField(default='Title', max_length=500), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='proponent', + field=models.CharField(default='test', max_length=500), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='publicly_announced', + field=models.CharField(blank=True, max_length=250, null=True), + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='reference_number', + field=models.CharField(default=0, max_length=50), + preserve_default=False, + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='start_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name='arcprojecttracking', + name='status', + field=models.CharField(default='Valid', max_length=250), + preserve_default=False, + ), + ] diff --git a/django/api/migrations/0033_regions.py b/django/api/migrations/0033_regions.py new file mode 100644 index 00000000..65e60d74 --- /dev/null +++ b/django/api/migrations/0033_regions.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-07-31 17:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0032_auto_20240726_2118'), + ] + + operations = [ + migrations.CreateModel( + name='Regions', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)), + ('create_user', models.CharField(default='SYSTEM', max_length=130)), + ('update_timestamp', models.DateTimeField(auto_now=True, null=True)), + ('update_user', models.CharField(max_length=130, null=True)), + ('name', models.CharField(max_length=250)), + ], + options={ + 'db_table': 'regions', + }, + ), + ] diff --git a/django/api/models/__init__.py b/django/api/models/__init__.py index 79c55108..1e02db9e 100644 --- a/django/api/models/__init__.py +++ b/django/api/models/__init__.py @@ -25,3 +25,4 @@ from . import uploaded_vins_file from . import uploaded_vin_record from . import decoded_vin_record +from . import regions diff --git a/django/api/models/arc_project_tracking.py b/django/api/models/arc_project_tracking.py index 8c66bc92..69579122 100644 --- a/django/api/models/arc_project_tracking.py +++ b/django/api/models/arc_project_tracking.py @@ -4,47 +4,63 @@ class ARCProjectTracking(Auditable): - funding_call = models.CharField(blank=True, null=True, max_length=50, unique=False) - - proponent = models.CharField(blank=True, null=True, max_length=500, unique=False) - reference_number = models.CharField( - blank=True, null=True, max_length=50, unique=False + blank=False, null=False, max_length=50, unique=False ) + proponent = models.CharField(blank=False, null=False, max_length=500, unique=False) + + status = models.CharField(blank=False, null=False, max_length=250, unique=False) + + funding_call = models.CharField(blank=False, null=False, max_length=50, unique=False) + project_title = models.CharField( - blank=True, null=True, max_length=500, unique=False + blank=False, null=False, max_length=500, unique=False ) - primary_location = models.CharField( - blank=True, null=True, max_length=250, unique=False + vehicle_category = models.CharField( + blank=False, null=False, max_length=250 ) - status = models.CharField(blank=True, null=True, max_length=250, unique=False) + zev_sub_sector = models.CharField( + blank=True, null=True, max_length=250, unique=False + ) - arc_funding = models.IntegerField(blank=True, null=True) + fuel_type = models.CharField(blank=True, null=True, max_length=250, unique=False) - funds_issued = models.IntegerField(blank=True, null=True) + retrofit = models.CharField(blank=True, null=True, max_length=250) - start_date = models.CharField(blank=True, null=True, max_length=250, unique=False) + primary_location = models.CharField( + blank=False, null=False, max_length=250, unique=False + ) - completion_date = models.CharField( - blank=True, null=True, max_length=250, unique=False + economic_region = models.CharField( + blank=False, null=False, max_length=250 ) + jobs = models.IntegerField(blank=True, null=True) + + funds_commited = models.IntegerField(blank=False, null=False) + + funds_disbursed = models.IntegerField(blank=True, null=True) + + remaining_disbursed = models.IntegerField(blank=True, null=True) + total_project_value = models.IntegerField(blank=True, null=True) - zev_sub_sector = models.CharField( - blank=True, null=True, max_length=250, unique=False + start_date = models.DateField(blank=True, null=True, unique=False) + + completion_date = models.DateField( + blank=True, null=True, unique=False ) - on_road_off_road = models.CharField( - blank=True, null=True, max_length=250, unique=False + complete_or_termination_date = models.DateField( + blank=True, null=True, unique=False ) - fuel_type = models.CharField(blank=True, null=True, max_length=250, unique=False) + publicly_announced = models.CharField(blank=True, null=True, max_length=250) - publicly_announced = models.BooleanField(default=False) + notes = models.CharField(blank=True, null=True, max_length=250) class Meta: db_table = "arc_project_tracking" diff --git a/django/api/models/regions.py b/django/api/models/regions.py new file mode 100644 index 00000000..095c238e --- /dev/null +++ b/django/api/models/regions.py @@ -0,0 +1,10 @@ +from auditable.models import Auditable +from django.db import models + + +class Regions(Auditable): + + name = models.CharField(blank=False, null=False, max_length=250) + + class Meta: + db_table = "regions" diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 1c9cb552..3c822361 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -1,6 +1,7 @@ from decimal import Decimal, ROUND_HALF_UP import pandas as pd import traceback +import numpy as np from django.db import transaction def get_field_default(model, field): @@ -56,30 +57,37 @@ def transform_data( df = prep_func(df) nullable_fields = get_nullable_fields(model) + + column_mapping = {e.value: e.name for e in column_mapping_enum} + errors_and_warnings = {} + df = df.replace({np.nan: None}) for index, row in df.iterrows(): row_dict = row.to_dict() for column, value in row_dict.items(): - expected_type = field_types.get(column) - is_nullable = column in nullable_fields + db_field_name = column_mapping.get(column) - if pd.isna(value) or value == "" or value is None: - if is_nullable: - row_dict[column] = None - else: - if column not in errors_and_warnings: - errors_and_warnings[column] = {} - if "Empty Value" not in errors_and_warnings[column]: - errors_and_warnings[column]["Empty Value"] = { - "Expected Type": "Expected value where there isn't one.", - "Rows": [], - "Severity": "Error" - } - errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) + if db_field_name: + is_nullable = db_field_name in nullable_fields + expected_type = field_types.get(column) - if expected_type in [int, float, Decimal] and value is not None and pd.notna(value): + if pd.isna(value) or value == "" or value is None: + if is_nullable: + row_dict[column] = None + else: + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Empty Value" not in errors_and_warnings[column]: + errors_and_warnings[column]["Empty Value"] = { + "Expected Type": "Expected value where there isn't one.", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) + + if expected_type in [int, float, Decimal] and value is not None and pd.notna(value) and value != '': value = str(value).replace(',', '').strip() try: if expected_type == int: @@ -198,7 +206,7 @@ def import_from_xls( else: print('no warnings') - result = load_data(df, model, replace_data, user, errors_and_warnings) + result = load_data(df, model, replace_data, user) total_rows = result["row_count"] inserted_rows = result["records_inserted"] diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 3753c823..8b082c66 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -2,6 +2,7 @@ import pandas as pd import difflib as dl from api.services.bcngws import get_placename_matches +from api.models.regions import Regions from email_validator import validate_email, EmailNotValidError from api.utilities.series import get_map_of_values_to_indices from api.constants.misc import AREA_CODES @@ -321,4 +322,28 @@ def validate_field_values(df, *columns, **kwargs): } } + return result + +def region_checker(df, *columns, **kwargs): + valid_regions = set(Regions.objects.values_list('name', flat=True)) + + result = {} + indices = [] + for column in columns: + for index, value in df[column].items(): + values_list = [item.strip() for item in value.split(',')] + if all(value in valid_regions for value in values_list): + continue + else: + indices.append(index + kwargs.get('indices_offset', 0)) + + if indices: + result[column] = { + "Invalid Region": { + "Expected Type": "The following rows have an invalid region", + "Rows": indices, + "Severity": "Error" + } + } + return result \ No newline at end of file diff --git a/django/api/viewsets/upload.py b/django/api/viewsets/upload.py index 5981809e..5f6d16ff 100644 --- a/django/api/viewsets/upload.py +++ b/django/api/viewsets/upload.py @@ -33,7 +33,6 @@ def datasets_list(self, request): "EV Charging Rebates", "Hydrogen Fueling", "Hydrogen Fleets", - "ARC Project Tracking", "Data Fleets", "Scrap It", ] From dd93fb97b0384c9076a6c5ede48bcf3c068f9c26 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 7 Aug 2024 14:06:33 -0700 Subject: [PATCH 36/67] Fix: Geographic Name Maximum Query Length #361 (#372) * Adding batch system to API call to avoid max query length error * Accidentally removed a couple relevant comments and code * Adding unique_values_list to loop in case of duplicates --- django/api/services/bcngws.py | 5 ++--- django/api/services/spreadsheet_uploader_prep.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/django/api/services/bcngws.py b/django/api/services/bcngws.py index 66bf4924..c42b7278 100644 --- a/django/api/services/bcngws.py +++ b/django/api/services/bcngws.py @@ -2,7 +2,6 @@ from django.conf import settings from api.constants.misc import RELEVANT_FEATURES - # names should be a list of location names, page_size should be an integer >=1, <=200 # start_index should be an integer, result should be a set def get_placename_matches(names, page_size, start_index, result): @@ -11,13 +10,13 @@ def get_placename_matches(names, page_size, start_index, result): query = { "outputFormat": "json", "name": names_string, - "itemsPerPage": 200, + "itemsPerPage": page_size, "startIndex": start_index, } try: response = requests.get(settings.PLACENAMES_ENDPOINT, params=query) - response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx) + response.raise_for_status() response = response.json() for feature in response["features"]: diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 8b082c66..91c1c03c 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -250,7 +250,7 @@ def validate_phone_numbers(df, *columns, **kwargs): return result -def location_checker(df, *columns, **kwargs): +def location_checker(df, *columns, batch_size=50, **kwargs): result = {} for column in columns: indices = [] @@ -258,15 +258,21 @@ def location_checker(df, *columns, **kwargs): map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) values = series.to_list() unique_values = set(series) + unique_values_list = list(values) communities = set() - # populate communities by calling the bcngws API with the values: - get_placename_matches(values, 200, 1, communities) + for i in range(0, len(unique_values_list), batch_size): + batch_values = unique_values_list[i:i + batch_size] + # Send request to API with list of names, returns all the communities that somewhat matched + get_placename_matches(batch_values, 200, 1, communities) + + # Find names that don't have a match in the locations_set names_without_match = unique_values.difference(communities) for name in names_without_match: indices_to_add = map_of_values_to_indices[name] indices.extend(indices_to_add) if indices: + indices.sort() result[column] = { "Unrecognized City Names": { "Expected Type": "The following city names are not in the list of geographic names. Please double check that these places exist or have correct spelling and adjust your dataset accordingly.", From 7e0607de07251ff69cafb8f87a1924b85df790bd Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 14 Aug 2024 14:47:09 -0700 Subject: [PATCH 37/67] Added check for string length being long enough to apply lambda function to prevent errors (#380) --- django/api/services/spreadsheet_uploader_prep.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 91c1c03c..0b93eabb 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -173,8 +173,10 @@ def make_prepositions_consistent(df): regex=True ) ##The first letter should be capitalized - df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer'] - ].applymap(lambda x: x[0].upper() + x[1:]) + df[['Applicant Name', 'Manufacturer']] = df[['Applicant Name', 'Manufacturer']].applymap( + lambda x: x[0].upper() + x[1:] if isinstance(x, str) and len(x) > 1 else x.upper() if isinstance(x, str) and len(x) == 1 else x +) + def adjust_ger_manufacturer_names(df): """"" From d607b7d1fca8e224d09ba231d73fa22725d89e66 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:30:26 -0700 Subject: [PATCH 38/67] Temporarily disabling the location_checker function to test for timeout errors (#382) --- django/api/constants/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 51357409..f5213aae 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -680,7 +680,7 @@ class GoElectricRebatesColumnMapping(Enum): "validation_functions": [ {"function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, - {"function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, + # {"function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, ] From c0b18a71ddb49744650f4ed4e4bd836360beb08f Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:18:01 -0700 Subject: [PATCH 39/67] fix: update location checker (#383) --- django/api/constants/constants.py | 4 +-- django/api/constants/misc.py | 30 ++++++------------- django/api/services/bcngws.py | 23 ++++++++++---- .../api/services/spreadsheet_uploader_prep.py | 12 ++++---- 4 files changed, 33 insertions(+), 36 deletions(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index f5213aae..74da2492 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -27,7 +27,7 @@ region_checker ) from api.services.resolvers import get_google_resolver -from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES +from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES, LOCALITY_FEATURES_MAP from enum import Enum @@ -680,7 +680,7 @@ class GoElectricRebatesColumnMapping(Enum): "validation_functions": [ {"function": validate_phone_numbers, "columns": ["Phone Number"], "kwargs": {"indices_offset": 2}}, {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, - # {"function": location_checker, "columns": ["City"], "kwargs": {"indices_offset":2}}, + {"function": location_checker, "columns": ["City"], "kwargs": {"columns_to_features_map": {"City": LOCALITY_FEATURES_MAP}, "indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, ] diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index 95de56bb..18efbfc8 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -55,27 +55,15 @@ 867, ] -RELEVANT_FEATURES = [ - "Canadian Forces Base", - "Canadian Forces Station", - "City", - "Community", - "District Municipality (1)", - "First Nation Village", - "Former Locality", - "Indian Government District", - "Indian Government District : Land Unit", - "Indian Reserve-Réserve indienne", - "Locality", - "Recreation Facility", - "Recreational Community", - "Region", - "Regional District", - "Resort Municipality", - "Urban Community", - "Village (1)", - "Town", -] +# map of feature category codes to feature types for locality features: +LOCALITY_FEATURES_MAP = { + 1: ["City", "District Municipality (1)", "Resort Municipality", "Village (1)", "Town"], + 2: ["Community", "First Nation Village", "Former Locality", "Locality", "Recreational Community"], + 3: ["Urban Community"], + 5: ["Indian Government District", "Indian Government District : Land Unit"], + 6: ["Indian Reserve-Réserve indienne", "Region", "Regional District"], + 28: ["Canadian Forces Base", "Canadian Forces Station", "Recreation Facility"], +} GER_VALID_FIELD_VALUES = { 'Approvals': ['Approved', 'Approved Fraudulent'], diff --git a/django/api/services/bcngws.py b/django/api/services/bcngws.py index c42b7278..55ce2aec 100644 --- a/django/api/services/bcngws.py +++ b/django/api/services/bcngws.py @@ -1,17 +1,21 @@ import requests from django.conf import settings -from api.constants.misc import RELEVANT_FEATURES -# names should be a list of location names, page_size should be an integer >=1, <=200 + +# names should be a list of location names, feature_category should be an integer or *, +# feature_types should be a list or *, page_size should be an integer >=1, <=200, # start_index should be an integer, result should be a set -def get_placename_matches(names, page_size, start_index, result): - names_string = ", ".join(names) +def get_placename_matches( + names, feature_category, feature_types, page_size, start_index, result +): + names_string = " ".join(names) query = { "outputFormat": "json", "name": names_string, "itemsPerPage": page_size, "startIndex": start_index, + "featureCategory": feature_category, } try: @@ -22,11 +26,18 @@ def get_placename_matches(names, page_size, start_index, result): for feature in response["features"]: name = feature["properties"]["name"] type = feature["properties"]["featureType"] - if type in RELEVANT_FEATURES: + if feature_types == "*" or type in feature_types: result.add(name) if response["properties"]["totalResults"] >= start_index + page_size: - get_placename_matches(names, page_size, start_index + page_size, result) + get_placename_matches( + names, + feature_category, + feature_types, + page_size, + start_index + page_size, + result, + ) except requests.RequestException as e: print(f"Error fetching data: {e}") diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 0b93eabb..9bd685f4 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -252,21 +252,19 @@ def validate_phone_numbers(df, *columns, **kwargs): return result -def location_checker(df, *columns, batch_size=50, **kwargs): +def location_checker(df, *columns, columns_to_features_map={}, **kwargs): result = {} for column in columns: indices = [] series = df[column] map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) - values = series.to_list() unique_values = set(series) - unique_values_list = list(values) + unique_values_list = list(unique_values) communities = set() - for i in range(0, len(unique_values_list), batch_size): - batch_values = unique_values_list[i:i + batch_size] - # Send request to API with list of names, returns all the communities that somewhat matched - get_placename_matches(batch_values, 200, 1, communities) + features_map = columns_to_features_map[column] + for category_code, feature_types in features_map.items(): + get_placename_matches(unique_values_list, category_code, feature_types, 200, 1, communities) # Find names that don't have a match in the locations_set names_without_match = unique_values.difference(communities) From 78a8e2006d90dc954722d4d50bc64c096d3c4136 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Tue, 17 Sep 2024 10:40:05 -0700 Subject: [PATCH 40/67] Adding new model and uploader logic for CVP Data (#389) --- django/api/constants/constants.py | 154 +++++++++++++- django/api/constants/misc.py | 27 ++- django/api/migrations/0034_cvpdata.py | 71 +++++++ .../api/migrations/0035_auto_20240910_2143.py | 165 +++++++++++++++ .../api/migrations/0036_auto_20240911_1758.py | 73 +++++++ .../api/migrations/0037_auto_20240911_1800.py | 23 +++ django/api/models/__init__.py | 1 + django/api/models/cvp_data.py | 188 ++++++++++++++++++ .../api/services/spreadsheet_uploader_prep.py | 8 +- 9 files changed, 707 insertions(+), 3 deletions(-) create mode 100644 django/api/migrations/0034_cvpdata.py create mode 100644 django/api/migrations/0035_auto_20240910_2143.py create mode 100644 django/api/migrations/0036_auto_20240911_1758.py create mode 100644 django/api/migrations/0037_auto_20240911_1800.py create mode 100644 django/api/models/cvp_data.py diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 74da2492..28a0b442 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -11,6 +11,7 @@ from api.models.public_charging import PublicCharging from api.models.scrap_it import ScrapIt from api.models.go_electric_rebates import GoElectricRebates +from api.models.cvp_data import CVPData from api.services.spreadsheet_uploader_prep import ( prepare_arc_project_tracking, prepare_hydrogen_fleets, @@ -19,6 +20,7 @@ prepare_public_charging, prepare_scrap_it, prepare_go_electric_rebates, + prepare_cvp_data, validate_phone_numbers, typo_checker, location_checker, @@ -27,7 +29,7 @@ region_checker ) from api.services.resolvers import get_google_resolver -from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES, LOCALITY_FEATURES_MAP +from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES, LOCALITY_FEATURES_MAP, CVP_DATA_VALID_FIELD_VALUES from enum import Enum @@ -423,6 +425,99 @@ class GoElectricRebatesColumnMapping(Enum): rebate_adjustment = "Rebate adjustment (discount)" notes = "Notes" +class CVPDataColumns(Enum): + FUNDING_CALL = "FC" + PROJECT_IDENTIFIER = "Project Identifier" + APPLICANT_NAME = "Name of Applicant" + RANK = "Rank" + STATUS = "Status" + SCORE = "Score" + VEHICLE_DEPLOYED = "Vehicle Deployed" + VEHICLE_CATEGORY = "Vehicle Category" + DRIVE_TYPE = "Drive Type" + VEHICLE_TYPE = "Vehicle Type" + PORTFOLIO = "Portfolio" + MAKE_AND_MODEL = "Vehicle Make and Model" + ECONOMIC_REGION = "Economic Region" + START_DATE = "Start Date" + COMPLETION_DATE = "Completion Date" + PROJECT_TYPE = "Project Type" + CLASS_3 = "Class 3" + CLASS_4 = "Class 4" + CLASS_5 = "Class 5" + CLASS_6 = "Class 6" + CLASS_7 = "Class 7" + CLASS_8 = "Class 8" + ON_ROAD_TOTAL = "On Road Total" + OFF_ROAD = "Off-Road" + LEVEL_2_CHARGER = "Level 2 Charger (3.3 kW to 19.2 kW)" + LEVEL_3_CHARGER = "Level 3 Charger (20 kW to 49 kW)" + HIGH_LEVEL_3_CHARGER = "Level 3 Charger (50 kW to 99kW)" + LEVEL_CHARGER = "Level Charger (100 kW and above)" + OTHER_CHARGER = "Other Charger" + H2_FUELING_STATION = "H2 Fueling Station" + CHARGER_BRAND = "Charger Brand" + H2_FUELLING_STATION_DESCRIPTION = "H2 Fuelling Station Description" + GHG_EMISSION_REDUCTION = "Proponent's GHG Emission Reduction (tCO2e/yr)" + ESTIMATED_GHG_EMISSION_REDUCTION = "Le-ef Estimated GHG Reduction (tCO2e/yr)" + FUNDING_EFFICIENCY = "Funding Efficiency for Emmision Abatment ($/tCO2e)" + MARKET_EMISSION_REDUCTIONS = "Market Emission Reductions (tCO2e by 2030)" + CVP_FUNDING_REQUEST = "CVP Program Funding Request (Initial)" + CVP_FUNDING_CONTRIBUTION = "CVP Funding (approved - Contribution Agreement)" + EXTERNAL_FUNDING = "External Funding" + PROPONENT_FUNDING = "Proponent funding" + PROJECT_COST_INITIAL = "Total project cost (initial)" + PROJECT_COST_REVISED = "Total Project Cost (revised)" + FUNDING_SOURCE = "Funding Source" + NOTES = "Notes" + IMHZEV = "iMHZEV" + +class CVPDataColumnMapping(Enum): + funding_call = "FC" + project_identifier = "Project Identifier" + applicant_name = "Name of Applicant" + rank = "Rank" + status = "Status" + score = "Score" + vehicle_deployed = "Vehicle Deployed" + vehicle_category = "Vehicle Category" + drive_type = "Drive Type" + vehicle_type = "Vehicle Type" + portfolio = "Portfolio" + make_and_model = "Vehicle Make and Model" + economic_region = "Economic Region" + start_date = "Start Date" + completion_date = "Completion Date" + project_type = "Project Type" + class_3 = "Class 3" + class_4 = "Class 4" + class_5 = "Class 5" + class_6 = "Class 6" + class_7 = "Class 7" + class_8 = "Class 8" + on_road_total = "On Road Total" + off_road = "Off-Road" + level_2_charger = "Level 2 Charger (3.3 kW to 19.2 kW)" + level_3_charger = "Level 3 Charger (20 kW to 49 kW)" + high_level_3_charger = "Level 3 Charger (50 kW to 99kW)" + level_charger = "Level Charger (100 kW and above)" + other_charger = "Other Charger" + h2_fuelling_station = "H2 Fueling Station" + charger_brand = "Charger Brand" + h2_fuelling_station_description = "H2 Fuelling Station Description" + ghg_emission_reduction = "Proponent's GHG Emission Reduction (tCO2e/yr)" + estimated_ghg_emission_reduction = "Le-ef Estimated GHG Reduction (tCO2e/yr)" + funding_efficiency = "Funding Efficiency for Emmision Abatment ($/tCO2e)" + market_emission_reductions = "Market Emission Reductions (tCO2e by 2030)" + cvp_funding_request = "CVP Program Funding Request (Initial)" + cvp_funding_contribution = "CVP Funding (approved - Contribution Agreement)" + external_funding = "External Funding" + proponent_funding = "Proponent funding" + project_cost_initial = "Total project cost (initial)" + project_cost_revised = "Total Project Cost (revised)" + funding_source = "Funding Source" + notes = "Notes" + imhzev = "iMHZEV" FIELD_TYPES = { @@ -607,6 +702,54 @@ class GoElectricRebatesColumnMapping(Enum): "rebate_adjustment": str, "notes": str, }, + "CVP Data": { + "funding_call": int, + "project_identifier": int, + "applicant_name": str, + "rank": int, + "status": str, + "score": int, + "vehicle_deployed": str, + "vehicle_category": str, + "drive_type": str, + "vehicle_type": str, + "portfolio": str, + "make_and_model": str, + "economic_region": str, + "start_date": datetime.date, + "completion_date": datetime.date, + "project_type": str, + "class_3": int, + "class_4": int, + "class_5": int, + "class_6": int, + "class_7": int, + "class_8": int, + "on_road_total": int, + "off_road": int, + "level_2_charger": int, + "level_3_charger": int, + "high_level_3_charger": int, + "level_charger": int, + "other_charger": int, + "h2_fuelling_station": int, + "charger_brand": str, + "h2_fuelling_station_description": str, + "ghg_emission_reduction": int, + "estimated_ghg_emission_reduction": int, + "funding_efficiency": int, + "market_emission_reductions": int, + "cvp_funding_request": int, + "cvp_funding_contribution": int, + "external_funding": int, + "proponent_funding": int, + "project_cost_initial": int, + "project_cost_revised": int, + "funding_source": str, + "notes": str, + "imhzev": str, + }, + } @@ -685,4 +828,13 @@ class GoElectricRebatesColumnMapping(Enum): {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, ] }, + "CVP Data": { + "model": CVPData, + "columns": CVPDataColumns, + "column_mapping": CVPDataColumnMapping, + "sheet_name": "Data", + "preparation_functions": [prepare_cvp_data], + "validation_functions": [{"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": CVP_DATA_VALID_FIELD_VALUES}},] + }, + } diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index 18efbfc8..10dc15f5 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -87,4 +87,29 @@ ], 'Fuel Type': ['H2', 'Electric'], 'Retrofit': ['BEV Retrofit', 'Hybrid Retrofit', 'H2 Retrofit', 'N/A'] -} \ No newline at end of file +} + +CVP_DATA_VALID_FIELD_VALUES = { + 'Funding Call': ['1', '2', '3', '4', '5', '6', '7', '8', '10'], + 'Status': ['Approved', 'Completed', 'Terminated', 'Not Approved', 'Application Withdrawn'], + 'Vehicles Deployed': ['Yes', 'No'], + 'Vehicle Category': ['On-Road', 'Off-Road', 'Marine', 'Rail', 'Aviation'], + 'Portfolio': [ + 'Road - 8', 'Rail', 'Marine', 'Road - 8C', 'School Bus', + 'Loader', 'Road - 6', 'Fork Lift', 'Recreation' + ], + 'Economic Region': [ + 'Nechako', 'Northeast', 'North Coast', 'Cariboo', 'Vancouver Island/Coast', + 'Mainland/Southwest', 'Thompson/Okanagan', 'Kootenay', 'Across BC' + ], + 'Drive Type': ['BEV', 'FC', 'PHEV'], + 'Vehicle Type': [ + 'On-Road', 'Loader', 'Excavator', 'Forklift', 'Outboard Motor', + 'Tugboat', 'Passenger Ferry', 'Ice Resurfacer', 'Locomotive', + 'Rail Maintenance', 'Rubber-tired Gantry Crane', 'Terminal/Yard truck', + 'Aircraft', 'Jet Fuel Pumper', 'Train Mover' + ], + 'Project Type': [ + 'Procurement', 'New Design', 'Hybrid Retrofit', 'BEV Retrofit', 'H2 Retrofit' + ] +} diff --git a/django/api/migrations/0034_cvpdata.py b/django/api/migrations/0034_cvpdata.py new file mode 100644 index 00000000..d8218f3c --- /dev/null +++ b/django/api/migrations/0034_cvpdata.py @@ -0,0 +1,71 @@ +# Generated by Django 3.2.25 on 2024-09-10 21:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0033_regions'), + ] + + operations = [ + migrations.CreateModel( + name='CVPData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)), + ('create_user', models.CharField(default='SYSTEM', max_length=130)), + ('update_timestamp', models.DateTimeField(auto_now=True, null=True)), + ('update_user', models.CharField(max_length=130, null=True)), + ('funding_call', models.IntegerField(max_length=5)), + ('project_identifier', models.IntegerField(max_length=5)), + ('applicant_name', models.CharField(max_length=50)), + ('rank', models.IntegerField(blank=True, max_length=3, null=True)), + ('status', models.CharField(max_length=50)), + ('score', models.IntegerField(blank=True, max_length=5, null=True)), + ('vehicle_deployed', models.CharField(max_length=50)), + ('vehicle_category', models.CharField(max_length=50)), + ('drive_type', models.CharField(max_length=50)), + ('vehicle_type', models.CharField(max_length=50)), + ('portfolio', models.CharField(max_length=50)), + ('make_and_model', models.CharField(max_length=50)), + ('economic_region', models.CharField(max_length=150)), + ('start_date', models.DateField(blank=True, null=True)), + ('completion_date', models.DateField(blank=True, null=True)), + ('project_type', models.CharField(max_length=50)), + ('class_3', models.IntegerField(blank=True, max_length=3, null=True)), + ('class_4', models.IntegerField(blank=True, max_length=3, null=True)), + ('class_5', models.IntegerField(blank=True, max_length=3, null=True)), + ('class_6', models.IntegerField(blank=True, max_length=3, null=True)), + ('class_7', models.IntegerField(blank=True, max_length=3, null=True)), + ('class_8', models.IntegerField(blank=True, max_length=3, null=True)), + ('on_road_total', models.IntegerField(blank=True, max_length=5, null=True)), + ('off_road', models.IntegerField(blank=True, max_length=5, null=True)), + ('level_2_charger', models.IntegerField(blank=True, max_length=5, null=True)), + ('level_3_charger', models.IntegerField(blank=True, max_length=5, null=True)), + ('high_level_3_charger', models.IntegerField(blank=True, max_length=5, null=True)), + ('level_charger', models.IntegerField(blank=True, max_length=5, null=True)), + ('other_charger', models.IntegerField(blank=True, max_length=5, null=True)), + ('h2_fuelling_station', models.IntegerField(blank=True, max_length=5, null=True)), + ('charger_brand', models.CharField(blank=True, max_length=50, null=True)), + ('h2_fuelling_station_description', models.CharField(blank=True, max_length=500, null=True)), + ('ghg_emission_reduction', models.IntegerField(blank=True, max_length=5, null=True)), + ('estimated_ghg_emission_reduction', models.IntegerField(blank=True, max_length=5, null=True)), + ('funding_efficiency', models.IntegerField(blank=True, max_length=5, null=True)), + ('market_emission_reductions', models.IntegerField(blank=True, max_length=5, null=True)), + ('cvp_funding_request', models.IntegerField(max_length=10)), + ('cvp_funding_contribution', models.IntegerField(max_length=10)), + ('external_funding', models.IntegerField(blank=True, max_length=10, null=True)), + ('proponent_funding', models.IntegerField(blank=True, max_length=10, null=True)), + ('project_cost_initial', models.IntegerField(max_length=10)), + ('project_cost_revised', models.IntegerField(max_length=10)), + ('funding_source', models.CharField(blank=True, max_length=50, null=True)), + ('notes', models.CharField(blank=True, max_length=500, null=True)), + ('imhzev', models.CharField(blank=True, max_length=50, null=True)), + ], + options={ + 'db_table': 'cvp_data', + }, + ), + ] diff --git a/django/api/migrations/0035_auto_20240910_2143.py b/django/api/migrations/0035_auto_20240910_2143.py new file mode 100644 index 00000000..1fb18028 --- /dev/null +++ b/django/api/migrations/0035_auto_20240910_2143.py @@ -0,0 +1,165 @@ +# Generated by Django 3.2.25 on 2024-09-10 21:43 + +from django.db import migrations, models + +def add_cvp_data_to_datasets(apps, schema_editor): + + Datasets = apps.get_model('api', 'Datasets') + + Datasets.objects.get_or_create( + create_timestamp="2024-09-10 00:00:00+00", + create_user="user", + name="CVP Data" + ) + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0034_cvpdata'), + ] + + + + operations = [ + migrations.AlterField( + model_name='cvpdata', + name='class_3', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='class_4', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='class_5', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='class_6', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='class_7', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='class_8', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='cvp_funding_contribution', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='cvp_funding_request', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='estimated_ghg_emission_reduction', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='external_funding', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='funding_call', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='funding_efficiency', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='ghg_emission_reduction', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='h2_fuelling_station', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='high_level_3_charger', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='level_2_charger', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='level_3_charger', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='level_charger', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='market_emission_reductions', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='off_road', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='on_road_total', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='other_charger', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='project_cost_initial', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='project_cost_revised', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='project_identifier', + field=models.IntegerField(), + ), + migrations.AlterField( + model_name='cvpdata', + name='proponent_funding', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='rank', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='score', + field=models.IntegerField(blank=True, null=True), + ), + migrations.RunPython(add_cvp_data_to_datasets), + ] diff --git a/django/api/migrations/0036_auto_20240911_1758.py b/django/api/migrations/0036_auto_20240911_1758.py new file mode 100644 index 00000000..8441653b --- /dev/null +++ b/django/api/migrations/0036_auto_20240911_1758.py @@ -0,0 +1,73 @@ +# Generated by Django 3.2.25 on 2024-09-11 17:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0035_auto_20240910_2143'), + ] + + operations = [ + migrations.AlterField( + model_name='cvpdata', + name='applicant_name', + field=models.CharField(max_length=500), + ), + migrations.AlterField( + model_name='cvpdata', + name='charger_brand', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='drive_type', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='funding_source', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='imhzev', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='make_and_model', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='portfolio', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='project_type', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='status', + field=models.CharField(max_length=500), + ), + migrations.AlterField( + model_name='cvpdata', + name='vehicle_category', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='vehicle_deployed', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='cvpdata', + name='vehicle_type', + field=models.CharField(max_length=100), + ), + ] diff --git a/django/api/migrations/0037_auto_20240911_1800.py b/django/api/migrations/0037_auto_20240911_1800.py new file mode 100644 index 00000000..b432e335 --- /dev/null +++ b/django/api/migrations/0037_auto_20240911_1800.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.25 on 2024-09-11 18:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0036_auto_20240911_1758'), + ] + + operations = [ + migrations.AlterField( + model_name='cvpdata', + name='funding_source', + field=models.CharField(blank=True, max_length=500, null=True), + ), + migrations.AlterField( + model_name='cvpdata', + name='imhzev', + field=models.CharField(blank=True, max_length=500, null=True), + ), + ] diff --git a/django/api/models/__init__.py b/django/api/models/__init__.py index 1e02db9e..d1fc14cc 100644 --- a/django/api/models/__init__.py +++ b/django/api/models/__init__.py @@ -26,3 +26,4 @@ from . import uploaded_vin_record from . import decoded_vin_record from . import regions +from . import cvp_data diff --git a/django/api/models/cvp_data.py b/django/api/models/cvp_data.py new file mode 100644 index 00000000..06a007a8 --- /dev/null +++ b/django/api/models/cvp_data.py @@ -0,0 +1,188 @@ +from auditable.models import Auditable +from django.db import models + + +class CVPData(Auditable): + + funding_call = models.IntegerField( + blank=False, null=False + ) + + project_identifier = models.IntegerField( + blank=False, null=False + ) + + applicant_name = models.CharField( + blank=False, null=False, max_length=500 + ) + + rank = models.IntegerField( + blank=True, null=True + ) + + status = models.CharField( + blank=False, null=False, max_length=500 + ) + + score = models.IntegerField( + blank=True, null=True + ) + + vehicle_deployed = models.CharField( + blank=False, null=False, max_length=100 + ) + + vehicle_category = models.CharField( + blank=False, null=False, max_length=100 + ) + + drive_type = models.CharField( + blank=False, null=False, max_length=100 + ) + + vehicle_type = models.CharField( + blank=False, null=False, max_length=100 + ) + + portfolio = models.CharField( + blank=False, null=False, max_length=100 + ) + + make_and_model = models.CharField( + blank=False, null=False, max_length=100 + ) + + economic_region = models.CharField( + blank=False, null=False, max_length=150 + ) + + start_date = models.DateField( + blank=True, null=True + ) + + completion_date = models.DateField( + blank=True, null=True + ) + + project_type = models.CharField( + blank=False, null=False, max_length=100 + ) + + class_3 = models.IntegerField( + blank=True, null=True + ) + + class_4 = models.IntegerField( + blank=True, null=True + ) + + class_5 = models.IntegerField( + blank=True, null=True + ) + + class_6 = models.IntegerField( + blank=True, null=True + ) + + class_7 = models.IntegerField( + blank=True, null=True + ) + + class_8 = models.IntegerField( + blank=True, null=True + ) + + on_road_total = models.IntegerField( + blank=True, null=True + ) + + off_road = models.IntegerField( + blank=True, null=True + ) + + level_2_charger = models.IntegerField( + blank=True, null=True + ) + + level_3_charger = models.IntegerField( + blank=True, null=True + ) + + high_level_3_charger = models.IntegerField( + blank=True, null=True + ) + + level_charger = models.IntegerField( + blank=True, null=True + ) + + other_charger = models.IntegerField( + blank=True, null=True + ) + + h2_fuelling_station = models.IntegerField( + blank=True, null=True + ) + + charger_brand = models.CharField( + blank=True, null=True, max_length=100 + ) + + h2_fuelling_station_description = models.CharField( + blank=True, null=True, max_length=500 + ) + + ghg_emission_reduction = models.IntegerField( + blank=True, null=True + ) + + estimated_ghg_emission_reduction = models.IntegerField( + blank=True, null=True + ) + + funding_efficiency = models.IntegerField( + blank=True, null=True + ) + + market_emission_reductions = models.IntegerField( + blank=True, null=True + ) + + cvp_funding_request = models.IntegerField( + blank=False, null=False + ) + + cvp_funding_contribution = models.IntegerField( + blank=False, null=False + ) + + external_funding = models.IntegerField( + blank=True, null=True + ) + + proponent_funding = models.IntegerField( + blank=True, null=True + ) + + project_cost_initial = models.IntegerField( + blank=False, null=False + ) + + project_cost_revised = models.IntegerField( + blank=False, null=False + ) + + funding_source = models.CharField( + blank=True, null=True, max_length=500 + ) + + notes = models.CharField( + blank=True, null=True, max_length=500 + ) + + imhzev = models.CharField( + blank=True, null=True, max_length=500 + ) + + class Meta: + db_table = "cvp_data" diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 9bd685f4..f85f6ceb 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -104,6 +104,12 @@ def prepare_go_electric_rebates(df): adjust_ger_manufacturer_names(df) return df +def prepare_cvp_data(df): + df = df.applymap(lambda s: s.upper() if type(s) == str else s) + df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in "biufc" else x.fillna("")) + + return df + def format_case(s, case = 'skip', ignore_list = []): s[s.notna()] = ( s[s.notna()] # I am applying this function to non NaN values only. If you do not, they get converted from NaN to nan and are more annoying to work with. @@ -317,7 +323,7 @@ def validate_field_values(df, *columns, **kwargs): indices = [] series = df[column] for index, value in series.items(): - if str(value) not in allowed_values[column] and value != '' and value is not None and not pd.isna(value): + if str(value).upper() not in (item.upper() for item in allowed_values[column]) and value != '' and value is not None and not pd.isna(value): indices.append(index + kwargs.get("indices_offset", 0)) if indices: result[column] = { From 900524f5b16c83c0ed0f04cb101084d2446f1112 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:17:46 -0700 Subject: [PATCH 41/67] Task: Remove Incorrect Warnings #384 (#392) * Checking for empty values in phone numbers and emails to avoid flagging them with an incorrect warning * Correcting statement checking for empty values --- django/api/services/spreadsheet_uploader_prep.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index f85f6ceb..1d40722c 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -245,7 +245,9 @@ def validate_phone_numbers(df, *columns, **kwargs): series = df[column] for index, phone_number in series.items(): formatted_number = str(phone_number).strip().replace('-', '') - if formatted_number == '' or len(formatted_number) != 10 or int(formatted_number[:3]) not in AREA_CODES: + if len(formatted_number) != 10 or int(formatted_number[:3]) not in AREA_CODES: + if pd.isna(formatted_number) or formatted_number == '': + continue indices.append(index + kwargs.get("indices_offset", 0)) if indices: result[column] = { @@ -302,6 +304,8 @@ def email_validator(df, *columns, **kwargs): try: validate_email(value, dns_resolver=resolver) except EmailNotValidError: + if pd.isna(value) or value == '': + continue indices.append(index + kwargs.get("indices_offset", 0)) if indices: result[column] = { From d1da2125e6fbbdd0ac823d15a4aead1ef6c7f0c0 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:34:52 -0700 Subject: [PATCH 42/67] Task: Uploader Error Descriptions #2208 (#391) * Added logic to include the expected values for restricted fields * Removed unneccessary logic, added a type conversion to display error messages in a better way. * Adding datetime expected type and error if a datetime field is incorrect --- django/api/services/spreadsheet_uploader.py | 29 +++++++++++-- .../api/services/spreadsheet_uploader_prep.py | 42 ++++++++++--------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 3c822361..95aefec3 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -3,6 +3,7 @@ import traceback import numpy as np from django.db import transaction +from datetime import datetime def get_field_default(model, field): field = model._meta.get_field(field) @@ -60,6 +61,14 @@ def transform_data( column_mapping = {e.value: e.name for e in column_mapping_enum} + type_to_string = { + int: "Integer", + float: "Float", + Decimal: "Decimal", + str: "String", + datetime: "Date (YYYY-MM-DD)" + } + errors_and_warnings = {} df = df.replace({np.nan: None}) @@ -81,12 +90,26 @@ def transform_data( errors_and_warnings[column] = {} if "Empty Value" not in errors_and_warnings[column]: errors_and_warnings[column]["Empty Value"] = { - "Expected Type": "Expected value where there isn't one.", + "Expected Type": "Cells in this column cannot be blank.", "Rows": [], "Severity": "Error" } errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) + if expected_type == datetime and value is not None and value != '': + try: + datetime.strptime(value, "%Y-%m-%d") + except ValueError: + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Incorrect Date Format" not in errors_and_warnings[column]: + errors_and_warnings[column]["Incorrect Date Format"] = { + "Expected Type": "The following rows contained an incorrect date format. Expected YYYY-MM-DD.", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Incorrect Date Format"]["Rows"].append(index + 1) + if expected_type in [int, float, Decimal] and value is not None and pd.notna(value) and value != '': value = str(value).replace(',', '').strip() try: @@ -101,7 +124,7 @@ def transform_data( errors_and_warnings[column] = {} if "Incorrect Type" not in errors_and_warnings[column]: errors_and_warnings[column]["Incorrect Type"] = { - "Expected Type": "The following rows contained incorrect value types for the " + column + " column", + "Expected Type": f"The following rows contained types for the column {column}. Expected {type_to_string.get(expected_type, str(expected_type))}", "Rows": [], "Severity": "Error" } @@ -113,7 +136,7 @@ def transform_data( errors_and_warnings[column] = {} if "Incorrect Type" not in errors_and_warnings[column]: errors_and_warnings[column]["Incorrect Type"] = { - "Expected Type": "The following rows contained incorrect value types for the " + column + " column", + "Expected Type": f"The following rows contained types for the column {column}. Expected {type_to_string.get(expected_type, str(expected_type))}", "Rows": [], "Severity": "Error" } diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 1d40722c..877d133a 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -82,10 +82,7 @@ def prepare_go_electric_rebates(df): df = df.applymap(lambda s: s.upper() if type(s) == str else s) - num_columns = df.select_dtypes(include=["number"]).columns.tolist() - df[num_columns] = df[num_columns].fillna(0) - - non_num_columns = df.columns.difference(num_columns) + non_num_columns = df.select_dtypes(exclude=["number"]).columns.tolist() df[non_num_columns] = df[non_num_columns].fillna("") format_dict = { 'title': ['Approvals', 'Applicant Name', 'Category', @@ -110,26 +107,28 @@ def prepare_cvp_data(df): return df -def format_case(s, case = 'skip', ignore_list = []): - s[s.notna()] = ( - s[s.notna()] # I am applying this function to non NaN values only. If you do not, they get converted from NaN to nan and are more annoying to work with. - .astype(str) # Convert to string - .str.strip() # Strip white spaces (this dataset suffers from extra tabs, lines, etc.) - ) +def format_case(s, case='skip', ignore_list=[]): + # Apply transformations to non-NaN values only + mask = s.notna() + + s.loc[mask] = ( + s.loc[mask] # I am applying this function to non-NaN values only. If you do not, they get converted from NaN to nan and are more annoying to work with. + .astype(str) # Convert to string + .str.strip() # Strip white spaces (this dataset suffers from extra tabs, lines, etc.) + ) + if case == 'title': - s = s.str.title() + s.loc[mask] = s.loc[mask].str.title() elif case == 'upper': - s = s.str.upper() + s.loc[mask] = s.loc[mask].str.upper() elif case == 'lower': - s = s.str.lower() + s.loc[mask] = s.loc[mask].str.lower() elif case == 'sentence': - ##filter out the temporary null records before changing to sentence case - s = s[s != 'TEMP_NULL'].str.capitalize() - elif case == 'skip': - pass + s.loc[mask] = s.loc[mask].str.capitalize() return s + def make_names_consistent(df): """ This step is done after formatting because people use all kinds of cases (`LTD`, `ltd', 'LIMITED'`, etc.). @@ -320,7 +319,8 @@ def email_validator(df, *columns, **kwargs): def validate_field_values(df, *columns, **kwargs): allowed_values = kwargs.get("fields_and_values") - + invalid_values = [] + result = {} for column in df.columns: if column in allowed_values: @@ -329,10 +329,12 @@ def validate_field_values(df, *columns, **kwargs): for index, value in series.items(): if str(value).upper() not in (item.upper() for item in allowed_values[column]) and value != '' and value is not None and not pd.isna(value): indices.append(index + kwargs.get("indices_offset", 0)) + if str(value) not in invalid_values: + invalid_values.append(str(value)) if indices: result[column] = { - "Invalid Values": { - "Expected Type": "The following rows only allow specific values", + ', '.join(invalid_values) + " - is not in the list of expected values": { + "Expected Type": ', '.join(allowed_values[column]), "Rows": indices, "Severity": "Error" } From 113996bcd9b6f9ec3b7c4994c43144aceabde885 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:38:56 -0700 Subject: [PATCH 43/67] Task: Economic Regions Table Data #381 (#396) * Migration to add regions table data * Removing comments * Removing id insertion and adding unique constraint to name --- django/api/migrations/0038_addregions.py | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 django/api/migrations/0038_addregions.py diff --git a/django/api/migrations/0038_addregions.py b/django/api/migrations/0038_addregions.py new file mode 100644 index 00000000..d6126ab5 --- /dev/null +++ b/django/api/migrations/0038_addregions.py @@ -0,0 +1,45 @@ +from django.db import migrations, models +from datetime import datetime + +def add_region_data(apps, schema_editor): + Regions = apps.get_model('api', 'Regions') + + current_timestamp = datetime.now() + + regions_data = [ + {"name": "Nechako"}, + {"name": "Northeast"}, + {"name": "Cariboo"}, + {"name": "North Coast"}, + {"name": "Vancouver Island/Coast"}, + {"name": "Mainland/Southwest"}, + {"name": "Thompson/Okanagan"}, + {"name": "Kootenay"}, + {"name": "Across BC"}, + ] + + for region in regions_data: + Regions.objects.get_or_create( + defaults={ + "create_timestamp": current_timestamp, + "create_user": "SYSTEM", + "update_timestamp": current_timestamp, + "update_user": "SYSTEM", + "name": region["name"] + } + ) + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0037_auto_20240911_1800'), + ] + + operations = [ + migrations.AlterField( + model_name='regions', + name='name', + field=models.CharField(max_length=250, null=False, unique=True) + ), + migrations.RunPython(add_region_data), + ] From c4ff84647cf1e584a7619e4f1366d4adcefa9e42 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:11:02 -0700 Subject: [PATCH 44/67] Refactoring errors and warnings to allow new data structure (#399) --- django/api/services/spreadsheet_uploader.py | 19 ++- .../api/services/spreadsheet_uploader_prep.py | 132 +++++++++++------- frontend/src/uploads/UploadContainer.js | 27 ++-- .../uploads/components/UploadIssuesDetail.js | 128 +++++++++++------ 4 files changed, 203 insertions(+), 103 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index 95aefec3..cbbd32ef 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -154,13 +154,20 @@ def transform_data( errors_and_warnings[column] = {} for issue, details in issues.items(): if issue not in errors_and_warnings[column]: - errors_and_warnings[column][issue] = { - "Expected Type": details.get("Expected Type", "Unknown"), - "Rows": details.get("Rows", []), - "Severity": details.get("Severity", "Error") - } + if(details.get("Severity", "Error") == 'Warning'): + errors_and_warnings[column][issue] = { + "Expected Type": details.get("Expected Type", "Unknown"), + "Groups": details.get("Groups", []), + "Severity": details.get("Severity", "Error") + } + else: + errors_and_warnings[column][issue] = { + "Expected Type": details.get("Expected Type", "Unknown"), + "Rows": details.get("Rows", []), + "Severity": details.get("Severity", "Error") + } else: - errors_and_warnings[column][issue]["Rows"].extend(details.get("Rows", [])) + errors_and_warnings[column][issue]["Groups"].extend(details.get("Groups", [])) column_mapping = {col.name: col.value for col in column_mapping_enum} inverse_column_mapping = {v: k for k, v in column_mapping.items()} diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 877d133a..391a41b9 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -204,119 +204,157 @@ def adjust_ger_manufacturer_names(df): def typo_checker(df, *columns, **kwargs): result = {} + for column in columns: - indices = [] series = df[column] unique_vals = set(series) - + map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) + + typo_groups = [] + processed_values = set() + for value in unique_vals: - singleton = set() - singleton.add(value) - matches = dl.get_close_matches( - value, - unique_vals.difference(singleton), - cutoff=kwargs["cutoff"] - ) + if value in processed_values: + continue + + matches = dl.get_close_matches(value, unique_vals.difference({value}), cutoff=kwargs.get("cutoff", 0.8)) + if matches: - value_indices = map_of_values_to_indices[value] - indices.extend(value_indices) - # it appears that difflib's "is similar" predicate S is not symmetric (i.e. aSb does not imply bSa) - # so we have to do: + current_group = { + "Typo Group": [value] + matches, + "Rows": [] + } + + current_group["Rows"].extend(map_of_values_to_indices[value]) + for match in matches: - match_indices = map_of_values_to_indices[match] - indices.extend(match_indices) - if indices: + current_group["Rows"].extend(map_of_values_to_indices[match]) + + processed_values.add(value) + processed_values.update(matches) + + typo_groups.append(current_group) + + if typo_groups: result[column] = { "Similar Values Detected": { "Expected Type": "We detected applicant names that sound very similar. If these names refer to the same person/entity, please replace the applicant names in your dataset to the preferred spelling to ensure consistency", - "Rows": sorted(list(set(indices))), + "Groups": typo_groups, "Severity": "Warning" } } + return result def validate_phone_numbers(df, *columns, **kwargs): result = {} for column in columns: - indices = [] series = df[column] - for index, phone_number in series.items(): + map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) + invalid_groups = [] + + for phone_number, indices in map_of_values_to_indices.items(): formatted_number = str(phone_number).strip().replace('-', '') if len(formatted_number) != 10 or int(formatted_number[:3]) not in AREA_CODES: if pd.isna(formatted_number) or formatted_number == '': continue - indices.append(index + kwargs.get("indices_offset", 0)) - if indices: + invalid_groups.append({ + "Invalid Phone Number": phone_number, + "Rows": indices + }) + + if invalid_groups: result[column] = { "Phone Number Appears Incorrect": { "Expected Type": "Ensure phone numbers match the Canadian format (XXX-XXX-XXXX)", - "Rows": indices, + "Groups": invalid_groups, "Severity": "Warning" } } return result + def location_checker(df, *columns, columns_to_features_map={}, **kwargs): result = {} + for column in columns: - indices = [] series = df[column] - map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) unique_values = set(series) - unique_values_list = list(unique_values) - + map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) + communities = set() - features_map = columns_to_features_map[column] + features_map = columns_to_features_map.get(column, {}) + for category_code, feature_types in features_map.items(): - get_placename_matches(unique_values_list, category_code, feature_types, 200, 1, communities) - - # Find names that don't have a match in the locations_set + get_placename_matches( + list(unique_values), category_code, feature_types, + 200, 1, communities + ) + names_without_match = unique_values.difference(communities) + unrecognized_groups = [] + for name in names_without_match: - indices_to_add = map_of_values_to_indices[name] - indices.extend(indices_to_add) - if indices: - indices.sort() + group = { + "Unrecognized Name": name, + "Rows": map_of_values_to_indices[name] + } + unrecognized_groups.append(group) + + if unrecognized_groups: result[column] = { "Unrecognized City Names": { - "Expected Type": "The following city names are not in the list of geographic names. Please double check that these places exist or have correct spelling and adjust your dataset accordingly.", - "Rows": sorted(list(set(indices))), + "Expected Type": ( + "The following city names are not in the list of geographic names. " + "Please double-check that these places exist or have correct spelling " + "and adjust your dataset accordingly." + ), + "Groups": unrecognized_groups, "Severity": "Warning" } } + return result + def email_validator(df, *columns, **kwargs): - resolver = None - get_resolver = kwargs.get("get_resolver") - if get_resolver is not None: - resolver = get_resolver() + resolver = kwargs.get("get_resolver", None) + if resolver: + resolver = resolver() + result = {} for column in columns: - indices = [] series = df[column] - for index, value in series.items(): + map_of_values_to_indices = get_map_of_values_to_indices(series, kwargs.get("indices_offset", 0)) + invalid_groups = [] + + for email, indices in map_of_values_to_indices.items(): try: - validate_email(value, dns_resolver=resolver) + validate_email(email, dns_resolver=resolver) except EmailNotValidError: - if pd.isna(value) or value == '': + if pd.isna(email) or email == '': continue - indices.append(index + kwargs.get("indices_offset", 0)) - if indices: + invalid_groups.append({ + "Invalid Email": email, + "Rows": indices + }) + + if invalid_groups: result[column] = { "Possible Errors in Email Addresses": { "Expected Type": "Verify email addresses are valid", - "Rows": indices, + "Groups": invalid_groups, "Severity": "Warning" } } return result + def validate_field_values(df, *columns, **kwargs): allowed_values = kwargs.get("fields_and_values") invalid_values = [] diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index a30cc902..53446c2e 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -64,44 +64,51 @@ const UploadContainer = () => { errors: 0, warnings: 0, }; - + issueArray.forEach((issue) => { - Object.keys(issue).forEach((column) => { const errorDetails = issue[column]; Object.keys(errorDetails).forEach((errorType) => { const severity = errorDetails[errorType].Severity; const expectedType = errorDetails[errorType]["Expected Type"]; - const rows = errorDetails[errorType].Rows; - const rowCount = rows.length; + const groups = errorDetails[errorType].Groups || []; if (severity === "Error") { + const rows = errorDetails[errorType].Rows; + const rowCount = rows.length; totalIssueCount.errors += rowCount; + if (!groupedErrors[column]) { groupedErrors[column] = {}; } if (!groupedErrors[column][errorType]) { groupedErrors[column][errorType] = { ExpectedType: expectedType, - Rows: rows, + Rows: [...rows], }; } else { groupedErrors[column][errorType].Rows.push(...rows); } } else if (severity === "Warning") { - totalIssueCount.warnings += rowCount; + let warningRowCount = 0; + if (!groupedWarnings[column]) { groupedWarnings[column] = {}; } if (!groupedWarnings[column][errorType]) { groupedWarnings[column][errorType] = { ExpectedType: expectedType, - Rows: rows, + Groups: [], }; - } else { - groupedWarnings[column][errorType].Rows.push(...rows); } + + groups.forEach((group) => { + groupedWarnings[column][errorType].Groups.push(group); + warningRowCount += group.Rows.length; + }); + + totalIssueCount.warnings += warningRowCount; } }); }); @@ -109,8 +116,6 @@ const UploadContainer = () => { return { groupedErrors, groupedWarnings, totalIssueCount }; }; - - const showError = (error) => { const { response: errorResponse } = error; diff --git a/frontend/src/uploads/components/UploadIssuesDetail.js b/frontend/src/uploads/components/UploadIssuesDetail.js index 6c504e0d..c4275c37 100644 --- a/frontend/src/uploads/components/UploadIssuesDetail.js +++ b/frontend/src/uploads/components/UploadIssuesDetail.js @@ -1,20 +1,57 @@ import PropTypes from "prop-types"; import React, { useState } from "react"; import { Box, Button } from "@mui/material"; -import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import ErrorOutlineIcon from "@mui/icons-material/ErrorOutline"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { - const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue + const [showAllRowsMap, setShowAllRowsMap] = useState({}); const classname = type === "error" ? "error" : "warning"; - const toggleShowAllRows = (column, errorType) => { - const key = `${column}_${errorType}`; + const toggleShowAllRows = (column, errorType, groupIndex) => { + const key = `${column}_${errorType}_${groupIndex}`; setShowAllRowsMap((prevState) => ({ ...prevState, [key]: !prevState[key], })); }; + const renderWarning = (group) => ( + <> +
    +
  • + Rows:{" "} + {group.Rows.join(", ")} + {Object.keys(group).map((key) => { + if (key !== "Rows") { + return ( + + {" "} {/* spacer */} + {Array.isArray(group[key]) + ? group[key].join(", ") + : group[key]} + + ); + } + return null; + })} +
    • +
+ + ); + + const renderError = (errorDetails) => ( + <> +
    +
  • +
    + Rows:{" "} + {errorDetails.Rows.join(", ")} +
    +
    • +
+ + ); + return ( { /> - {totalIssueCount} {type == 'error' ? 'Errors' : 'Warnings'}  + {totalIssueCount} {type === "error" ? "Errors" : "Warnings"}  ({msg}) {Object.keys(issues).map((column) => ( Column: {column} - {Object.keys(issues[column]).map((errorType, index) => ( -
-
- {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')} - {type.charAt(0).toUpperCase() + type.slice(1)} Name: {errorType} + {Object.keys(issues[column]).map((errorType, index) => { + const errorDetails = issues[column][errorType]; + + return ( +
+
+
    +
  • +
    + {(Object.keys(issues[column]).length > 1 + ? `(${index + 1}) ` + : "")} + {type.charAt(0).toUpperCase() + type.slice(1)} Name:{" "} + {errorType} +
    +
    • +
+
    +
  • + Expected Value:{" "} + {errorDetails.ExpectedType || errorDetails.ExpectedFormat} +
    • +
-
- Expected value:{" "} - {issues[column][errorType].ExpectedType || - issues[column][errorType].ExpectedFormat} -
-
- Rows with {type}:{" "} - - {issues[column][errorType].Rows.slice( - 0, - showAllRowsMap[`${column}_${errorType}`] ? undefined : 15, - ).join(", ")} - {issues[column][errorType].Rows.length > 15 && - !showAllRowsMap[`${column}_${errorType}`] && - "..."} - + {errorDetails.Groups ? ( + errorDetails.Groups.map((group, groupIndex) => ( +
+ {renderWarning(group)} + {group.Rows.length > 15 && ( + + )} +
+ )) + ) : ( + renderError(errorDetails) + )}
- {issues[column][errorType].Rows.length > 15 && ( - - )} -
- ))} + ); + })} ))} From d98e374153805424fb57d300ce11f0e72b63616a Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:45:54 -0700 Subject: [PATCH 45/67] Added new critical error type (#401) --- django/api/services/spreadsheet_uploader.py | 42 ++++++++++----- frontend/src/uploads/UploadContainer.js | 45 +++++++++++++--- .../src/uploads/components/UploadIssues.js | 21 +++++++- .../uploads/components/UploadIssuesDetail.js | 51 ++++++++++++++++--- 4 files changed, 132 insertions(+), 27 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index cbbd32ef..e6dd2285 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -33,8 +33,7 @@ def extract_data(excel_file, sheet_name, header_row): df = trim_all_columns(df) return df except Exception as e: - traceback.print_exc() - raise + return None def transform_data( @@ -50,9 +49,17 @@ def transform_data( df = df[[col for col in df.columns if col in required_columns]] + errors_and_warnings = {} + missing_columns = [col for col in required_columns if col not in df.columns] if (missing_columns): - raise ValueError(f"Missing columns: {', '.join(missing_columns)}") + errors_and_warnings['Headers'] = {} + errors_and_warnings['Headers']['Missing Headers'] = { + "Expected Type": "The spreadsheet provided is missing headers", + "Rows": missing_columns, + "Severity": "Critical" + } + return df, errors_and_warnings for prep_func in preparation_functions: df = prep_func(df) @@ -69,7 +76,6 @@ def transform_data( datetime: "Date (YYYY-MM-DD)" } - errors_and_warnings = {} df = df.replace({np.nan: None}) for index, row in df.iterrows(): @@ -211,17 +217,27 @@ def import_from_xls( validation_functions=[], check_for_warnings=True, ): + errors_and_warnings = {} try: df = extract_data(excel_file, sheet_name, header_row) - df, errors_and_warnings = transform_data( - df, - dataset_columns, - column_mapping_enum, - field_types, - model, - preparation_functions, - validation_functions, - ) + if df is not None: + df, errors_and_warnings = transform_data( + df, + dataset_columns, + column_mapping_enum, + field_types, + model, + preparation_functions, + validation_functions, + ) + + else: + errors_and_warnings['Spreadsheet'] = {} + errors_and_warnings['Spreadsheet']['Missing Worksheet'] = { + 'Expected Type': 'The worksheet is missing or incorrectly named', + 'Rows': [sheet_name], + 'Severity': 'Critical' + } if check_for_warnings: ## do the error checking diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index 53446c2e..5ed58c1c 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -25,6 +25,7 @@ const UploadContainer = () => { const [openDialog, setOpenDialog] = useState(false); const [adminUser, setAdminUser] = useState(false); const [totalIssueCount, setTotalIssueCount] = useState({}); + const [groupedCriticalErrors, setGroupedCriticalErrors] = useState({}); const [groupedErrors, setGroupedErrors] = useState({}); const [groupedWarnings, setGroupedWarnings] = useState({}); const [alertDialogText, setAlertDialogText] = useState({ @@ -58,25 +59,45 @@ const UploadContainer = () => { }; const groupAndCountRows = (issueArray) => { + const groupedCriticalErrors = {} const groupedErrors = {}; const groupedWarnings = {}; const totalIssueCount = { + criticalErrors: 0, errors: 0, warnings: 0, }; + issueArray.forEach((issue) => { Object.keys(issue).forEach((column) => { const errorDetails = issue[column]; + Object.keys(errorDetails).forEach((errorType) => { + const severity = errorDetails[errorType].Severity; const expectedType = errorDetails[errorType]["Expected Type"]; const groups = errorDetails[errorType].Groups || []; - - if (severity === "Error") { + + if (severity === "Critical") { const rows = errorDetails[errorType].Rows; const rowCount = rows.length; + totalIssueCount.criticalErrors += rowCount; + if (!groupedCriticalErrors[column]) { + groupedCriticalErrors[column] = {}; + } + if (!groupedCriticalErrors[column][errorType]) { + groupedCriticalErrors[column][errorType] = { + ExpectedType: expectedType, + Rows: rows, + }; + } else { + groupedCriticalErrors[column][errorType].Rows.push(...rows); + } + } else if (severity === "Error") { + const rows = errorDetails[errorType].Rows || null; + const rowCount = rows.length || groups.length; totalIssueCount.errors += rowCount; if (!groupedErrors[column]) { @@ -113,8 +134,9 @@ const UploadContainer = () => { }); }); }); + - return { groupedErrors, groupedWarnings, totalIssueCount }; + return { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount }; }; const showError = (error) => { @@ -181,17 +203,27 @@ const UploadContainer = () => { }); if (Object.keys(warnings).length > 0 && checkForWarnings) { - const { groupedErrors, groupedWarnings, totalIssueCount } = + const { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount } = groupAndCountRows(Object.values(warnings)); + + setGroupedCriticalErrors(groupedCriticalErrors) setGroupedErrors(groupedErrors); setGroupedWarnings(groupedWarnings); setTotalIssueCount(totalIssueCount); setAlertDialogText({ title: - "Your file has been processed and contains the following errors and warnings!", + totalIssueCount.criticalErrors > 0 ? "Your upload contained critical errors that must be fixed before it can be processed!" : "Your file has been processed and contains the following errors and warnings!", content: ( <> + {totalIssueCount.criticalErrors >= 1 && ( +
+ + {totalIssueCount.criticalErrors} Critical Errors + + - Must fix before file can be processed +
+ )} {totalIssueCount.errors >= 1 && (
@@ -323,10 +355,11 @@ const UploadContainer = () => { handleConfirm={alertDialogText.confirmAction} /> - {(totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( + {(totalIssueCount.criticalErrors || totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( - Your file has been processed and contains the following errors and - warnings. Please review them below: + {totalIssueCount.criticalErrors > 0 ? 'Your file cannot be processed because it contains critical errors. Please review them below.': 'Your file has been processed and contains the following errors and warnings. Please review them below'} + {totalIssueCount.criticalErrors >= 1 && ( + + + {totalIssueCount.criticalErrors} Critical Errors   + + - {criticalMsg} + + )} {totalIssueCount.errors >= 1 && ( @@ -72,6 +81,14 @@ const UploadIssues = ({ + {totalIssueCount.criticalErrors >= 1 && ( + + )} {totalIssueCount.errors >= 1 && ( { - const [showAllRowsMap, setShowAllRowsMap] = useState({}); - const classname = type === "error" ? "error" : "warning"; - const toggleShowAllRows = (column, errorType, groupIndex) => { - const key = `${column}_${errorType}_${groupIndex}`; + const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue + const errorTypes = ['critical', 'error'] + const classname = errorTypes.includes(type) ? "error" : "warning"; + const toggleShowAllRows = (column, errorType) => { + const key = `${column}_${errorType}`; setShowAllRowsMap((prevState) => ({ ...prevState, [key]: !prevState[key], @@ -56,7 +57,7 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { @@ -66,13 +67,51 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { /> - {totalIssueCount} {type === "error" ? "Errors" : "Warnings"}  + {totalIssueCount} {type === 'critical' ? 'Critical Errors' : type === 'error' ? 'Errors' : 'Warnings'}  ({msg}) {Object.keys(issues).map((column) => ( Column: {column} + {Object.keys(issues[column]).map((errorType, index) => ( +
+
+ {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')} + {type.charAt(0).toUpperCase() + type.slice(1)} {type === `critical` ? `Error` : `Name:`} {errorType} +
+
+ Expected value:{" "} + + {issues[column][errorType].ExpectedType || + issues[column][errorType].ExpectedFormat} + +
+
+ {column === 'Headers' ? `Missing Headers: ` : column === 'Spreadsheet' ? 'Missing Spreadsheet: ' : `Rows with ${type}: `} + + {issues[column][errorType]?.Rows?.slice( + 0, + showAllRowsMap[`${column}_${errorType}`] ? undefined : 15, + ).join(", ")} + {issues[column][errorType]?.Rows?.length > 15 && + !showAllRowsMap[`${column}_${errorType}`] && + "..."} + +
+ {issues[column][errorType]?.Rows?.length > 15 && ( + + )} +
+ ))} {Object.keys(issues[column]).map((errorType, index) => { const errorDetails = issues[column][errorType]; From 21814e5a7d4d097c0052a0f2387a9569d877c0ed Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:02:53 -0700 Subject: [PATCH 46/67] fix: 381 - fix add regions migration file (#407) --- django/api/migrations/0038_addregions.py | 2 +- django/api/models/regions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django/api/migrations/0038_addregions.py b/django/api/migrations/0038_addregions.py index d6126ab5..ffffcbd5 100644 --- a/django/api/migrations/0038_addregions.py +++ b/django/api/migrations/0038_addregions.py @@ -20,12 +20,12 @@ def add_region_data(apps, schema_editor): for region in regions_data: Regions.objects.get_or_create( + name=region["name"], defaults={ "create_timestamp": current_timestamp, "create_user": "SYSTEM", "update_timestamp": current_timestamp, "update_user": "SYSTEM", - "name": region["name"] } ) diff --git a/django/api/models/regions.py b/django/api/models/regions.py index 095c238e..854cf98d 100644 --- a/django/api/models/regions.py +++ b/django/api/models/regions.py @@ -4,7 +4,7 @@ class Regions(Auditable): - name = models.CharField(blank=False, null=False, max_length=250) + name = models.CharField(blank=False, null=False, max_length=250, unique=True) class Meta: db_table = "regions" From 3e68241b227f41e7c5ec770e732a5ff7426cecb8 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:07:00 -0800 Subject: [PATCH 47/67] Task: CVP Column Update #405 (#408) * Changed portfolio column to Use Case and Class, updated validate_field_values function to check for multiple values in a string * Making delimiter for splitting values in a function a kwarg argument --- django/api/constants/constants.py | 15 ++++++----- django/api/constants/misc.py | 6 ++--- .../api/migrations/0039_auto_20241031_2123.py | 27 +++++++++++++++++++ django/api/models/cvp_data.py | 8 ++++-- .../api/services/spreadsheet_uploader_prep.py | 17 +++++++----- 5 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 django/api/migrations/0039_auto_20241031_2123.py diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 28a0b442..cb6c5ea9 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -436,7 +436,8 @@ class CVPDataColumns(Enum): VEHICLE_CATEGORY = "Vehicle Category" DRIVE_TYPE = "Drive Type" VEHICLE_TYPE = "Vehicle Type" - PORTFOLIO = "Portfolio" + ROAD_CLASS = "Class" + USE_CASE = "Use Case" MAKE_AND_MODEL = "Vehicle Make and Model" ECONOMIC_REGION = "Economic Region" START_DATE = "Start Date" @@ -483,7 +484,8 @@ class CVPDataColumnMapping(Enum): vehicle_category = "Vehicle Category" drive_type = "Drive Type" vehicle_type = "Vehicle Type" - portfolio = "Portfolio" + road_class = "Class" + use_case = "Use Case" make_and_model = "Vehicle Make and Model" economic_region = "Economic Region" start_date = "Start Date" @@ -713,7 +715,8 @@ class CVPDataColumnMapping(Enum): "vehicle_category": str, "drive_type": str, "vehicle_type": str, - "portfolio": str, + "road_class": str, + "use_case": str, "make_and_model": str, "economic_region": str, "start_date": datetime.date, @@ -761,7 +764,7 @@ class CVPDataColumnMapping(Enum): "sheet_name": "ARC Data", "preparation_functions": [prepare_arc_project_tracking], "validation_functions": [ - {'function': validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": ARC_VALID_FIELD_VALUES}}, + {'function': validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": ARC_VALID_FIELD_VALUES, "delimiter": ","}}, {"function": region_checker, "columns": ['Economic Region'], "kwargs": {"indices_offset":2}}, ] }, @@ -825,7 +828,7 @@ class CVPDataColumnMapping(Enum): {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, {"function": location_checker, "columns": ["City"], "kwargs": {"columns_to_features_map": {"City": LOCALITY_FEATURES_MAP}, "indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, - {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, + {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES, "delimiter": ","}}, ] }, "CVP Data": { @@ -834,7 +837,7 @@ class CVPDataColumnMapping(Enum): "column_mapping": CVPDataColumnMapping, "sheet_name": "Data", "preparation_functions": [prepare_cvp_data], - "validation_functions": [{"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": CVP_DATA_VALID_FIELD_VALUES}},] + "validation_functions": [{"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": CVP_DATA_VALID_FIELD_VALUES, "delimiter": ","}},] }, } diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index 10dc15f5..c5cf5af8 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -94,9 +94,9 @@ 'Status': ['Approved', 'Completed', 'Terminated', 'Not Approved', 'Application Withdrawn'], 'Vehicles Deployed': ['Yes', 'No'], 'Vehicle Category': ['On-Road', 'Off-Road', 'Marine', 'Rail', 'Aviation'], - 'Portfolio': [ - 'Road - 8', 'Rail', 'Marine', 'Road - 8C', 'School Bus', - 'Loader', 'Road - 6', 'Fork Lift', 'Recreation' + 'Class': [ + 'Road - 3', 'Road - 4', 'Road - 5', 'Road - 6', 'Road - 7', + 'Road - 8', 'Road - 8C' ], 'Economic Region': [ 'Nechako', 'Northeast', 'North Coast', 'Cariboo', 'Vancouver Island/Coast', diff --git a/django/api/migrations/0039_auto_20241031_2123.py b/django/api/migrations/0039_auto_20241031_2123.py new file mode 100644 index 00000000..eb120c7f --- /dev/null +++ b/django/api/migrations/0039_auto_20241031_2123.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-10-31 21:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0038_addregions'), + ] + + operations = [ + migrations.RemoveField( + model_name='cvpdata', + name='portfolio', + ), + migrations.AddField( + model_name='cvpdata', + name='road_class', + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AddField( + model_name='cvpdata', + name='use_case', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/django/api/models/cvp_data.py b/django/api/models/cvp_data.py index 06a007a8..9f5c8770 100644 --- a/django/api/models/cvp_data.py +++ b/django/api/models/cvp_data.py @@ -44,8 +44,12 @@ class CVPData(Auditable): blank=False, null=False, max_length=100 ) - portfolio = models.CharField( - blank=False, null=False, max_length=100 + road_class = models.CharField( + blank=True, null=True, max_length=100 + ) + + use_case = models.CharField( + blank=True, null=True, max_length=100 ) make_and_model = models.CharField( diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 391a41b9..ed30de8d 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -353,22 +353,27 @@ def email_validator(df, *columns, **kwargs): } return result - - def validate_field_values(df, *columns, **kwargs): allowed_values = kwargs.get("fields_and_values") invalid_values = [] result = {} + delimiter = kwargs.get("delimiter") for column in df.columns: if column in allowed_values: indices = [] series = df[column] for index, value in series.items(): - if str(value).upper() not in (item.upper() for item in allowed_values[column]) and value != '' and value is not None and not pd.isna(value): - indices.append(index + kwargs.get("indices_offset", 0)) - if str(value) not in invalid_values: - invalid_values.append(str(value)) + if delimiter is not None: + items = [item.strip() for item in value.split(delimiter)] + + for item in items: + if str(item).upper() not in (valid.upper() for valid in allowed_values[column]) and item != '' and item is not None and not pd.isna(item): + if index + kwargs.get("indices_offset", 0) not in indices: + indices.append(index + kwargs.get("indices_offset", 0)) + if str(item) not in invalid_values: + invalid_values.append(str(item)) + if indices: result[column] = { ', '.join(invalid_values) + " - is not in the list of expected values": { From dba29f30439efd89710f346ab88dae678f1285e2 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:51:27 -0800 Subject: [PATCH 48/67] cthub 405: small change (#411) --- django/api/constants/constants.py | 4 ++-- .../api/services/spreadsheet_uploader_prep.py | 21 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index cb6c5ea9..227dab2c 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -764,7 +764,7 @@ class CVPDataColumnMapping(Enum): "sheet_name": "ARC Data", "preparation_functions": [prepare_arc_project_tracking], "validation_functions": [ - {'function': validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": ARC_VALID_FIELD_VALUES, "delimiter": ","}}, + {'function': validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": ARC_VALID_FIELD_VALUES}}, {"function": region_checker, "columns": ['Economic Region'], "kwargs": {"indices_offset":2}}, ] }, @@ -828,7 +828,7 @@ class CVPDataColumnMapping(Enum): {"function": typo_checker, "columns": ["Applicant Name"], "kwargs": {"cutoff": 0.8, "indices_offset": 2}}, {"function": location_checker, "columns": ["City"], "kwargs": {"columns_to_features_map": {"City": LOCALITY_FEATURES_MAP}, "indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, - {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES, "delimiter": ","}}, + {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, ] }, "CVP Data": { diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index ed30de8d..34910356 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -364,15 +364,18 @@ def validate_field_values(df, *columns, **kwargs): indices = [] series = df[column] for index, value in series.items(): - if delimiter is not None: - items = [item.strip() for item in value.split(delimiter)] - - for item in items: - if str(item).upper() not in (valid.upper() for valid in allowed_values[column]) and item != '' and item is not None and not pd.isna(item): - if index + kwargs.get("indices_offset", 0) not in indices: - indices.append(index + kwargs.get("indices_offset", 0)) - if str(item) not in invalid_values: - invalid_values.append(str(item)) + if value is not None and pd.notna(value): + str_value = str(value) + items = [str_value.strip()] + if delimiter is not None: + items = [item.strip() for item in str_value.split(delimiter)] + + for item in items: + if str(item).upper() not in (valid.upper() for valid in allowed_values[column]) and item != '' and item is not None and not pd.isna(item): + if index + kwargs.get("indices_offset", 0) not in indices: + indices.append(index + kwargs.get("indices_offset", 0)) + if str(item) not in invalid_values: + invalid_values.append(str(item)) if indices: result[column] = { From c78a780ccf720eadc175e280262afc29bf804dd6 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:43:10 -0800 Subject: [PATCH 49/67] Task: 403 - tooltip, download template button (#412) * task: styles download template button task: adds tooltip for disabled file drop area, adds condition for disabling (file already selected) * removes commented text --- frontend/src/app/styles/App.scss | 3 +++ frontend/src/app/styles/variables.scss | 2 ++ frontend/src/uploads/components/FileDrop.js | 2 +- .../src/uploads/components/FileDropArea.js | 23 ++++++++++++++----- frontend/src/uploads/components/UploadPage.js | 14 ++++++++--- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/frontend/src/app/styles/App.scss b/frontend/src/app/styles/App.scss index 9e130984..5570fda0 100644 --- a/frontend/src/app/styles/App.scss +++ b/frontend/src/app/styles/App.scss @@ -98,6 +98,9 @@ h4 { color: #1a5a96 !important; text-decoration: underline !important; text-transform: none !important; + &:disabled { + color: $default-disabled-grey !important; + } } .button-lowercase { diff --git a/frontend/src/app/styles/variables.scss b/frontend/src/app/styles/variables.scss index af1bdba0..4daa9f65 100644 --- a/frontend/src/app/styles/variables.scss +++ b/frontend/src/app/styles/variables.scss @@ -18,3 +18,5 @@ $default-background-grey: #f2f2f2; $background-light-blue: #e7f4f7; $table-border: rgba(49, 49, 50, 0.33); + +$default-disabled-grey: #9a9a9a; \ No newline at end of file diff --git a/frontend/src/uploads/components/FileDrop.js b/frontend/src/uploads/components/FileDrop.js index 31650e1d..d0e4d71c 100644 --- a/frontend/src/uploads/components/FileDrop.js +++ b/frontend/src/uploads/components/FileDrop.js @@ -22,7 +22,7 @@ const FileDrop = (props) => {
Drag and Drop files here or
- diff --git a/frontend/src/uploads/components/FileDropArea.js b/frontend/src/uploads/components/FileDropArea.js index 97dd05e0..69ef484e 100644 --- a/frontend/src/uploads/components/FileDropArea.js +++ b/frontend/src/uploads/components/FileDropArea.js @@ -1,6 +1,6 @@ import React from "react"; import PropTypes from "prop-types"; -import { Box, Button, Grid } from "@mui/material"; +import { Box, Button, Grid, Tooltip } from "@mui/material"; import ClearIcon from "@mui/icons-material/Clear"; import FileDrop from "./FileDrop"; import getFileSize from "../../app/utilities/getFileSize"; @@ -44,11 +44,22 @@ const FileDropArea = (props) => {
- + + + + +
{uploadFiles.length > 0 && ( diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index d3c5ec7d..6260dd8b 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -11,6 +11,7 @@ import { FormControl, } from "@mui/material"; import UploadIcon from "@mui/icons-material/Upload"; +import DownloadIcon from "@mui/icons-material/Download"; import FileDropArea from "./FileDropArea"; import Loading from "../../app/components/Loading"; @@ -54,8 +55,15 @@ const UploadPage = (props) => { {selectionList} {datasetSelected && ( - )}
@@ -85,7 +93,7 @@ const UploadPage = (props) => {
Date: Thu, 21 Nov 2024 10:18:18 -0800 Subject: [PATCH 50/67] Removing duplicate formatting, fixing returned structure of region error info, updating alert to only display on successful import (#413) --- .../api/services/spreadsheet_uploader_prep.py | 2 +- frontend/src/uploads/UploadContainer.js | 2 +- .../uploads/components/UploadIssuesDetail.js | 121 ++++++------------ frontend/src/uploads/components/UploadPage.js | 3 +- 4 files changed, 40 insertions(+), 88 deletions(-) diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 34910356..8d8317bf 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -404,7 +404,7 @@ def region_checker(df, *columns, **kwargs): if indices: result[column] = { "Invalid Region": { - "Expected Type": "The following rows have an invalid region", + "Expected Type": ", ".join(valid_regions), "Rows": indices, "Severity": "Error" } diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index 5ed58c1c..1f8277d0 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -328,7 +328,6 @@ const UploadContainer = () => { if (refresh) { return ; } - const alertElement = alert && alertContent && alertSeverity ? ( @@ -381,6 +380,7 @@ const UploadContainer = () => { downloadSpreadsheet={downloadSpreadsheet} setAlert={setAlert} loading={loading} + totalIssueCount={totalIssueCount} /> {adminUser && ( diff --git a/frontend/src/uploads/components/UploadIssuesDetail.js b/frontend/src/uploads/components/UploadIssuesDetail.js index 75331ba6..adf4ee96 100644 --- a/frontend/src/uploads/components/UploadIssuesDetail.js +++ b/frontend/src/uploads/components/UploadIssuesDetail.js @@ -6,8 +6,9 @@ import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { const [showAllRowsMap, setShowAllRowsMap] = useState({}); // State to toggle showing all rows for each issue - const errorTypes = ['critical', 'error'] + const errorTypes = ["critical", "error"]; const classname = errorTypes.includes(type) ? "error" : "warning"; + const toggleShowAllRows = (column, errorType) => { const key = `${column}_${errorType}`; setShowAllRowsMap((prevState) => ({ @@ -17,40 +18,32 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { }; const renderWarning = (group) => ( - <> -
    -
  • - Rows:{" "} - {group.Rows.join(", ")} - {Object.keys(group).map((key) => { - if (key !== "Rows") { - return ( - - {" "} {/* spacer */} - {Array.isArray(group[key]) - ? group[key].join(", ") - : group[key]} - - ); - } - return null; - })} -
    • -
- +
    +
  • + Rows: {group.Rows.join(", ")} + {Object.keys(group).map((key) => { + if (key !== "Rows") { + return ( + + {" "} {/* spacer */} + {Array.isArray(group[key]) ? group[key].join(", ") : group[key]} + + ); + } + return null; + })} +
    • +
); - + const renderError = (errorDetails) => ( - <> -
    -
  • -
    - Rows:{" "} - {errorDetails.Rows.join(", ")} -
    -
    • -
- +
    +
  • +
    + Rows: {errorDetails.Rows.join(", ")} +
    +
    • +
); return ( @@ -67,75 +60,33 @@ const UploadIssuesDetail = ({ type, issues, totalIssueCount, msg }) => { /> - {totalIssueCount} {type === 'critical' ? 'Critical Errors' : type === 'error' ? 'Errors' : 'Warnings'}  + {totalIssueCount}{" "} + {type === "critical" ? "Critical Errors" : type === "error" ? "Errors" : "Warnings"}  ({msg}) {Object.keys(issues).map((column) => ( Column: {column} - {Object.keys(issues[column]).map((errorType, index) => ( -
-
- {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : '')} - {type.charAt(0).toUpperCase() + type.slice(1)} {type === `critical` ? `Error` : `Name:`} {errorType} -
-
- Expected value:{" "} - - {issues[column][errorType].ExpectedType || - issues[column][errorType].ExpectedFormat} - -
-
- {column === 'Headers' ? `Missing Headers: ` : column === 'Spreadsheet' ? 'Missing Spreadsheet: ' : `Rows with ${type}: `} - - {issues[column][errorType]?.Rows?.slice( - 0, - showAllRowsMap[`${column}_${errorType}`] ? undefined : 15, - ).join(", ")} - {issues[column][errorType]?.Rows?.length > 15 && - !showAllRowsMap[`${column}_${errorType}`] && - "..."} - -
- {issues[column][errorType]?.Rows?.length > 15 && ( - - )} -
- ))} {Object.keys(issues[column]).map((errorType, index) => { const errorDetails = issues[column][errorType]; - return (
-
  • - {(Object.keys(issues[column]).length > 1 - ? `(${index + 1}) ` - : "")} + {(Object.keys(issues[column]).length > 1 ? `(${index + 1}) ` : "")} {type.charAt(0).toUpperCase() + type.slice(1)} Name:{" "} {errorType}
    • -
-
    -
  • - Expected Value:{" "} - {errorDetails.ExpectedType || errorDetails.ExpectedFormat} -
    • -
-
+ +
    +
  • + Expected Value:{" "} + {errorDetails.ExpectedType || errorDetails.ExpectedFormat} +
    • +
{errorDetails.Groups ? ( errorDetails.Groups.map((group, groupIndex) => (
diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index 6260dd8b..3094add1 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -29,6 +29,7 @@ const UploadPage = (props) => { downloadSpreadsheet, setAlert, loading, + totalIssueCount } = props; const selectionList = datasetList.map((obj, index) => ( @@ -39,7 +40,7 @@ const UploadPage = (props) => { <>

Upload Program Data

- {alertElement} + {totalIssueCount < 1 && alertElement}

Select Program    

From 49c589af90b0e35a8949f4c325b6e2c49b50e932 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:33:54 -0800 Subject: [PATCH 51/67] feat: 402 - file requirements (#410) --- .../api/migrations/0039_filerequirements.py | 31 +++++++++ ...0040_add_datasets_and_file_requirements.py | 63 +++++++++++++++++++ django/api/models/__init__.py | 1 + django/api/models/file_requirements.py | 20 ++++++ django/api/serializers/file_requirements.py | 9 +++ django/api/services/file_requirements.py | 5 ++ django/api/viewsets/upload.py | 11 ++++ .../uploads/components/FileRequirements.js | 42 +++++++++++++ frontend/src/uploads/components/UploadPage.js | 6 ++ frontend/src/uploads/routes.js | 1 + 10 files changed, 189 insertions(+) create mode 100644 django/api/migrations/0039_filerequirements.py create mode 100644 django/api/migrations/0040_add_datasets_and_file_requirements.py create mode 100644 django/api/models/file_requirements.py create mode 100644 django/api/serializers/file_requirements.py create mode 100644 django/api/services/file_requirements.py create mode 100644 frontend/src/uploads/components/FileRequirements.js diff --git a/django/api/migrations/0039_filerequirements.py b/django/api/migrations/0039_filerequirements.py new file mode 100644 index 00000000..d38af6f6 --- /dev/null +++ b/django/api/migrations/0039_filerequirements.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.25 on 2024-11-07 22:15 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0038_addregions'), + ] + + operations = [ + migrations.CreateModel( + name='FileRequirements', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)), + ('create_user', models.CharField(default='SYSTEM', max_length=130)), + ('update_timestamp', models.DateTimeField(auto_now=True, null=True)), + ('update_user', models.CharField(max_length=130, null=True)), + ('sheet', models.TextField(blank=True, null=True)), + ('columns', models.TextField(blank=True, null=True)), + ('formats', models.TextField(blank=True, null=True)), + ('dataset', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='file_requirements', to='api.datasets')), + ], + options={ + 'db_table': 'file_requirements', + }, + ), + ] diff --git a/django/api/migrations/0040_add_datasets_and_file_requirements.py b/django/api/migrations/0040_add_datasets_and_file_requirements.py new file mode 100644 index 00000000..b2ec042a --- /dev/null +++ b/django/api/migrations/0040_add_datasets_and_file_requirements.py @@ -0,0 +1,63 @@ +# Generated by Django 3.2.25 on 2024-11-07 22:17 + +from django.db import migrations + + +def add_datasets_and_file_requirements(apps, schema_editor): + Datasets = apps.get_model("api", "Datasets") + FileRequirements = apps.get_model("api", "FileRequirements") + columns_default_text = "All required columns must match the column names exactly as they appear in the provided template" + formats_default_text = "File format should be xlsx. xlsm. only" + + ger_obj, ger_created = Datasets.objects.get_or_create( + name="Go Electric Rebates Program", defaults={"update_user": "SYSTEM"} + ) + FileRequirements.objects.get_or_create( + dataset=ger_obj, + defaults={ + "sheet": """ + The sheet name must be "Distribution List - Master" + """, + "columns": columns_default_text, + "formats": formats_default_text, + }, + ) + + arc_obj, arc_created = Datasets.objects.get_or_create( + name="ARC Project Tracking", defaults={"update_user": "SYSTEM"} + ) + FileRequirements.objects.get_or_create( + dataset=arc_obj, + defaults={ + "sheet": """ + The sheet name must be "ARC Data" + """, + "columns": columns_default_text, + "formats": formats_default_text, + }, + ) + + cvp_obj, cvp_created = Datasets.objects.get_or_create( + name="CVP Data", defaults={"update_user": "SYSTEM"} + ) + FileRequirements.objects.get_or_create( + dataset=cvp_obj, + defaults={ + "sheet": """ + The sheet name must be "Data" + """, + "columns": columns_default_text, + "formats": formats_default_text, + }, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0039_filerequirements"), + ] + + operations = [ + migrations.RunPython(add_datasets_and_file_requirements), + ] diff --git a/django/api/models/__init__.py b/django/api/models/__init__.py index d1fc14cc..c163f3f8 100644 --- a/django/api/models/__init__.py +++ b/django/api/models/__init__.py @@ -27,3 +27,4 @@ from . import decoded_vin_record from . import regions from . import cvp_data +from . import file_requirements diff --git a/django/api/models/file_requirements.py b/django/api/models/file_requirements.py new file mode 100644 index 00000000..d9b3cbe5 --- /dev/null +++ b/django/api/models/file_requirements.py @@ -0,0 +1,20 @@ +from django.db import models +from auditable.models import Auditable +from api.models.datasets import Datasets + + +class FileRequirements(Auditable): + dataset = models.OneToOneField( + Datasets, + related_name="file_requirements", + on_delete=models.CASCADE, + ) + + sheet = models.TextField(blank=True, null=True) + + columns = models.TextField(blank=True, null=True) + + formats = models.TextField(blank=True, null=True) + + class Meta: + db_table = "file_requirements" diff --git a/django/api/serializers/file_requirements.py b/django/api/serializers/file_requirements.py new file mode 100644 index 00000000..f53b9718 --- /dev/null +++ b/django/api/serializers/file_requirements.py @@ -0,0 +1,9 @@ +from rest_framework.serializers import ModelSerializer +from api.models.file_requirements import FileRequirements + + +class FileRequirementsSerializer(ModelSerializer): + + class Meta: + model = FileRequirements + fields = ("sheet", "columns", "formats") diff --git a/django/api/services/file_requirements.py b/django/api/services/file_requirements.py new file mode 100644 index 00000000..06b029ae --- /dev/null +++ b/django/api/services/file_requirements.py @@ -0,0 +1,5 @@ +from api.models.file_requirements import FileRequirements + + +def get_file_requirements(dataset_name): + return FileRequirements.objects.filter(dataset__name=dataset_name).first() diff --git a/django/api/viewsets/upload.py b/django/api/viewsets/upload.py index 5f6d16ff..16132204 100644 --- a/django/api/viewsets/upload.py +++ b/django/api/viewsets/upload.py @@ -18,6 +18,8 @@ import api.constants.constants as constants from api.services.spreadsheet_uploader_prep import * from api.services.uploaded_vins_file import create_vins_file +from api.services.file_requirements import get_file_requirements +from api.serializers.file_requirements import FileRequirementsSerializer class UploadViewset(GenericViewSet): @@ -131,3 +133,12 @@ def download_dataset(self, request): return response except ValueError as e: return HttpResponse(str(e), status=400) + + @action(detail=False, methods=["get"]) + def file_requirements(self, request): + dataset_name = request.query_params.get("dataset") + file_requirements = get_file_requirements(dataset_name) + if file_requirements is None: + return Response({}) + serializer = FileRequirementsSerializer(file_requirements) + return Response(serializer.data) diff --git a/frontend/src/uploads/components/FileRequirements.js b/frontend/src/uploads/components/FileRequirements.js new file mode 100644 index 00000000..aa8e0834 --- /dev/null +++ b/frontend/src/uploads/components/FileRequirements.js @@ -0,0 +1,42 @@ +import React, { useState, useEffect } from "react"; +import useAxios from "../../app/utilities/useAxios"; +import ROUTES_UPLOAD from "../routes"; + +const FileRequirements = ({ datasetSelected }) => { + const axios = useAxios(); + const [requirements, setRequirements] = useState([]); + + useEffect(() => { + if (datasetSelected) { + axios + .get( + ROUTES_UPLOAD.FILE_REQUIREMENTS.replace(":dataset", datasetSelected), + ) + .then((response) => { + const list = []; + for (const [key, value] of Object.entries(response.data)) { + list.push(
  • {value}
    • ); + } + setRequirements(list); + }) + .catch((error) => { + //do something here? + }); + } + }, [datasetSelected]); + + if (requirements.length > 0) { + return ( +
    +

    File Requirements

    +
    + Ensure your file meets the following conditions before uploading: +
    +
      {requirements}
    +
    + ); + } + return null; +}; + +export default FileRequirements; diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index 3094add1..fc1b1bca 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -14,6 +14,7 @@ import UploadIcon from "@mui/icons-material/Upload"; import DownloadIcon from "@mui/icons-material/Download"; import FileDropArea from "./FileDropArea"; import Loading from "../../app/components/Loading"; +import FileRequirements from "./FileRequirements"; const UploadPage = (props) => { const { @@ -100,6 +101,11 @@ const UploadPage = (props) => { uploadFiles={uploadFiles} />
    + + + Date: Mon, 25 Nov 2024 17:13:14 -0800 Subject: [PATCH 52/67] fix: 402 - fix migration graph (#414) --- .../{0039_filerequirements.py => 0040_filerequirements.py} | 2 +- ...quirements.py => 0041_add_datasets_and_file_requirements.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename django/api/migrations/{0039_filerequirements.py => 0040_filerequirements.py} (96%) rename django/api/migrations/{0040_add_datasets_and_file_requirements.py => 0041_add_datasets_and_file_requirements.py} (97%) diff --git a/django/api/migrations/0039_filerequirements.py b/django/api/migrations/0040_filerequirements.py similarity index 96% rename from django/api/migrations/0039_filerequirements.py rename to django/api/migrations/0040_filerequirements.py index d38af6f6..72601506 100644 --- a/django/api/migrations/0039_filerequirements.py +++ b/django/api/migrations/0040_filerequirements.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ('api', '0038_addregions'), + ('api', '0039_auto_20241031_2123'), ] operations = [ diff --git a/django/api/migrations/0040_add_datasets_and_file_requirements.py b/django/api/migrations/0041_add_datasets_and_file_requirements.py similarity index 97% rename from django/api/migrations/0040_add_datasets_and_file_requirements.py rename to django/api/migrations/0041_add_datasets_and_file_requirements.py index b2ec042a..6b71d6ac 100644 --- a/django/api/migrations/0040_add_datasets_and_file_requirements.py +++ b/django/api/migrations/0041_add_datasets_and_file_requirements.py @@ -55,7 +55,7 @@ def add_datasets_and_file_requirements(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ("api", "0039_filerequirements"), + ("api", "0040_filerequirements"), ] operations = [ From 90aa0ada5401f2320655f419e4fccaa65fcb626b Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:28:18 -0800 Subject: [PATCH 53/67] feat: (#417) -changes alert and banner display for failed uploads -adds failed uploads list to the the list of files below the file drop area --- django/api/services/spreadsheet_uploader.py | 2 +- frontend/src/app/components/AlertDialog.js | 5 +- frontend/src/uploads/UploadContainer.js | 35 ++++++++--- frontend/src/uploads/components/FileDrop.js | 3 +- .../src/uploads/components/FileDropArea.js | 41 ++++++++----- .../src/uploads/components/UploadIssues.js | 60 +++++++++++++++---- frontend/src/uploads/components/UploadPage.js | 8 ++- 7 files changed, 114 insertions(+), 40 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index e6dd2285..c76059e5 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -55,7 +55,7 @@ def transform_data( if (missing_columns): errors_and_warnings['Headers'] = {} errors_and_warnings['Headers']['Missing Headers'] = { - "Expected Type": "The spreadsheet provided is missing headers", + "Expected Type": "missing one or more required columns", "Rows": missing_columns, "Severity": "Critical" } diff --git a/frontend/src/app/components/AlertDialog.js b/frontend/src/app/components/AlertDialog.js index 8a2b4cb2..c592bd27 100644 --- a/frontend/src/app/components/AlertDialog.js +++ b/frontend/src/app/components/AlertDialog.js @@ -6,8 +6,7 @@ import DialogActions from "@mui/material/DialogActions"; import DialogContent from "@mui/material/DialogContent"; import DialogContentText from "@mui/material/DialogContentText"; import DialogTitle from "@mui/material/DialogTitle"; -import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; - +import WarningAmberIcon from '@mui/icons-material/WarningAmber'; const AlertDialog = (props) => { const { open, @@ -33,7 +32,7 @@ const AlertDialog = (props) => { aria-describedby="alert-dialog-description" > - {title} + {title} diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index 1f8277d0..c5e76419 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -36,6 +36,7 @@ const UploadContainer = () => { cancelAction: () => {}, cancelText: "cancel", }); + const [failedFiles, setFailedFiles] = useState([]) const axios = useAxios(); const axiosDefault = useAxios(true); @@ -84,6 +85,8 @@ const UploadContainer = () => { const rows = errorDetails[errorType].Rows; const rowCount = rows.length; totalIssueCount.criticalErrors += rowCount; + setFailedFiles([...failedFiles, uploadFiles]) + setUploadFiles([]) if (!groupedCriticalErrors[column]) { groupedCriticalErrors[column] = {}; } @@ -138,6 +141,12 @@ const UploadContainer = () => { return { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount }; }; + const clearErrors = () => { + setGroupedCriticalErrors({}) + setGroupedErrors({}) + setGroupedWarnings({}) + setTotalIssueCount({}) + } const showError = (error) => { const { response: errorResponse } = error; @@ -183,7 +192,6 @@ const UploadContainer = () => { const errorCheck = responses.some( (response) => !response.data.success ); - setAlertSeverity(errorCheck ? "error" : "success"); const message = responses .map( @@ -193,7 +201,6 @@ const UploadContainer = () => { .join("\n"); setAlert(true); setAlertContent(message); - const warnings = {}; responses.forEach((response, index) => { const responseWarnings = response.data.errors_and_warnings; @@ -210,18 +217,27 @@ const UploadContainer = () => { setGroupedErrors(groupedErrors); setGroupedWarnings(groupedWarnings); setTotalIssueCount(totalIssueCount); - setAlertDialogText({ title: - totalIssueCount.criticalErrors > 0 ? "Your upload contained critical errors that must be fixed before it can be processed!" : "Your file has been processed and contains the following errors and warnings!", + totalIssueCount.criticalErrors > 0 ? "File upload failed" : "Your file has been processed and contains the following errors and warnings!", content: ( <> {totalIssueCount.criticalErrors >= 1 && (
    - - {totalIssueCount.criticalErrors} Critical Errors - - - Must fix before file can be processed + {groupedCriticalErrors && groupedCriticalErrors.Spreadsheet && + groupedCriticalErrors.Spreadsheet['Missing Worksheet'] && +
    + File Upload Failed - The sheet name doesn't match the required + “{groupedCriticalErrors.Spreadsheet['Missing Worksheet'].Rows[0]}”.
    + +
    + } + {groupedCriticalErrors && groupedCriticalErrors.Headers && + groupedCriticalErrors.Headers['Missing Headers'] && +
    + The file is missing one or more required columns. +
    + }
    )} {totalIssueCount.errors >= 1 && ( @@ -339,7 +355,6 @@ const UploadContainer = () => { ))} ) : null; - return (
    @@ -381,6 +396,8 @@ const UploadContainer = () => { setAlert={setAlert} loading={loading} totalIssueCount={totalIssueCount} + clearErrors={clearErrors} + failedFiles={failedFiles} /> {adminUser && ( diff --git a/frontend/src/uploads/components/FileDrop.js b/frontend/src/uploads/components/FileDrop.js index d0e4d71c..063a3548 100644 --- a/frontend/src/uploads/components/FileDrop.js +++ b/frontend/src/uploads/components/FileDrop.js @@ -5,12 +5,13 @@ import UploadIcon from "@mui/icons-material/Upload"; import { useDropzone } from "react-dropzone"; const FileDrop = (props) => { - const { disabled, setFiles, setAlert } = props; + const { disabled, setFiles, setAlert, clearErrors} = props; const [dropMessage, setDropMessage] = useState(""); const onDrop = useCallback((files) => { setAlert(false); setDropMessage(""); setFiles(files); + clearErrors(); }, []); const { getRootProps, getInputProps } = useDropzone({ onDrop }); const uploadBoxClassNames = disabled ? "file-upload disabled" : "file-upload"; diff --git a/frontend/src/uploads/components/FileDropArea.js b/frontend/src/uploads/components/FileDropArea.js index 69ef484e..eb59dbb5 100644 --- a/frontend/src/uploads/components/FileDropArea.js +++ b/frontend/src/uploads/components/FileDropArea.js @@ -6,7 +6,7 @@ import FileDrop from "./FileDrop"; import getFileSize from "../../app/utilities/getFileSize"; const FileDropArea = (props) => { - const { disabled, setUploadFiles, uploadFiles, setAlert } = props; + const { disabled, setUploadFiles, uploadFiles, setAlert, totalIssueCount, clearErrors, failedFiles } = props; const removeFile = (removedFile) => { const found = uploadFiles.findIndex((file) => file === removedFile); @@ -14,27 +14,34 @@ const FileDropArea = (props) => { setUploadFiles([...uploadFiles]); }; - function FormRow(file) { + function FormRow(file, success) { const { name, size } = file; + const uploadRowClassname = totalIssueCount.criticalErrors >= 1? 'error': success==false? 'error': 'upload-row' return ( - + {name} - + {getFileSize(size)} - + + {success == true && + } + {success == false && + <>Failed Upload + } ); @@ -57,12 +64,13 @@ const FileDropArea = (props) => { disabled={disabled} setAlert={setAlert} setFiles={setUploadFiles} + clearErrors={clearErrors} />
    - {uploadFiles.length > 0 && ( + {(uploadFiles.length > 0 || failedFiles.length > 0) && ( @@ -72,7 +80,12 @@ const FileDropArea = (props) => {

    Size

     - {uploadFiles.map((file) => FormRow(file))} + {failedFiles.map((failed, index) => { + return failed.map((file) => { + return FormRow(file, false); + }); + })} + {uploadFiles.map((file) =>FormRow(file, true))}     )} @@ -86,4 +99,4 @@ FileDropArea.propTypes = { uploadFiles: PropTypes.arrayOf(PropTypes.shape()).isRequired, setAlert: PropTypes.func.isRequired, }; -export default FileDropArea; +export default FileDropArea; \ No newline at end of file diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js index 3f4bebec..9b0e2080 100644 --- a/frontend/src/uploads/components/UploadIssues.js +++ b/frontend/src/uploads/components/UploadIssues.js @@ -27,6 +27,42 @@ const UploadIssues = ({ const criticalMsg = "Must fix before file can be processed"; const errorMsg = "Must fix before uploading"; const warningMsg = "Can upload without fixing"; + const renderUploadFailed = () => { + const missingHeadersError = groupedCriticalErrors?.Headers?.["Missing Headers"]; + let missingHeadersMsg = ''; + if (missingHeadersError) { + const missingColumns = missingHeadersError.Rows; + const columnsText = missingColumns.length === 1 + ? `column "${missingColumns[0]} is "` + : `columns "${missingColumns.join(', ')}" are `; + + missingHeadersMsg = `Your file has been processed and the ${columnsText} not found in the dataset. + Please ensure that your dataset matches the provided template, and that all required columns are present. + You can download the template for reference. Once corrected, you can upload your file again.`; + } + const missingWorksheetError = groupedCriticalErrors?.Spreadsheet?.["Missing Worksheet"]; + let missingWorksheetMsg = ''; + if (missingWorksheetError) { + const sheetName = groupedCriticalErrors.Spreadsheet['Missing Worksheet'].Rows[0] + missingWorksheetMsg = missingWorksheetError ? + `File Upload Failed - The sheet name doesn't match the required “${sheetName}”. + Please rename the sheet to the required “${sheetName}” before the next upload.` : ''; + } + const errorMsg = missingHeadersMsg || missingWorksheetMsg; + return ( + + {errorMsg} + + ) + } + + + const errorHeading = () => { + const missingHeadersError = groupedCriticalErrors?.Headers?.['Missing Headers']?.ExpectedType; + const missingWorksheetError = groupedCriticalErrors?.Spreadsheet?.['Missing Worksheet']?.ExpectedType; + return missingHeadersError || missingWorksheetError; + } + return ( <> @@ -36,19 +72,16 @@ const UploadIssues = ({ className="error" sx={{ marginLeft: 1, marginRight: 1 }} /> - Your file upload results + {totalIssueCount.criticalErrors >= 1 ? `File upload failed - ${errorHeading()}`: 'Your file upload results'}

    - - {totalIssueCount.criticalErrors > 0 ? 'Your file cannot be processed because it contains critical errors. Please review them below.': 'Your file has been processed and contains the following errors and warnings. Please review them below'} - {totalIssueCount.criticalErrors >= 1 && ( - - - {totalIssueCount.criticalErrors} Critical Errors   - - - {criticalMsg} - + renderUploadFailed() )} + {totalIssueCount.criticalErrors == 0 && + + Your file has been processed and contains the following errors and warnings. Please review them below + + } {totalIssueCount.errors >= 1 && ( @@ -65,6 +98,8 @@ const UploadIssues = ({ - {warningMsg} )} + {totalIssueCount.criticalErrors == 0 && ( + <> - {totalIssueCount.warnings >= 1 && totalIssueCount.errors === 0 && ( + + )} + { + totalIssueCount.warnings >= 1 && totalIssueCount.errors === 0 && (

    Do you want to upload the file regardless of the warnings?

    diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index fc1b1bca..4db57c7f 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -30,8 +30,11 @@ const UploadPage = (props) => { downloadSpreadsheet, setAlert, loading, - totalIssueCount + totalIssueCount, + clearErrors, + failedFiles } = props; + const selectionList = datasetList.map((obj, index) => ( {obj.name} @@ -99,6 +102,9 @@ const UploadPage = (props) => { setAlert={setAlert} setUploadFiles={setUploadFiles} uploadFiles={uploadFiles} + totalIssueCount={totalIssueCount} + clearErrors={clearErrors} + failedFiles={failedFiles} />
    From 90faae59ec22588eb41b5a4d54c7d65f6493c4c8 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:26:14 -0800 Subject: [PATCH 54/67] Adding prep function that formats postal codes and validates them if asked to do so (#418) --- django/api/constants/constants.py | 4 +- .../api/services/spreadsheet_uploader_prep.py | 39 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/django/api/constants/constants.py b/django/api/constants/constants.py index 227dab2c..ffe20391 100644 --- a/django/api/constants/constants.py +++ b/django/api/constants/constants.py @@ -26,7 +26,8 @@ location_checker, email_validator, validate_field_values, - region_checker + region_checker, + format_postal_codes ) from api.services.resolvers import get_google_resolver from api.constants.misc import GER_VALID_FIELD_VALUES, ARC_VALID_FIELD_VALUES, LOCALITY_FEATURES_MAP, CVP_DATA_VALID_FIELD_VALUES @@ -829,6 +830,7 @@ class CVPDataColumnMapping(Enum): {"function": location_checker, "columns": ["City"], "kwargs": {"columns_to_features_map": {"City": LOCALITY_FEATURES_MAP}, "indices_offset":2}}, {"function": email_validator, "columns": ["Email"], "kwargs": {"indices_offset":2, "get_resolver": get_google_resolver}}, {"function": validate_field_values, "columns": [], "kwargs": {"indices_offset":2, "fields_and_values": GER_VALID_FIELD_VALUES}}, + {"function": format_postal_codes, "columns": ["Postal code"], "kwargs": {"indices_offset":2, "validate": True}} ] }, "CVP Data": { diff --git a/django/api/services/spreadsheet_uploader_prep.py b/django/api/services/spreadsheet_uploader_prep.py index 8d8317bf..7e4db9a6 100644 --- a/django/api/services/spreadsheet_uploader_prep.py +++ b/django/api/services/spreadsheet_uploader_prep.py @@ -410,4 +410,41 @@ def region_checker(df, *columns, **kwargs): } } - return result \ No newline at end of file + return result + +def format_postal_codes(df, *columns, **kwargs): + validate = kwargs.get('validate', False) + indices_offset = kwargs.get("indices_offset", 0) + + result = {} + + for column in columns: + series = df[column] + map_of_values_to_indices = get_map_of_values_to_indices(series, indices_offset) + invalid_groups = [] + + for value, indices in map_of_values_to_indices.items(): + clean_value = value.replace(" ", "") if isinstance(value, str) else "" + + if len(clean_value) == 6: + formatted_value = clean_value[:3] + " " + clean_value[3:] + for index in indices: + df.at[index - indices_offset, column] = formatted_value + elif validate: + if pd.isna(value) or value == "": + value = "Empty" + invalid_groups.append({ + "Invalid Postal Code": value, + "Rows": indices + }) + + if validate and invalid_groups: + result[column] = { + "Invalid Postal Codes": { + "Expected Type": "Postal Code should be formatted as (XXX XXX)", + "Groups": invalid_groups, + "Severity": "Warning" + } + } + + return result if validate else None From 169fdc2521d816c74bc58997c310eb0ce596ea71 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Mon, 2 Dec 2024 02:14:26 -0800 Subject: [PATCH 55/67] CTHUB 419 - Cancel uploads with warnings/errors (#420) * task: adds code to cancel buttons * fix: adds success dialog back * chore: linting --- frontend/src/uploads/UploadContainer.js | 219 ++++++++++-------- frontend/src/uploads/components/FileDrop.js | 2 +- .../src/uploads/components/FileDropArea.js | 60 +++-- .../src/uploads/components/UploadIssues.js | 166 ++++++------- .../uploads/components/UploadIssuesDetail.js | 63 ++--- frontend/src/uploads/components/UploadPage.js | 18 +- 6 files changed, 294 insertions(+), 234 deletions(-) diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index c5e76419..cf48ad5b 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -36,7 +36,7 @@ const UploadContainer = () => { cancelAction: () => {}, cancelText: "cancel", }); - const [failedFiles, setFailedFiles] = useState([]) + const [failedFiles, setFailedFiles] = useState([]); const axios = useAxios(); const axiosDefault = useAxios(true); @@ -60,7 +60,7 @@ const UploadContainer = () => { }; const groupAndCountRows = (issueArray) => { - const groupedCriticalErrors = {} + const groupedCriticalErrors = {}; const groupedErrors = {}; const groupedWarnings = {}; const totalIssueCount = { @@ -69,14 +69,11 @@ const UploadContainer = () => { warnings: 0, }; - issueArray.forEach((issue) => { Object.keys(issue).forEach((column) => { const errorDetails = issue[column]; - Object.keys(errorDetails).forEach((errorType) => { - const severity = errorDetails[errorType].Severity; const expectedType = errorDetails[errorType]["Expected Type"]; const groups = errorDetails[errorType].Groups || []; @@ -85,8 +82,8 @@ const UploadContainer = () => { const rows = errorDetails[errorType].Rows; const rowCount = rows.length; totalIssueCount.criticalErrors += rowCount; - setFailedFiles([...failedFiles, uploadFiles]) - setUploadFiles([]) + setFailedFiles([...failedFiles, uploadFiles]); + setUploadFiles([]); if (!groupedCriticalErrors[column]) { groupedCriticalErrors[column] = {}; } @@ -102,7 +99,7 @@ const UploadContainer = () => { const rows = errorDetails[errorType].Rows || null; const rowCount = rows.length || groups.length; totalIssueCount.errors += rowCount; - + if (!groupedErrors[column]) { groupedErrors[column] = {}; } @@ -116,7 +113,7 @@ const UploadContainer = () => { } } else if (severity === "Warning") { let warningRowCount = 0; - + if (!groupedWarnings[column]) { groupedWarnings[column] = {}; } @@ -126,27 +123,31 @@ const UploadContainer = () => { Groups: [], }; } - + groups.forEach((group) => { groupedWarnings[column][errorType].Groups.push(group); warningRowCount += group.Rows.length; }); - + totalIssueCount.warnings += warningRowCount; } }); }); }); - - return { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount }; + return { + groupedCriticalErrors, + groupedErrors, + groupedWarnings, + totalIssueCount, + }; }; const clearErrors = () => { - setGroupedCriticalErrors({}) - setGroupedErrors({}) - setGroupedWarnings({}) - setTotalIssueCount({}) - } + setGroupedCriticalErrors({}); + setGroupedErrors({}); + setGroupedWarnings({}); + setTotalIssueCount({}); + }; const showError = (error) => { const { response: errorResponse } = error; @@ -189,14 +190,12 @@ const UploadContainer = () => { Promise.all(uploadPromises) .then((responses) => { - const errorCheck = responses.some( - (response) => !response.data.success - ); + const errorCheck = responses.some((response) => !response.data.success); setAlertSeverity(errorCheck ? "error" : "success"); const message = responses .map( (response) => - `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}` + `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, ) .join("\n"); setAlert(true); @@ -210,35 +209,51 @@ const UploadContainer = () => { }); if (Object.keys(warnings).length > 0 && checkForWarnings) { - const { groupedCriticalErrors, groupedErrors, groupedWarnings, totalIssueCount } = - groupAndCountRows(Object.values(warnings)); + const { + groupedCriticalErrors, + groupedErrors, + groupedWarnings, + totalIssueCount, + } = groupAndCountRows(Object.values(warnings)); - setGroupedCriticalErrors(groupedCriticalErrors) + setGroupedCriticalErrors(groupedCriticalErrors); setGroupedErrors(groupedErrors); setGroupedWarnings(groupedWarnings); setTotalIssueCount(totalIssueCount); setAlertDialogText({ title: - totalIssueCount.criticalErrors > 0 ? "File upload failed" : "Your file has been processed and contains the following errors and warnings!", + totalIssueCount.criticalErrors > 0 + ? "File upload failed" + : "Your file has been processed and contains the following errors and warnings!", content: ( <> {totalIssueCount.criticalErrors >= 1 && (
    - {groupedCriticalErrors && groupedCriticalErrors.Spreadsheet && - groupedCriticalErrors.Spreadsheet['Missing Worksheet'] && -
    - File Upload Failed - The sheet name doesn't match the required - “{groupedCriticalErrors.Spreadsheet['Missing Worksheet'].Rows[0]}”.
    - -
    - } - {groupedCriticalErrors && groupedCriticalErrors.Headers && - groupedCriticalErrors.Headers['Missing Headers'] && -
    - The file is missing one or more required columns. -
    - } -
    + {groupedCriticalErrors && + groupedCriticalErrors.Spreadsheet && + groupedCriticalErrors.Spreadsheet[ + "Missing Worksheet" + ] && ( +
    + File Upload Failed - The sheet name doesn't match the + required “ + { + groupedCriticalErrors.Spreadsheet[ + "Missing Worksheet" + ].Rows[0] + } + ”. +
    +
    + )} + {groupedCriticalErrors && + groupedCriticalErrors.Headers && + groupedCriticalErrors.Headers["Missing Headers"] && ( +
    + The file is missing one or more required columns. +
    + )} + )} {totalIssueCount.errors >= 1 && (
    @@ -258,12 +273,15 @@ const UploadContainer = () => { )} ), - cancelAction: () => setOpenDialog(false), + cancelAction: () => { + setOpenDialog(false); + clearErrors(); + setUploadFiles([]); + }, confirmText: "View Details", confirmAction: () => setOpenDialog(false), }); setOpenDialog(true); - } }) .catch((error) => { @@ -318,14 +336,14 @@ const UploadContainer = () => { }; const handleConfirmDataInsert = () => { - setGroupedWarnings({}) - setGroupedErrors({}) - setTotalIssueCount({}) + setGroupedWarnings({}); + setGroupedErrors({}); + setTotalIssueCount({}); setOpenDialog(false); setAlert(false); setAlertContent(""); doUpload(false); // Upload with the checkForWarnings flag set to false! - setUploadFiles([]) + setUploadFiles([]); }; const handleReplaceDataConfirm = () => { @@ -344,6 +362,7 @@ const UploadContainer = () => { if (refresh) { return ; } + const alertElement = alert && alertContent && alertSeverity ? ( @@ -355,61 +374,65 @@ const UploadContainer = () => { ))} ) : null; - return ( -
    -
    - <> - - - {(totalIssueCount.criticalErrors || totalIssueCount.errors > 0 || totalIssueCount.warnings > 0) && ( - - - - )} - - +
    + <> + + + {(totalIssueCount.criticalErrors || + totalIssueCount.errors > 0 || + totalIssueCount.warnings > 0) && ( + + - {adminUser && ( - - - - )} - - -
    + )} + + + + {adminUser && ( + + + + )} +     +
    - ); - }; +
    + ); +}; export default withRouter(UploadContainer); diff --git a/frontend/src/uploads/components/FileDrop.js b/frontend/src/uploads/components/FileDrop.js index 063a3548..139eebab 100644 --- a/frontend/src/uploads/components/FileDrop.js +++ b/frontend/src/uploads/components/FileDrop.js @@ -5,7 +5,7 @@ import UploadIcon from "@mui/icons-material/Upload"; import { useDropzone } from "react-dropzone"; const FileDrop = (props) => { - const { disabled, setFiles, setAlert, clearErrors} = props; + const { disabled, setFiles, setAlert, clearErrors } = props; const [dropMessage, setDropMessage] = useState(""); const onDrop = useCallback((files) => { setAlert(false); diff --git a/frontend/src/uploads/components/FileDropArea.js b/frontend/src/uploads/components/FileDropArea.js index eb59dbb5..189da83d 100644 --- a/frontend/src/uploads/components/FileDropArea.js +++ b/frontend/src/uploads/components/FileDropArea.js @@ -6,7 +6,15 @@ import FileDrop from "./FileDrop"; import getFileSize from "../../app/utilities/getFileSize"; const FileDropArea = (props) => { - const { disabled, setUploadFiles, uploadFiles, setAlert, totalIssueCount, clearErrors, failedFiles } = props; + const { + disabled, + setUploadFiles, + uploadFiles, + setAlert, + totalIssueCount, + clearErrors, + failedFiles, + } = props; const removeFile = (removedFile) => { const found = uploadFiles.findIndex((file) => file === removedFile); @@ -16,7 +24,12 @@ const FileDropArea = (props) => { function FormRow(file, success) { const { name, size } = file; - const uploadRowClassname = totalIssueCount.criticalErrors >= 1? 'error': success==false? 'error': 'upload-row' + const uploadRowClassname = + totalIssueCount.criticalErrors >= 1 + ? "error" + : success == false + ? "error" + : "upload-row"; return ( @@ -26,22 +39,20 @@ const FileDropArea = (props) => { {getFileSize(size)} - {success == true && - - } - {success == false && - <>Failed Upload - } + {success == true && ( + + )} + {success == false && <>Failed Upload} ); @@ -53,11 +64,12 @@ const FileDropArea = (props) => { { return FormRow(file, false); }); })} - {uploadFiles.map((file) =>FormRow(file, true))} + {uploadFiles.map((file) => FormRow(file, true))} )} @@ -99,4 +111,4 @@ FileDropArea.propTypes = { uploadFiles: PropTypes.arrayOf(PropTypes.shape()).isRequired, setAlert: PropTypes.func.isRequired, }; -export default FileDropArea; \ No newline at end of file +export default FileDropArea; diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js index 9b0e2080..538a5a2a 100644 --- a/frontend/src/uploads/components/UploadIssues.js +++ b/frontend/src/uploads/components/UploadIssues.js @@ -7,7 +7,7 @@ import { Accordion, Button, } from "@mui/material"; -import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import ErrorOutlineIcon from "@mui/icons-material/ErrorOutline"; import UploadIssuesDetail from "./UploadIssuesDetail"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; @@ -17,6 +17,8 @@ const UploadIssues = ({ groupedErrors, groupedWarnings, totalIssueCount, + clearErrors, + setUploadFiles, }) => { const [showAllIssues, setShowAllIssues] = useState(false); @@ -28,41 +30,46 @@ const UploadIssues = ({ const errorMsg = "Must fix before uploading"; const warningMsg = "Can upload without fixing"; const renderUploadFailed = () => { - const missingHeadersError = groupedCriticalErrors?.Headers?.["Missing Headers"]; - let missingHeadersMsg = ''; - if (missingHeadersError) { - const missingColumns = missingHeadersError.Rows; - const columnsText = missingColumns.length === 1 + const missingHeadersError = + groupedCriticalErrors?.Headers?.["Missing Headers"]; + let missingHeadersMsg = ""; + if (missingHeadersError) { + const missingColumns = missingHeadersError.Rows; + const columnsText = + missingColumns.length === 1 ? `column "${missingColumns[0]} is "` - : `columns "${missingColumns.join(', ')}" are `; - - missingHeadersMsg = `Your file has been processed and the ${columnsText} not found in the dataset. + : `columns "${missingColumns.join(", ")}" are `; + + missingHeadersMsg = `Your file has been processed and the ${columnsText} not found in the dataset. Please ensure that your dataset matches the provided template, and that all required columns are present. You can download the template for reference. Once corrected, you can upload your file again.`; - } - const missingWorksheetError = groupedCriticalErrors?.Spreadsheet?.["Missing Worksheet"]; - let missingWorksheetMsg = ''; - if (missingWorksheetError) { - const sheetName = groupedCriticalErrors.Spreadsheet['Missing Worksheet'].Rows[0] - missingWorksheetMsg = missingWorksheetError ? - `File Upload Failed - The sheet name doesn't match the required “${sheetName}”. - Please rename the sheet to the required “${sheetName}” before the next upload.` : ''; - } - const errorMsg = missingHeadersMsg || missingWorksheetMsg; + } + const missingWorksheetError = + groupedCriticalErrors?.Spreadsheet?.["Missing Worksheet"]; + let missingWorksheetMsg = ""; + if (missingWorksheetError) { + const sheetName = + groupedCriticalErrors.Spreadsheet["Missing Worksheet"].Rows[0]; + missingWorksheetMsg = missingWorksheetError + ? `File Upload Failed - The sheet name doesn't match the required “${sheetName}”. + Please rename the sheet to the required “${sheetName}” before the next upload.` + : ""; + } + const errorMsg = missingHeadersMsg || missingWorksheetMsg; return ( {errorMsg} - ) - } - + ); + }; const errorHeading = () => { - const missingHeadersError = groupedCriticalErrors?.Headers?.['Missing Headers']?.ExpectedType; - const missingWorksheetError = groupedCriticalErrors?.Spreadsheet?.['Missing Worksheet']?.ExpectedType; + const missingHeadersError = + groupedCriticalErrors?.Headers?.["Missing Headers"]?.ExpectedType; + const missingWorksheetError = + groupedCriticalErrors?.Spreadsheet?.["Missing Worksheet"]?.ExpectedType; return missingHeadersError || missingWorksheetError; - } - + }; return ( <> @@ -72,16 +79,17 @@ const UploadIssues = ({ className="error" sx={{ marginLeft: 1, marginRight: 1 }} /> - {totalIssueCount.criticalErrors >= 1 ? `File upload failed - ${errorHeading()}`: 'Your file upload results'} + {totalIssueCount.criticalErrors >= 1 + ? `File upload failed - ${errorHeading()}` + : "Your file upload results"} - {totalIssueCount.criticalErrors >= 1 && ( - renderUploadFailed() - )} - {totalIssueCount.criticalErrors == 0 && + {totalIssueCount.criticalErrors >= 1 && renderUploadFailed()} + {totalIssueCount.criticalErrors == 0 && ( - Your file has been processed and contains the following errors and warnings. Please review them below + Your file has been processed and contains the following errors and + warnings. Please review them below - } + )} {totalIssueCount.errors >= 1 && ( @@ -100,59 +108,59 @@ const UploadIssues = ({ )} {totalIssueCount.criticalErrors == 0 && ( <> - - - - {showAllIssues ? "Show less" : "Show more"} - - - - - {totalIssueCount.criticalErrors >= 1 && ( - - )} - {totalIssueCount.errors >= 1 && ( - - )} - {totalIssueCount.warnings >= 1 && ( - - )} - - - + + + + {showAllIssues ? "Show less" : "Show more"} + + + + + {totalIssueCount.criticalErrors >= 1 && ( + + )} + {totalIssueCount.errors >= 1 && ( + + )} + {totalIssueCount.warnings >= 1 && ( + + )} + + + )} - { - totalIssueCount.warnings >= 1 && totalIssueCount.errors === 0 && ( + {totalIssueCount.warnings >= 1 && totalIssueCount.errors === 0 && (

    Do you want to upload the file regardless of the warnings?

    - )} -
    - )) - ) : ( - renderError(errorDetails) - )} + {errorDetails.Groups + ? errorDetails.Groups.map((group, groupIndex) => ( +
    + {renderWarning(group)} + {group.Rows.length > 15 && ( + + )} +
    + )) + : renderError(errorDetails)} ); })} diff --git a/frontend/src/uploads/components/UploadPage.js b/frontend/src/uploads/components/UploadPage.js index 4db57c7f..6dedd1cd 100644 --- a/frontend/src/uploads/components/UploadPage.js +++ b/frontend/src/uploads/components/UploadPage.js @@ -32,7 +32,7 @@ const UploadPage = (props) => { loading, totalIssueCount, clearErrors, - failedFiles + failedFiles, } = props; const selectionList = datasetList.map((obj, index) => ( @@ -40,11 +40,19 @@ const UploadPage = (props) => { {obj.name} )); + + const noIssues = (totalIssueCount) => { + return ( + Object.keys(totalIssueCount).length === 0 && + totalIssueCount.constructor === Object + ); + }; + return ( <>

    Upload Program Data

    - {totalIssueCount < 1 && alertElement} + {noIssues && alertElement}

    Select Program    

    @@ -65,7 +73,7 @@ const UploadPage = (props) => { type="button" variant="contained" onClick={downloadSpreadsheet} - sx={{ ml: 2 }} + sx={{ ml: 2 }} > Download Dataset Template @@ -108,9 +116,7 @@ const UploadPage = (props) => { />
    - + Date: Tue, 3 Dec 2024 10:37:26 -0800 Subject: [PATCH 56/67] Removing value checking on vehicle type column, now it only checks if a value exists (#422) --- django/api/constants/misc.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/django/api/constants/misc.py b/django/api/constants/misc.py index c5cf5af8..5ede204a 100644 --- a/django/api/constants/misc.py +++ b/django/api/constants/misc.py @@ -103,12 +103,6 @@ 'Mainland/Southwest', 'Thompson/Okanagan', 'Kootenay', 'Across BC' ], 'Drive Type': ['BEV', 'FC', 'PHEV'], - 'Vehicle Type': [ - 'On-Road', 'Loader', 'Excavator', 'Forklift', 'Outboard Motor', - 'Tugboat', 'Passenger Ferry', 'Ice Resurfacer', 'Locomotive', - 'Rail Maintenance', 'Rubber-tired Gantry Crane', 'Terminal/Yard truck', - 'Aircraft', 'Jet Fuel Pumper', 'Train Mover' - ], 'Project Type': [ 'Procurement', 'New Design', 'Hybrid Retrofit', 'BEV Retrofit', 'H2 Retrofit' ] From bceef60e5a009cbade7ab33edc9c20706031108a Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Thu, 12 Dec 2024 16:50:34 -0800 Subject: [PATCH 57/67] apply artifact pull secret --- openshift/templates/backend/backend-bc.yaml | 4 ---- .../frontend/frontend-bc-docker.yaml | 19 +++------------ .../templates/task-queue/task-queue-bc.yaml | 17 ++----------- openshift/templates/vinpower/vinpower-bc.yaml | 24 ++++--------------- 4 files changed, 10 insertions(+), 54 deletions(-) diff --git a/openshift/templates/backend/backend-bc.yaml b/openshift/templates/backend/backend-bc.yaml index 5299ee8b..64c4b91e 100644 --- a/openshift/templates/backend/backend-bc.yaml +++ b/openshift/templates/backend/backend-bc.yaml @@ -90,9 +90,5 @@ objects: forcePull: true noCache: true type: Source - triggers: - - imageChange: {} - type: ImageChange - - type: ConfigChange status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/frontend/frontend-bc-docker.yaml b/openshift/templates/frontend/frontend-bc-docker.yaml index 4a1146d2..9665bc6c 100644 --- a/openshift/templates/frontend/frontend-bc-docker.yaml +++ b/openshift/templates/frontend/frontend-bc-docker.yaml @@ -69,24 +69,11 @@ objects: strategy: dockerStrategy: dockerfilePath: ./Dockerfile-Openshift - env: - - name: ARTIFACTORY_USER - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: username - - name: ARTIFACTORY_PASSWORD - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: password + pullSecret: + name: artifacts-pull-default-idxprm noCache: true forcePull: true type: Docker - successfulBuildsHistoryLimit: 5 - triggers: - - imageChange: {} - type: ImageChange - - type: ConfigChange + successfulBuildsHistoryLimit: 5 status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/task-queue/task-queue-bc.yaml b/openshift/templates/task-queue/task-queue-bc.yaml index 668c3f22..9dfe8d42 100644 --- a/openshift/templates/task-queue/task-queue-bc.yaml +++ b/openshift/templates/task-queue/task-queue-bc.yaml @@ -69,24 +69,11 @@ objects: strategy: dockerStrategy: dockerfilePath: ./Dockerfile.taskq.Openshift - env: - - name: ARTIFACTORY_USER - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: username - - name: ARTIFACTORY_PASSWORD - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: password + pullSecret: + name: artifacts-pull-default-idxprm noCache: true forcePull: true type: Docker successfulBuildsHistoryLimit: 5 - triggers: - - imageChange: {} - type: ImageChange - - type: ConfigChange status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/vinpower/vinpower-bc.yaml b/openshift/templates/vinpower/vinpower-bc.yaml index 68c2e94d..a5d029cf 100644 --- a/openshift/templates/vinpower/vinpower-bc.yaml +++ b/openshift/templates/vinpower/vinpower-bc.yaml @@ -67,23 +67,9 @@ objects: strategy: dockerStrategy: dockerfilePath: ./Dockerfile-Openshift - env: - - name: ARTIFACTORY_USER - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: username - - name: ARTIFACTORY_PASSWORD - valueFrom: - secretKeyRef: - name: artifacts-default-idxprm - key: password - - name: ARTIFACTORY_URL - valueFrom: - secretKeyRef: - name: cthub-artifactory - key: ARTIFACTORY_URL + pullSecret: + name: artifacts-pull-default-idxprm + noCache: true + forcePull: true type: Docker - triggers: - - type: ConfigChange - - type: ImageChange + From be5287451dffcfb817cc42fa7b68ed0b655e32c9 Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Thu, 12 Dec 2024 16:59:22 -0800 Subject: [PATCH 58/67] rollback trigger --- openshift/templates/backend/backend-bc.yaml | 4 ++++ openshift/templates/frontend/frontend-bc-docker.yaml | 4 ++++ openshift/templates/task-queue/task-queue-bc.yaml | 4 ++++ openshift/templates/vinpower/vinpower-bc.yaml | 5 ++++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/openshift/templates/backend/backend-bc.yaml b/openshift/templates/backend/backend-bc.yaml index 64c4b91e..5299ee8b 100644 --- a/openshift/templates/backend/backend-bc.yaml +++ b/openshift/templates/backend/backend-bc.yaml @@ -90,5 +90,9 @@ objects: forcePull: true noCache: true type: Source + triggers: + - imageChange: {} + type: ImageChange + - type: ConfigChange status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/frontend/frontend-bc-docker.yaml b/openshift/templates/frontend/frontend-bc-docker.yaml index 9665bc6c..50bd9b25 100644 --- a/openshift/templates/frontend/frontend-bc-docker.yaml +++ b/openshift/templates/frontend/frontend-bc-docker.yaml @@ -75,5 +75,9 @@ objects: forcePull: true type: Docker successfulBuildsHistoryLimit: 5 + triggers: + - imageChange: {} + type: ImageChange + - type: ConfigChange status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/task-queue/task-queue-bc.yaml b/openshift/templates/task-queue/task-queue-bc.yaml index 9dfe8d42..6fde26cb 100644 --- a/openshift/templates/task-queue/task-queue-bc.yaml +++ b/openshift/templates/task-queue/task-queue-bc.yaml @@ -75,5 +75,9 @@ objects: forcePull: true type: Docker successfulBuildsHistoryLimit: 5 + triggers: + - imageChange: {} + type: ImageChange + - type: ConfigChange status: lastVersion: 0 \ No newline at end of file diff --git a/openshift/templates/vinpower/vinpower-bc.yaml b/openshift/templates/vinpower/vinpower-bc.yaml index a5d029cf..496208fb 100644 --- a/openshift/templates/vinpower/vinpower-bc.yaml +++ b/openshift/templates/vinpower/vinpower-bc.yaml @@ -72,4 +72,7 @@ objects: noCache: true forcePull: true type: Docker - + triggers: + - imageChange: {} + type: ImageChange + - type: ConfigChange From a12b797728379bde5f5685131720f6a96be1b4af Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Thu, 12 Dec 2024 17:21:13 -0800 Subject: [PATCH 59/67] roll back vinpower build --- openshift/templates/vinpower/vinpower-bc.yaml | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/openshift/templates/vinpower/vinpower-bc.yaml b/openshift/templates/vinpower/vinpower-bc.yaml index 496208fb..68c2e94d 100644 --- a/openshift/templates/vinpower/vinpower-bc.yaml +++ b/openshift/templates/vinpower/vinpower-bc.yaml @@ -67,12 +67,23 @@ objects: strategy: dockerStrategy: dockerfilePath: ./Dockerfile-Openshift - pullSecret: - name: artifacts-pull-default-idxprm - noCache: true - forcePull: true + env: + - name: ARTIFACTORY_USER + valueFrom: + secretKeyRef: + name: artifacts-default-idxprm + key: username + - name: ARTIFACTORY_PASSWORD + valueFrom: + secretKeyRef: + name: artifacts-default-idxprm + key: password + - name: ARTIFACTORY_URL + valueFrom: + secretKeyRef: + name: cthub-artifactory + key: ARTIFACTORY_URL type: Docker - triggers: - - imageChange: {} - type: ImageChange - - type: ConfigChange + triggers: + - type: ConfigChange + - type: ImageChange From 21a16f7ccafba91f6039cc75ef3564cb26dcc68f Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Mon, 16 Dec 2024 16:54:33 -0800 Subject: [PATCH 60/67] install and cache oc in pipelines --- .github/workflows/dev-ci.yaml | 33 ++++++++++++++++++++++++++++ .github/workflows/prod-ci.yaml | 39 +++++++++++++++++++++++++++++++++- .github/workflows/test-ci.yaml | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev-ci.yaml b/.github/workflows/dev-ci.yaml index 0a8285b6..9c846cb1 100644 --- a/.github/workflows/dev-ci.yaml +++ b/.github/workflows/dev-ci.yaml @@ -23,9 +23,36 @@ concurrency: jobs: + install-oc: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - name: Check out repository + uses: actions/checkout@v4.1.1 + + - name: Set up cache for OpenShift CLI + id: cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc # Path where the `oc` binary will be installed + key: oc-cli-${{ runner.os }} + + - name: Install OpenShift CLI (if not cached) + if: steps.cache.outputs.cache-hit != 'true' + run: | + curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz + tar -xvf openshift-client-linux.tar.gz + sudo mv oc /usr/local/bin/ + oc version --client + + - name: Confirm OpenShift CLI is Available + run: oc version --client + set-pre-release: name: Calculate pre-release number runs-on: ubuntu-latest + needs: [install-oc] outputs: output1: ${{ steps.set-pre-release.outputs.PRE_RELEASE }} @@ -49,6 +76,12 @@ jobs: - name: Check out repository uses: actions/checkout@v4.1.1 + - name: Restore oc command from Cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc + key: oc-cli-${{ runner.os }} + - name: Log in to Openshift uses: redhat-actions/oc-login@v1.3 with: diff --git a/.github/workflows/prod-ci.yaml b/.github/workflows/prod-ci.yaml index d73e41a8..590fa91e 100644 --- a/.github/workflows/prod-ci.yaml +++ b/.github/workflows/prod-ci.yaml @@ -16,15 +16,46 @@ concurrency: cancel-in-progress: true jobs: + install-oc: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - name: Check out repository + uses: actions/checkout@v4.1.1 + + - name: Set up cache for OpenShift CLI + id: cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc # Path where the `oc` binary will be installed + key: oc-cli-${{ runner.os }} + + - name: Install OpenShift CLI (if not cached) + if: steps.cache.outputs.cache-hit != 'true' + run: | + curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz + tar -xvf openshift-client-linux.tar.gz + sudo mv oc /usr/local/bin/ + oc version --client + + - name: Confirm OpenShift CLI is Available + run: oc version --client set-pre-release: name: Find Test deployment pre-release number runs-on: ubuntu-latest + needs: [install-oc] outputs: output1: ${{ steps.set-pre-release.outputs.PRE_RELEASE }} steps: + - name: Restore oc command from Cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc + key: oc-cli-${{ runner.os }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.3 @@ -55,7 +86,13 @@ jobs: secret: ${{ github.TOKEN }} approvers: emi-hi,kuanfandevops,tim738745,JulianForeman minimum-approvals: 2 - issue-title: "CTHUB release-${{ env.VERSION }}-${{ env.PRE_RELEASE }} PRODUCTION Deployment" + issue-title: "CTHUB release-${{ env.VERSION }}-${{ env.PRE_RELEASE }} PRODUCTION Deployment" + + - name: Restore oc command from Cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc + key: oc-cli-${{ runner.os }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.3 diff --git a/.github/workflows/test-ci.yaml b/.github/workflows/test-ci.yaml index 29bdad84..29161e48 100644 --- a/.github/workflows/test-ci.yaml +++ b/.github/workflows/test-ci.yaml @@ -16,15 +16,46 @@ concurrency: cancel-in-progress: true jobs: + install-oc: + runs-on: ubuntu-latest + outputs: + cache-hit: ${{ steps.cache.outputs.cache-hit }} + steps: + - name: Check out repository + uses: actions/checkout@v4.1.1 + + - name: Set up cache for OpenShift CLI + id: cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc # Path where the `oc` binary will be installed + key: oc-cli-${{ runner.os }} + + - name: Install OpenShift CLI (if not cached) + if: steps.cache.outputs.cache-hit != 'true' + run: | + curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz + tar -xvf openshift-client-linux.tar.gz + sudo mv oc /usr/local/bin/ + oc version --client + + - name: Confirm OpenShift CLI is Available + run: oc version --client set-pre-release: name: Find Dev deployment pre-release number runs-on: ubuntu-latest + needs: [install-oc] outputs: output1: ${{ steps.set-pre-release.outputs.PRE_RELEASE }} steps: + - name: Restore oc command from Cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc + key: oc-cli-${{ runner.os }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.3 @@ -56,6 +87,12 @@ jobs: approvers: emi-hi,kuanfandevops,tim738745,JulianForeman minimum-approvals: 1 issue-title: "CTHUB release-${{ env.VERSION }}-${{ env.PRE_RELEASE }} Test Deployment" + + - name: Restore oc command from Cache + uses: actions/cache@v4.2.0 + with: + path: /usr/local/bin/oc + key: oc-cli-${{ runner.os }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.3 From 7ad80b2397a220112e1602dc68c632494c7edf15 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:34:15 -0800 Subject: [PATCH 61/67] Task: Numeric and Datetime Errors #415 (#426) * Fixing type error checking * Removing import variable casting and redundant try except block --- django/api/services/spreadsheet_uploader.py | 96 +++++++++------------ 1 file changed, 43 insertions(+), 53 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index c76059e5..f8292b38 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -3,7 +3,7 @@ import traceback import numpy as np from django.db import transaction -from datetime import datetime +from datetime import datetime, date def get_field_default(model, field): field = model._meta.get_field(field) @@ -52,10 +52,10 @@ def transform_data( errors_and_warnings = {} missing_columns = [col for col in required_columns if col not in df.columns] - if (missing_columns): + if missing_columns: errors_and_warnings['Headers'] = {} errors_and_warnings['Headers']['Missing Headers'] = { - "Expected Type": "missing one or more required columns", + "Expected Type": "Missing one or more required columns", "Rows": missing_columns, "Severity": "Critical" } @@ -73,7 +73,7 @@ def transform_data( float: "Float", Decimal: "Decimal", str: "String", - datetime: "Date (YYYY-MM-DD)" + datetime.date: "Date (YYYY-MM-DD)" } df = df.replace({np.nan: None}) @@ -86,7 +86,7 @@ def transform_data( if db_field_name: is_nullable = db_field_name in nullable_fields - expected_type = field_types.get(column) + expected_type = field_types.get(db_field_name) if pd.isna(value) or value == "" or value is None: if is_nullable: @@ -101,52 +101,42 @@ def transform_data( "Severity": "Error" } errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) - - if expected_type == datetime and value is not None and value != '': - try: - datetime.strptime(value, "%Y-%m-%d") - except ValueError: - if column not in errors_and_warnings: - errors_and_warnings[column] = {} - if "Incorrect Date Format" not in errors_and_warnings[column]: - errors_and_warnings[column]["Incorrect Date Format"] = { - "Expected Type": "The following rows contained an incorrect date format. Expected YYYY-MM-DD.", - "Rows": [], - "Severity": "Error" - } - errors_and_warnings[column]["Incorrect Date Format"]["Rows"].append(index + 1) - - if expected_type in [int, float, Decimal] and value is not None and pd.notna(value) and value != '': - value = str(value).replace(',', '').strip() - try: - if expected_type == int: - row_dict[column] = int(float(value)) - elif expected_type == Decimal: - row_dict[column] = Decimal(value).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) - else: - row_dict[column] = float(value) - except ValueError: - if column not in errors_and_warnings: - errors_and_warnings[column] = {} - if "Incorrect Type" not in errors_and_warnings[column]: - errors_and_warnings[column]["Incorrect Type"] = { - "Expected Type": f"The following rows contained types for the column {column}. Expected {type_to_string.get(expected_type, str(expected_type))}", - "Rows": [], - "Severity": "Error" - } - errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) - - # Check if expected_type is valid before using isinstance - elif expected_type is not None and isinstance(expected_type, type) and not isinstance(row_dict[column], expected_type) and value != "": - if column not in errors_and_warnings: - errors_and_warnings[column] = {} - if "Incorrect Type" not in errors_and_warnings[column]: - errors_and_warnings[column]["Incorrect Type"] = { - "Expected Type": f"The following rows contained types for the column {column}. Expected {type_to_string.get(expected_type, str(expected_type))}", - "Rows": [], - "Severity": "Error" - } - errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) + else: + if expected_type: + try: + if expected_type == int: + row_dict[column] = int(float(value)) + elif expected_type == float: + row_dict[column] = float(value) + elif expected_type == Decimal: + row_dict[column] = Decimal(value).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + elif expected_type == date: + if isinstance(value, datetime): + parsed_date = value.date() + else: + parsed_date = datetime.strptime(value, "%Y-%m-%d").date() + row_dict[column] = parsed_date + elif expected_type == str and type(value) == bool: + row_dict[column] = str(value) + except (ValueError, TypeError): + if column not in errors_and_warnings: + errors_and_warnings[column] = {} + if "Incorrect Type" not in errors_and_warnings[column]: + if expected_type == date: + errors_and_warnings[column]["Incorrect Type"] = { + "Expected Type": "Date in the format YYYY-MM-DD", + "Rows": [], + "Severity": "Error" + } + else: + errors_and_warnings[column]["Incorrect Type"] = { + "Expected Type": f"Expected {type_to_string.get(expected_type, str(expected_type))}", + "Rows": [], + "Severity": "Error" + } + errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) for x in validation_functions: validate = x["function"] @@ -160,7 +150,7 @@ def transform_data( errors_and_warnings[column] = {} for issue, details in issues.items(): if issue not in errors_and_warnings[column]: - if(details.get("Severity", "Error") == 'Warning'): + if details.get("Severity", "Error") == 'Warning': errors_and_warnings[column][issue] = { "Expected Type": details.get("Expected Type", "Unknown"), "Groups": details.get("Groups", []), @@ -170,7 +160,7 @@ def transform_data( errors_and_warnings[column][issue] = { "Expected Type": details.get("Expected Type", "Unknown"), "Rows": details.get("Rows", []), - "Severity": details.get("Severity", "Error") + "Severity": "Error" } else: errors_and_warnings[column][issue]["Groups"].extend(details.get("Groups", [])) From 774c44cc384242dc8ac9d820d133e8bbf234290f Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:37:28 -0800 Subject: [PATCH 62/67] Adding popup dialog to delete selected dataset, added optional styling to confirmation button in AlertDialog component (#429) --- frontend/src/app/components/AlertDialog.js | 2 + .../src/uploads/components/FileDropArea.js | 89 ++++++++++++++----- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/frontend/src/app/components/AlertDialog.js b/frontend/src/app/components/AlertDialog.js index c592bd27..e5e6aece 100644 --- a/frontend/src/app/components/AlertDialog.js +++ b/frontend/src/app/components/AlertDialog.js @@ -16,6 +16,7 @@ const AlertDialog = (props) => { handleCancel, confirmText, handleConfirm, + confirmButtonStyle } = props; if (!open) { @@ -53,6 +54,7 @@ const AlertDialog = (props) => { handleConfirm(); }} autoFocus + sx={confirmButtonStyle} > {confirmText} diff --git a/frontend/src/uploads/components/FileDropArea.js b/frontend/src/uploads/components/FileDropArea.js index 189da83d..a876ffc5 100644 --- a/frontend/src/uploads/components/FileDropArea.js +++ b/frontend/src/uploads/components/FileDropArea.js @@ -1,9 +1,10 @@ -import React from "react"; +import React, { useState } from "react"; import PropTypes from "prop-types"; import { Box, Button, Grid, Tooltip } from "@mui/material"; import ClearIcon from "@mui/icons-material/Clear"; import FileDrop from "./FileDrop"; import getFileSize from "../../app/utilities/getFileSize"; +import AlertDialog from "../../app/components/AlertDialog" const FileDropArea = (props) => { const { @@ -22,6 +23,29 @@ const FileDropArea = (props) => { setUploadFiles([...uploadFiles]); }; + const [fileToDelete, setFileToDelete] = useState(null); + const [openDialog, setOpenDialog] = useState(false); + + const handleDeleteConfirm = () => { + + removeFile(fileToDelete) + + clearErrors(); + setFileToDelete(null); + + setOpenDialog(false); + }; + + const handleDeleteCancel = () => { + setFileToDelete(null); + setOpenDialog(false); + }; + + const handleDeleteClick = (file) => { + setFileToDelete(file); + setOpenDialog(true); + }; + function FormRow(file, success) { const { name, size } = file; const uploadRowClassname = @@ -31,30 +55,47 @@ const FileDropArea = (props) => { ? "error" : "upload-row"; return ( - - - {name} - - - {getFileSize(size)} - - - {success == true && ( - - )} - {success == false && <>Failed Upload} + <> + + + + {name} + + + {getFileSize(size)} + + + {success == true && ( + + )} + {success == false && <>Failed Upload} + - + ); } return ( From a265b4045fd5eb2953d509578111aa8bcf55c167 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:57:31 -0800 Subject: [PATCH 63/67] fix: adds an extra 1 row to the row index to account for starting at 0 (#430) --- django/api/services/spreadsheet_uploader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index f8292b38..f7e672ca 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -100,7 +100,7 @@ def transform_data( "Rows": [], "Severity": "Error" } - errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 1) + errors_and_warnings[column]["Empty Value"]["Rows"].append(index + 2) else: if expected_type: try: @@ -136,7 +136,7 @@ def transform_data( "Rows": [], "Severity": "Error" } - errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 1) + errors_and_warnings[column]["Incorrect Type"]["Rows"].append(index + 2) for x in validation_functions: validate = x["function"] From cc253286858590dac7ba40d7c154a42e0f5fbe92 Mon Sep 17 00:00:00 2001 From: Kuan Fan Date: Wed, 8 Jan 2025 14:04:42 -0800 Subject: [PATCH 64/67] update frontend builder to node 20.18.1 --- frontend/Dockerfile-Openshift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/Dockerfile-Openshift b/frontend/Dockerfile-Openshift index 13f50d19..c8286538 100644 --- a/frontend/Dockerfile-Openshift +++ b/frontend/Dockerfile-Openshift @@ -6,7 +6,7 @@ # && npm install --omit=dev \ # && npm install -D webpack webpack-cli # RUN yes | npm run dist -FROM artifacts.developer.gov.bc.ca/docker-remote/node:20 as builder +FROM artifacts.developer.gov.bc.ca/docker-remote/node:20.18.1 as builder WORKDIR /usr/src/app COPY ./ ./ # RUN npm install -g npm@latest doesn't work for node 16 From a3abc1d3e8f3dd85cc86e3d954ebe83b797a5ec9 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:11:38 -0800 Subject: [PATCH 65/67] feat: 395 - download issues (#431) * feat: 395 - download issues * small change --- frontend/package.json | 3 +- .../src/uploads/components/UploadIssues.js | 34 ++++++++++++- .../src/uploads/utilities/downloadIssues.js | 48 +++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 frontend/src/uploads/utilities/downloadIssues.js diff --git a/frontend/package.json b/frontend/package.json index 868ba3bc..fdfce0df 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -28,7 +28,8 @@ "regenerator-runtime": "^0.13.9", "web-vitals": "^2.1.4", "stream-browserify": "^3.0.0", - "util": "^0.12.4" + "util": "^0.12.4", + "xlsx": "^0.18.5" }, "devDependencies": { "css-loader": "^5.2.7", diff --git a/frontend/src/uploads/components/UploadIssues.js b/frontend/src/uploads/components/UploadIssues.js index 538a5a2a..9b91b906 100644 --- a/frontend/src/uploads/components/UploadIssues.js +++ b/frontend/src/uploads/components/UploadIssues.js @@ -10,6 +10,9 @@ import { import ErrorOutlineIcon from "@mui/icons-material/ErrorOutline"; import UploadIssuesDetail from "./UploadIssuesDetail"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import DownloadIcon from "@mui/icons-material/Download"; +import getSpreadsheetRows from "../utilities/downloadIssues"; +import { writeFile, utils } from "xlsx"; const UploadIssues = ({ confirmUpload, @@ -26,6 +29,21 @@ const UploadIssues = ({ setShowAllIssues(!showAllIssues); }; + const handleDownloadIssues = () => { + const workbook = utils.book_new(); + const errors = getSpreadsheetRows(groupedErrors); + const warnings = getSpreadsheetRows(groupedWarnings); + if (errors.length > 0) { + const errorsSheet = utils.json_to_sheet(errors); + utils.book_append_sheet(workbook, errorsSheet, "Errors"); + } + if (warnings.length > 0) { + const warningsSheet = utils.json_to_sheet(warnings); + utils.book_append_sheet(workbook, warningsSheet, "Warnings"); + } + writeFile(workbook, "errors_and_warnings.xlsx"); + }; + const criticalMsg = "Must fix before file can be processed"; const errorMsg = "Must fix before uploading"; const warningMsg = "Can upload without fixing"; @@ -84,7 +102,7 @@ const UploadIssues = ({ : "Your file upload results"} {totalIssueCount.criticalErrors >= 1 && renderUploadFailed()} - {totalIssueCount.criticalErrors == 0 && ( + {totalIssueCount.criticalErrors === 0 && ( Your file has been processed and contains the following errors and warnings. Please review them below @@ -106,7 +124,19 @@ const UploadIssues = ({ - {warningMsg} )} - {totalIssueCount.criticalErrors == 0 && ( + {(totalIssueCount.errors >= 1 || totalIssueCount.warnings >= 1) && ( + + + + )} + {totalIssueCount.criticalErrors === 0 && ( <> { + const result = []; + if (issues) { + for (const [columnName, value] of Object.entries(issues)) { + for (const [issueName, innerValue] of Object.entries(value)) { + const expectedValue = + innerValue.ExpectedType || innerValue.ExpectedFormat; + const rows = innerValue.Rows; + const groups = innerValue.Groups; + const [isGroup, items] = rows ? [false, [rows]] : [true, groups]; + for (const item of items) { + result.push({ + "Column Name": columnName, + Issue: issueName, + "Expected Value": expectedValue, + Rows: isGroup ? concatGroup(item) : concatRow(item), + }); + } + } + } + } + return result; +}; + +const concatGroup = (group) => { + let result = ""; + if (group) { + if (group.Rows) { + result = group.Rows.join(", "); + for (const groupKey of Object.keys(group)) { + if (groupKey !== "Rows") { + result = result + " - " + group[groupKey]; + } + } + } + } + return result; +}; + +const concatRow = (row) => { + let result = ""; + if (row) { + result = row.join(", "); + } + return result; +}; + +export default getSpreadsheetRows; From ed5f1b8016c66367cb16a51a1c44cb12ff16a9b7 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:14:45 -0800 Subject: [PATCH 66/67] Making error for expected integer type read as expected numeric for clarity (#434) --- django/api/services/spreadsheet_uploader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django/api/services/spreadsheet_uploader.py b/django/api/services/spreadsheet_uploader.py index f7e672ca..345f35f8 100644 --- a/django/api/services/spreadsheet_uploader.py +++ b/django/api/services/spreadsheet_uploader.py @@ -130,6 +130,12 @@ def transform_data( "Rows": [], "Severity": "Error" } + elif expected_type == int: + errors_and_warnings[column]["Incorrect Type"] = { + "Expected Type": "Expected numeric", + "Rows": [], + "Severity": "Error" + } else: errors_and_warnings[column]["Incorrect Type"] = { "Expected Type": f"Expected {type_to_string.get(expected_type, str(expected_type))}", From 7ad1de1f68653433d6a887d388aabb0a88cbf549 Mon Sep 17 00:00:00 2001 From: JulianForeman <71847719+JulianForeman@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:27:52 -0800 Subject: [PATCH 67/67] Making alert message only appear on successful upload (#435) --- frontend/src/uploads/UploadContainer.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/uploads/UploadContainer.js b/frontend/src/uploads/UploadContainer.js index cf48ad5b..f2e37e59 100644 --- a/frontend/src/uploads/UploadContainer.js +++ b/frontend/src/uploads/UploadContainer.js @@ -198,8 +198,11 @@ const UploadContainer = () => { `${response.data.message}${response.data.errors ? "\nErrors: " + response.data.errors.join("\n") : ""}`, ) .join("\n"); - setAlert(true); - setAlertContent(message); + + if(!errorCheck && responses.some((response) => !response.data.warning)){ + setAlert(true); + setAlertContent(message); + } const warnings = {}; responses.forEach((response, index) => { const responseWarnings = response.data.errors_and_warnings;