From dfab6dd272cb5419d97218828c8b326733c70d26 Mon Sep 17 00:00:00 2001 From: Emily <44536222+emi-hi@users.noreply.github.com> Date: Fri, 9 Feb 2024 09:51:26 -0800 Subject: [PATCH] feat: adds error handling for remainder of scripts (#153) --- django/api/services/arc_project_tracking.py | 2 +- django/api/services/charger_rebates.py | 2 -- django/api/services/data_fleets.py | 5 +++-- django/api/services/hydrogen_fleets.py | 11 ++++++----- django/api/services/ldv_rebates.py | 10 +++++----- django/api/services/public_charging.py | 15 ++++++--------- django/api/services/scrap_it.py | 13 +++++++------ .../services/speciality_use_vehicle_incentives.py | 11 ++++++----- django/api/viewsets/upload.py | 7 ++++--- 9 files changed, 38 insertions(+), 38 deletions(-) diff --git a/django/api/services/arc_project_tracking.py b/django/api/services/arc_project_tracking.py index 99cf1271..a45a221b 100644 --- a/django/api/services/arc_project_tracking.py +++ b/django/api/services/arc_project_tracking.py @@ -51,8 +51,8 @@ def import_from_xls(excel_file): try: for _, row in df.iterrows(): - if row["Publicly Announced"] == '': continue # Skip rows without this field row_count += 1 + if row["Publicly Announced"] == '': continue # Skip rows without this field ARCProjectTracking.objects.create( funding_call=row["Funding Call"], proponent=row["Proponent"], diff --git a/django/api/services/charger_rebates.py b/django/api/services/charger_rebates.py index a3170d80..189ddcf7 100644 --- a/django/api/services/charger_rebates.py +++ b/django/api/services/charger_rebates.py @@ -28,8 +28,6 @@ def import_from_xls(excel_file): ]), axis=1, inplace=True) df = trim_all_columns(df) df = df.applymap(lambda s: s.upper() if type(s) == str else s) - - # df.fillna('') df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) try: for _, row in df.iterrows(): diff --git a/django/api/services/data_fleets.py b/django/api/services/data_fleets.py index 5dbd4e88..d416c6c3 100644 --- a/django/api/services/data_fleets.py +++ b/django/api/services/data_fleets.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 1 df = pd.read_excel(excel_file, 'Data Fleets') df.drop(df.columns.difference([ "Current Stage", @@ -47,6 +48,7 @@ def import_from_xls(excel_file): df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) for _, row in df.iterrows(): + row_count += 1 try: DataFleets.objects.create( current_stage=row["Current Stage"], @@ -77,6 +79,5 @@ def import_from_xls(excel_file): potential_rebate=row["Potential Rebate"] ) except Exception as error: - print(error) - print(row) + return (error,'data',row_count) return True diff --git a/django/api/services/hydrogen_fleets.py b/django/api/services/hydrogen_fleets.py index f6955e20..874060b6 100644 --- a/django/api/services/hydrogen_fleets.py +++ b/django/api/services/hydrogen_fleets.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 1 df = pd.read_excel(excel_file, 'Fleets') df.drop(df.columns.difference([ "Application #", @@ -32,8 +33,9 @@ def import_from_xls(excel_file): df = trim_all_columns(df) df = df.applymap(lambda s: s.upper() if type(s) == str else s) df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): + row_count += 1 HydrogenFleets.objects.create( application_number=row["Application #"], fleet_number=row["Fleet #"], @@ -51,7 +53,6 @@ def import_from_xls(excel_file): dealer_name=row["Dealer Name"], rebate_amount=row["Rebate Amount"] ) - except Exception as error: - print("error: ", error) - print(row) + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/services/ldv_rebates.py b/django/api/services/ldv_rebates.py index 70434f28..e7e9bcd9 100644 --- a/django/api/services/ldv_rebates.py +++ b/django/api/services/ldv_rebates.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 1 df = pd.read_excel(excel_file, 'Raw Data') df = trim_all_columns(df) df = df.applymap(lambda s: s.upper() if type(s) == str else s) @@ -47,8 +48,8 @@ def import_from_xls(excel_file): ) df.fillna('') - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): LdvRebates.objects.create( casl_consent=row["CASL Consent"], date_approved=row["DATE APPROVED"], @@ -78,7 +79,6 @@ def import_from_xls(excel_file): delivered=row["Delivered"], consent_to_contact=row["Consent to Contact"] ) - except Exception as error: - print(error) - print(row) + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/services/public_charging.py b/django/api/services/public_charging.py index a0a7f8d6..79be8c33 100644 --- a/django/api/services/public_charging.py +++ b/django/api/services/public_charging.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 3 df = pd.read_excel(excel_file, 'Project_applications', header=2) df = trim_all_columns(df) df = df.applymap(lambda s: s.upper() if type(s) == str else s) @@ -25,12 +26,11 @@ def import_from_xls(excel_file): value=True, inplace=True ) - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): + row_count += 1 PublicCharging.objects.create( applicant_name=row["Applicant Name"], - # city=row["City"], - # postal_code=row["Postal Code"], address=row["Address"], charging_station_info=row["Charging Station Info"], between_25kw_and_50kw=row[">25kW; <50kW"], @@ -48,9 +48,6 @@ def import_from_xls(excel_file): review_number=row["Review Number"], rebate_paid=row["Paid out rebate amount"], ) - except Exception as error: - print('-----------------------') - print(error) - print(row) - print('-----------------------') + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/services/scrap_it.py b/django/api/services/scrap_it.py index 6c8f4da0..4f9bc669 100644 --- a/django/api/services/scrap_it.py +++ b/django/api/services/scrap_it.py @@ -11,6 +11,7 @@ def trim_all_columns(df): def import_from_xls(excel_file): + row_count = 6 df = pd.read_excel(excel_file, 'TOP OTHER TRANSACTIONS', header=5) df = trim_all_columns(df) @@ -18,9 +19,10 @@ def import_from_xls(excel_file): df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna('')) - for _, row in df.iterrows(): - if row["VIN"] == '': continue # Skip rows without this field - try: + try: + for _, row in df.iterrows(): + row_count += 1 + if row["VIN"] == '': continue # Skip rows without this field ScrapIt.objects.create( approval_number=row["Approval Num"], application_received_date=row["App Recv'd Date"], @@ -34,7 +36,6 @@ def import_from_xls(excel_file): budget_code=row["Budget Code"], scrap_date=row["Scrap Date"] ) - except Exception as error: - print(error) - print(row) + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/services/speciality_use_vehicle_incentives.py b/django/api/services/speciality_use_vehicle_incentives.py index 65c7348f..fa645eef 100644 --- a/django/api/services/speciality_use_vehicle_incentives.py +++ b/django/api/services/speciality_use_vehicle_incentives.py @@ -22,6 +22,7 @@ def applicant_type(row): def import_from_xls(excel_file): + row_count = 1 df = pd.read_excel(excel_file, 'Sheet1') df.drop(df.columns.difference([ "Approvals", @@ -46,8 +47,9 @@ def import_from_xls(excel_file): df = df.applymap(lambda s: s.upper() if type(s) == str else s) df['Applicant Type'] = df.apply(lambda row: applicant_type(row), axis=1) - for _, row in df.iterrows(): - try: + try: + for _, row in df.iterrows(): + row_count += 1 SpecialityUseVehicleIncentives.objects.create( approvals=row["Approvals"], date=row["Date"], @@ -60,7 +62,6 @@ def import_from_xls(excel_file): manufacturer=row["Manufacturer"], model=row["Model"], ) - except Exception as error: - print(error) - print(row) + except Exception as error: + return (error,'data',row_count) return True diff --git a/django/api/viewsets/upload.py b/django/api/viewsets/upload.py index 9a18a5bf..2a2d6a49 100644 --- a/django/api/viewsets/upload.py +++ b/django/api/viewsets/upload.py @@ -107,6 +107,7 @@ def import_data(self, request): done = (error, 'file') final_count = model.objects.all().count() records_inserted = final_count - starting_count + records_inserted_msg = "{} records inserted. This table currently contains {} records.".format(records_inserted, final_count) if done != True: try: error_location = done[1] @@ -120,7 +121,7 @@ def import_data(self, request): if error_location == 'data': if error_type in (type(LookupError), type(KeyError), 'KeyError') : error_msg = "Please make sure you've uploaded a file with the correct data including the correctly named columns. There was an error finding: {}. This dataset requires the following columns: {}".format(error, field_names) - elif error_type == type(ValueError): + elif error_type == 'ValueError' or type(ValueError): ## note for next batch of scripts, possibly add str(type(ValueError)) ## to this but check for impacts to other exceptions error_msg = "{} on row {}. Please make sure you've uploaded a file with the correct data.".format(error, error_row) @@ -130,10 +131,10 @@ def import_data(self, request): error_msg = "{}. Please make sure you've uploaded a file with the correct data including the correctly named worksheets.".format(error) if error_msg[-1] != '.': error_msg+='.' - error_msg += " {} records inserted.".format(records_inserted) + error_msg += records_inserted_msg return Response(error_msg, status=status.HTTP_400_BAD_REQUEST) except Exception as error: print(error) return Response('There was an issue!', status=status.HTTP_400_BAD_REQUEST) else: - return Response("{} records inserted.".format(records_inserted), status=status.HTTP_201_CREATED) + return Response(records_inserted_msg, status=status.HTTP_201_CREATED)