More result details, minor enhancements

common/base_converter/base_converter.py

@@ -8,6 +8,7 @@
 
 from validataclass.validators import DataclassValidator
 
+from common.exceptions import ImportParkingSiteException
 from common.models import ImportSourceResult, SourceStatus
 from common.validators import RealtimeParkingSiteInput, StaticParkingSiteInput
 from util import SourceInfo
@@ -26,16 +27,16 @@ def generate_import_source_result(
         self,
         static_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None,
         realtime_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None,
-        static_parking_site_error_count: int = 0,
-        realtime_parking_site_error_count: int = 0,
+        static_parking_site_errors: Optional[list[ImportParkingSiteException]] = None,
+        realtime_parking_site_errors: Optional[list[ImportParkingSiteException]] = None,
     ) -> ImportSourceResult:
         return ImportSourceResult(
             uid=self.source_info.id,
             name=self.source_info.name,
             public_url=self.source_info.public_url,
             status=SourceStatus.ACTIVE,
-            static_parking_site_inputs=static_parking_site_inputs or [],
-            realtime_parking_site_inputs=realtime_parking_site_inputs or [],
-            static_parking_site_error_count=static_parking_site_error_count,
-            realtime_parking_site_error_count=realtime_parking_site_error_count,
+            static_parking_site_inputs=static_parking_site_inputs,
+            realtime_parking_site_inputs=realtime_parking_site_inputs,
+            static_parking_site_errors=static_parking_site_errors,
+            realtime_parking_site_errors=realtime_parking_site_errors,
         )

common/models.py

@@ -3,10 +3,11 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional
+from typing import Optional
 
+from common.exceptions import ImportParkingSiteException
 from common.validators import StaticParkingSiteInput
 
 
@@ -29,8 +30,20 @@ class ImportSourceResult:
     attribution_contributor: Optional[str] = None
     attribution_url: Optional[str] = None
 
-    static_parking_site_inputs: List[StaticParkingSiteInput] = field(default_factory=list)
-    realtime_parking_site_inputs: List[StaticParkingSiteInput] = field(default_factory=list)
+    static_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None
+    realtime_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None
 
-    static_parking_site_error_count: int = 0
-    realtime_parking_site_error_count: int = 0
+    static_parking_site_errors: Optional[list[ImportParkingSiteException]] = None
+    realtime_parking_site_errors: Optional[list[ImportParkingSiteException]] = None
+
+    @property
+    def static_parking_site_error_count(self) -> Optional[int]:
+        if self.static_parking_site_errors is None:
+            return None
+        return len(self.static_parking_site_errors)
+
+    @property
+    def realtime_parking_site_error_count(self) -> Optional[int]:
+        if self.realtime_parking_site_errors is None:
+            return None
+        return len(self.realtime_parking_site_errors)

requirements.txt

@@ -17,3 +17,5 @@ python-decouple==3.5
 # for v3
 validataclass~=0.9.0
 openpyxl~=3.1.2
+lxml~=4.9.3
+

test-push-converter.py

@@ -88,6 +88,8 @@ def print_result(result: ImportSourceResult):
     print('### failures ###')  # noqa: T201
     print(f'static_parking_site_error_count: {result.static_parking_site_error_count}')  # noqa: T201
     print(f'realtime_parking_site_error_count: {result.realtime_parking_site_error_count}')  # noqa: T201
+    print(f'static_parking_site_errors: {result.static_parking_site_errors}')  # noqa: T201
+    print(f'realtime_parking_site_errors: {result.realtime_parking_site_errors}')  # noqa: T201
 
 
 def filter_none(data: dict) -> dict:

v3/vrs_p_r.py

@@ -64,10 +64,14 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult:
         worksheet = workbook.active
         mapping = self.get_mapping_by_header(next(worksheet.rows))
 
-        validation_exceptions: list[ImportParkingSiteException] = []
+        static_parking_site_errors: list[ImportParkingSiteException] = []
         static_parking_site_inputs: list[StaticParkingSiteInput] = []
 
         for row in worksheet.iter_rows(min_row=2):
+            # ignore empty lines as LibreOffice sometimes adds empty rows at the end of a file
+            if row[0].value is None:
+                continue
+
             parking_site_raw_dict: dict[str, str] = {}
             for position, field in enumerate(mapping):
                 parking_site_raw_dict[field] = row[position].value
@@ -83,7 +87,7 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult:
             try:
                 static_parking_site_inputs.append(self.static_parking_site_validator.validate(parking_site_dict))
             except ValidationError as e:
-                validation_exceptions.append(
+                static_parking_site_errors.append(
                     ImportParkingSiteException(
                         uid=parking_site_dict.get('uid'),
                         message=f'invalid static parking site data: {e.to_dict()}',
@@ -93,5 +97,5 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult:
 
         return self.generate_import_source_result(
             static_parking_site_inputs=static_parking_site_inputs,
-            static_parking_site_error_count=len(validation_exceptions),
+            static_parking_site_errors=static_parking_site_errors,
         )