From 02eabb9e4b1d7b27862cf8d752475fef5a3f6a63 Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Sun, 3 Dec 2023 08:43:38 +0100 Subject: [PATCH 1/2] More result details, minor enhancements --- common/base_converter/base_converter.py | 13 +++++++------ common/models.py | 25 +++++++++++++++++++------ requirements.txt | 2 ++ test-push-converter.py | 2 ++ v3/vrs_p_r.py | 10 +++++++--- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/common/base_converter/base_converter.py b/common/base_converter/base_converter.py index 3fd5a7a..a248f2a 100644 --- a/common/base_converter/base_converter.py +++ b/common/base_converter/base_converter.py @@ -8,6 +8,7 @@ from validataclass.validators import DataclassValidator +from common.exceptions import ImportParkingSiteException from common.models import ImportSourceResult, SourceStatus from common.validators import RealtimeParkingSiteInput, StaticParkingSiteInput from util import SourceInfo @@ -26,16 +27,16 @@ def generate_import_source_result( self, static_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None, realtime_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None, - static_parking_site_error_count: int = 0, - realtime_parking_site_error_count: int = 0, + static_parking_site_errors: Optional[list[ImportParkingSiteException]] = None, + realtime_parking_site_errors: Optional[list[ImportParkingSiteException]] = None, ) -> ImportSourceResult: return ImportSourceResult( uid=self.source_info.id, name=self.source_info.name, public_url=self.source_info.public_url, status=SourceStatus.ACTIVE, - static_parking_site_inputs=static_parking_site_inputs or [], - realtime_parking_site_inputs=realtime_parking_site_inputs or [], - static_parking_site_error_count=static_parking_site_error_count, - realtime_parking_site_error_count=realtime_parking_site_error_count, + static_parking_site_inputs=static_parking_site_inputs, + realtime_parking_site_inputs=realtime_parking_site_inputs, + static_parking_site_errors=static_parking_site_errors, + realtime_parking_site_errors=realtime_parking_site_errors, ) diff --git a/common/models.py b/common/models.py index 8886ff9..52c2204 100644 --- a/common/models.py +++ b/common/models.py @@ -3,10 +3,11 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from enum import Enum -from typing import List, Optional +from typing import Optional +from common.exceptions import ImportParkingSiteException from common.validators import StaticParkingSiteInput @@ -29,8 +30,20 @@ class ImportSourceResult: attribution_contributor: Optional[str] = None attribution_url: Optional[str] = None - static_parking_site_inputs: List[StaticParkingSiteInput] = field(default_factory=list) - realtime_parking_site_inputs: List[StaticParkingSiteInput] = field(default_factory=list) + static_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None + realtime_parking_site_inputs: Optional[list[StaticParkingSiteInput]] = None - static_parking_site_error_count: int = 0 - realtime_parking_site_error_count: int = 0 + static_parking_site_errors: Optional[list[ImportParkingSiteException]] = None + realtime_parking_site_errors: Optional[list[ImportParkingSiteException]] = None + + @property + def static_parking_site_error_count(self) -> Optional[int]: + if self.static_parking_site_errors is None: + return None + return len(self.static_parking_site_errors) + + @property + def realtime_parking_site_error_count(self) -> Optional[int]: + if self.realtime_parking_site_errors is None: + return None + return len(self.realtime_parking_site_errors) diff --git a/requirements.txt b/requirements.txt index 706c91f..6563aa0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,5 @@ python-decouple==3.5 # for v3 validataclass~=0.9.0 openpyxl~=3.1.2 +lxml~=4.9.3 + diff --git a/test-push-converter.py b/test-push-converter.py index 26c64d0..c0aa2c0 100644 --- a/test-push-converter.py +++ b/test-push-converter.py @@ -88,6 +88,8 @@ def print_result(result: ImportSourceResult): print('### failures ###') # noqa: T201 print(f'static_parking_site_error_count: {result.static_parking_site_error_count}') # noqa: T201 print(f'realtime_parking_site_error_count: {result.realtime_parking_site_error_count}') # noqa: T201 + print(f'static_parking_site_errors: {result.static_parking_site_errors}') # noqa: T201 + print(f'realtime_parking_site_errors: {result.realtime_parking_site_errors}') # noqa: T201 def filter_none(data: dict) -> dict: diff --git a/v3/vrs_p_r.py b/v3/vrs_p_r.py index 6697b76..0aa0520 100644 --- a/v3/vrs_p_r.py +++ b/v3/vrs_p_r.py @@ -64,10 +64,14 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult: worksheet = workbook.active mapping = self.get_mapping_by_header(next(worksheet.rows)) - validation_exceptions: list[ImportParkingSiteException] = [] + static_parking_site_errors: list[ImportParkingSiteException] = [] static_parking_site_inputs: list[StaticParkingSiteInput] = [] for row in worksheet.iter_rows(min_row=2): + # ignore empty lines + if row[0].value is None: + continue + parking_site_raw_dict: dict[str, str] = {} for position, field in enumerate(mapping): parking_site_raw_dict[field] = row[position].value @@ -83,7 +87,7 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult: try: static_parking_site_inputs.append(self.static_parking_site_validator.validate(parking_site_dict)) except ValidationError as e: - validation_exceptions.append( + static_parking_site_errors.append( ImportParkingSiteException( uid=parking_site_dict.get('uid'), message=f'invalid static parking site data: {e.to_dict()}', @@ -93,5 +97,5 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult: return self.generate_import_source_result( static_parking_site_inputs=static_parking_site_inputs, - static_parking_site_error_count=len(validation_exceptions), + static_parking_site_errors=static_parking_site_errors, ) From 8c0575cb1e5c479e1db1cf4cee2246109d0a689c Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Sat, 9 Dec 2023 08:59:47 +0100 Subject: [PATCH 2/2] explain empty lines --- v3/vrs_p_r.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v3/vrs_p_r.py b/v3/vrs_p_r.py index 0aa0520..2d3ea0b 100644 --- a/v3/vrs_p_r.py +++ b/v3/vrs_p_r.py @@ -68,7 +68,7 @@ def handle_xlsx(self, workbook: Workbook) -> ImportSourceResult: static_parking_site_inputs: list[StaticParkingSiteInput] = [] for row in worksheet.iter_rows(min_row=2): - # ignore empty lines + # ignore empty lines as LibreOffice sometimes adds empty rows at the end of a file if row[0].value is None: continue