feat: add a error count to latest dataset object in gtfs-feeds (#396)

api/.openapi-generator/FILES

@@ -18,6 +18,7 @@ src/feeds_gen/models/gtfs_dataset.py
 src/feeds_gen/models/gtfs_feed.py
 src/feeds_gen/models/gtfs_rt_feed.py
 src/feeds_gen/models/latest_dataset.py
+src/feeds_gen/models/latest_dataset_validation_report.py
 src/feeds_gen/models/location.py
 src/feeds_gen/models/metadata.py
 src/feeds_gen/models/redirect.py

api/src/feeds/impl/feeds_api_impl.py

@@ -1,4 +1,3 @@
-import json
 from typing import List, Type, Set, Union
 from datetime import datetime
 from sqlalchemy.orm import Query, aliased
@@ -33,14 +32,13 @@
     gtfs_rt_feed_not_found,
 )
 from feeds.impl.models.location_impl import LocationImpl
+from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl
 from feeds_gen.apis.feeds_api_base import BaseFeedsApi
 from feeds_gen.models.basic_feed import BasicFeed
-from feeds_gen.models.bounding_box import BoundingBox
 from feeds_gen.models.external_id import ExternalId
 from feeds_gen.models.gtfs_dataset import GtfsDataset
 from feeds_gen.models.gtfs_feed import GtfsFeed
 from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
-from feeds_gen.models.latest_dataset import LatestDataset
 from feeds_gen.models.location import Location as ApiLocation
 from feeds_gen.models.source_info import SourceInfo
 from feeds_gen.models.redirect import Redirect
@@ -212,22 +210,7 @@ def _get_gtfs_feeds(
                 ),
                 (None, None),
             )
-            if latest_dataset:
-                api_dataset = LatestDataset(
-                    id=latest_dataset.stable_id,
-                    downloaded_at=latest_dataset.downloaded_at.isoformat() if latest_dataset.downloaded_at else None,
-                    hash=latest_dataset.hash,
-                    hosted_url=latest_dataset.hosted_url,
-                )
-                if bounding_box:
-                    coordinates = json.loads(bounding_box)["coordinates"][0]
-                    api_dataset.bounding_box = BoundingBox(
-                        minimum_latitude=coordinates[0][1],
-                        maximum_latitude=coordinates[2][1],
-                        minimum_longitude=coordinates[0][0],
-                        maximum_longitude=coordinates[2][0],
-                    )
-                gtfs_feed.latest_dataset = api_dataset
+            gtfs_feed.latest_dataset = LatestDatasetImpl.from_orm(latest_dataset)
 
             gtfs_feeds.append(gtfs_feed)
 

api/src/feeds/impl/models/latest_dataset_impl.py

@@ -0,0 +1,48 @@
+from functools import reduce
+
+from packaging.version import Version
+
+from database_gen.sqlacodegen_models import Gtfsdataset
+from feeds.impl.models.bounding_box_impl import BoundingBoxImpl
+from feeds.impl.models.validation_report_impl import ValidationReportImpl
+from feeds_gen.models.latest_dataset import LatestDataset
+from feeds_gen.models.latest_dataset_validation_report import LatestDatasetValidationReport
+
+
+class LatestDatasetImpl(LatestDataset):
+    """Implementation of the `LatestDataset` model.
+    This class converts a SQLAlchemy row DB object to a Pydantic model.
+    """
+
+    class Config:
+        """Pydantic configuration.
+        Enabling `from_orm` method to create a model instance from a SQLAlchemy row object."""
+
+        from_attributes = True
+        orm_mode = True
+
+    @classmethod
+    def from_orm(cls, dataset: Gtfsdataset | None) -> LatestDataset | None:
+        """Create a model instance from a SQLAlchemy a Latest Dataset row object."""
+        if not dataset:
+            return None
+        validation_report: LatestDatasetValidationReport | None = None
+        if dataset.validation_reports:
+            latest_report = reduce(
+                lambda a, b: a if Version(a.validator_version) > Version(b.validator_version) else b,
+                dataset.validation_reports,
+            )
+            total_error, total_info, total_warning = ValidationReportImpl.compute_totals(latest_report)
+            validation_report = LatestDatasetValidationReport(
+                total_error=total_error,
+                total_warning=total_warning,
+                total_info=total_info,
+            )
+        return cls(
+            id=dataset.stable_id,
+            hosted_url=dataset.hosted_url,
+            bounding_box=BoundingBoxImpl.from_orm(dataset.bounding_box),
+            downloaded_at=dataset.downloaded_at,
+            hash=dataset.hash,
+            validation_report=validation_report,
+        )

api/src/feeds/impl/models/validation_report_impl.py

@@ -17,15 +17,9 @@ class Config:
         from_attributes = True
         orm_mode = True
 
-    @classmethod
-    def _get_logger(cls):
-        return Logger(ValidationReportImpl.__class__.__module__).get_logger()
-
-    @classmethod
-    def from_orm(cls, validation_report: Validationreport | None) -> ValidationReport | None:
-        """Create a model instance from a SQLAlchemy a Validation Report row object."""
-        if not validation_report:
-            return None
+    @staticmethod
+    def compute_totals(validation_report) -> tuple[int, int, int]:
+        """Compute the total number of errors, info, and warnings from a validation report."""
         total_info, total_warning, total_error = 0, 0, 0
         for notice in validation_report.notices:
             match notice.severity:
@@ -36,7 +30,19 @@ def from_orm(cls, validation_report: Validationreport | None) -> ValidationRepor
                 case "ERROR":
                     total_error += notice.total_notices
                 case _:
-                    cls._get_logger().warning(f"Unknown severity: {notice.severity}")
+                    ValidationReportImpl._get_logger().warning(f"Unknown severity: {notice.severity}")
+        return total_error, total_info, total_warning
+
+    @classmethod
+    def _get_logger(cls):
+        return Logger(ValidationReportImpl.__class__.__module__).get_logger()
+
+    @classmethod
+    def from_orm(cls, validation_report: Validationreport | None) -> ValidationReport | None:
+        """Create a model instance from a SQLAlchemy a Validation Report row object."""
+        if not validation_report:
+            return None
+        total_error, total_info, total_warning = cls.compute_totals(validation_report)
         return cls(
             validated_at=validation_report.validated_at,
             features=[feature.name for feature in validation_report.features],

api/tests/unittest/models/test_latest_dataset_impl.py

@@ -0,0 +1,55 @@
+import unittest
+from datetime import datetime
+
+from geoalchemy2 import WKTElement
+
+from database_gen.sqlacodegen_models import Gtfsdataset, Feed, Validationreport, Notice
+from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl
+from feeds_gen.models.latest_dataset import LatestDataset
+
+POLYGON = "POLYGON ((3.0 1.0, 4.0 1.0, 4.0 2.0, 3.0 2.0, 3.0 1.0))"
+
+
+class TestLatestDatasetImpl(unittest.TestCase):
+    def test_from_orm(self):
+        now = datetime.now()
+        assert LatestDatasetImpl.from_orm(
+            Gtfsdataset(
+                id="10",
+                stable_id="stable_id",
+                feed=Feed(stable_id="feed_stable_id"),
+                hosted_url="http://example.com",
+                note="note",
+                downloaded_at=now,
+                hash="hash",
+                bounding_box=WKTElement(POLYGON, srid=4326),
+                validation_reports=[
+                    Validationreport(validator_version="1.0.0"),
+                    Validationreport(
+                        validator_version="1.2.0",
+                        notices=[
+                            Notice(severity="INFO", total_notices=1),
+                            Notice(severity="ERROR", total_notices=2),
+                            Notice(severity="WARNING", total_notices=3),
+                        ],
+                    ),
+                    Validationreport(validator_version="1.1.1"),
+                ],
+            )
+        ) == LatestDataset(
+            id="stable_id",
+            feed_id="feed_stable_id",
+            hosted_url="http://example.com",
+            note="note",
+            downloaded_at=now,
+            hash="hash",
+            bounding_box={
+                "minimum_latitude": 1.0,
+                "maximum_latitude": 2.0,
+                "minimum_longitude": 3.0,
+                "maximum_longitude": 4.0,
+            },
+            validation_report={"validator_version": "1.2.0", "total_error": 2, "total_info": 1, "total_warning": 3},
+        )
+
+        assert LatestDatasetImpl.from_orm(None) is None

docs/DatabaseCatalogAPI.yaml

@@ -424,6 +424,21 @@ components:
           description: A hash of the dataset.
           type: string
           example: ad3805c4941cd37881ff40c342e831b5f5224f3d8a9a2ec3ac197d3652c78e42
+        validation_report:
+          type: object
+          properties:
+            total_error:
+              type: integer
+              example: 1
+              minimum: 0
+            total_warning:
+              type: integer
+              example: 2
+              minimum: 0
+            total_info:
+              type: integer
+              example: 3
+              minimum: 0
 
 #    Have to put the enum inline because of a bug in openapi-generator
 #    EntityTypes: