From daeef8851be8fa774a065edbc6ae6bcc233de602 Mon Sep 17 00:00:00 2001 From: cka-y <60586858+cka-y@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:44:05 -0400 Subject: [PATCH] feat: automate locations - modified API responses (#661) --- api/requirements.txt | 3 +- api/src/feeds/impl/feeds_api_impl.py | 153 +++++++++++------- api/src/feeds/impl/models/basic_feed_impl.py | 2 +- api/src/feeds/impl/models/gtfs_feed_impl.py | 12 +- .../feeds/impl/models/gtfs_rt_feed_impl.py | 11 +- api/src/feeds/impl/models/location_impl.py | 1 + .../models/search_feed_item_result_impl.py | 30 ++++ api/src/utils/location_translation.py | 90 +++++++++++ api/tests/test_utils/db_utils.py | 3 +- .../unittest/models/test_basic_feed_impl.py | 3 +- .../unittest/models/test_gtfs_feed_impl.py | 8 +- .../unittest/models/test_location_impl.py | 8 +- .../test_search_feed_item_result_impl.py | 3 + docs/DatabaseCatalogAPI.yaml | 8 +- .../reverse_geolocation/location_extractor.py | 20 ++- .../extract_location/tests/test_geocoding.py | 2 +- functions-python/test_utils/database_utils.py | 3 +- liquibase/changelog.xml | 1 + liquibase/changes/feat_622.sql | 29 ++++ web-app/src/app/services/feeds/types.ts | 9 +- 20 files changed, 315 insertions(+), 84 deletions(-) create mode 100644 api/src/utils/location_translation.py create mode 100644 liquibase/changes/feat_622.sql diff --git a/api/requirements.txt b/api/requirements.txt index fd98f2b24..71c34089c 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -47,4 +47,5 @@ cloud-sql-python-connector[pg8000] fastapi-filter[sqlalchemy]==1.0.0 PyJWT shapely -google-cloud-pubsub \ No newline at end of file +google-cloud-pubsub +pycountry \ No newline at end of file diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py index bb20d762b..c7213e75f 100644 --- a/api/src/feeds/impl/feeds_api_impl.py +++ b/api/src/feeds/impl/feeds_api_impl.py @@ -1,8 +1,8 @@ from datetime import datetime -from typing import List, Union +from typing import List, Union, TypeVar from sqlalchemy.orm import joinedload - +from sqlalchemy.orm.query import Query from database.database import Database from database_gen.sqlacodegen_models import ( Feed, @@ -12,6 +12,7 @@ Location, Validationreport, Entitytype, + t_location_with_translations_en, ) from feeds.filters.feed_filter import FeedFilter from feeds.filters.gtfs_dataset_filter import GtfsDatasetFilter @@ -36,6 +37,9 @@ from feeds_gen.models.gtfs_feed import GtfsFeed from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed from utils.date_utils import valid_iso_date +from utils.location_translation import create_location_translation_object, LocationTranslation + +T = TypeVar("T", bound="BasicFeed") class FeedsApiImpl(BaseFeedsApi): @@ -91,20 +95,35 @@ def get_gtfs_feed( id: str, ) -> GtfsFeed: """Get the specified gtfs feed from the Mobility Database.""" - feed = ( + feed, translations = self._get_gtfs_feed(id) + if feed: + return GtfsFeedImpl.from_orm(feed, translations) + else: + raise_http_error(404, gtfs_feed_not_found.format(id)) + + @staticmethod + def _get_gtfs_feed(stable_id: str) -> tuple[Gtfsfeed | None, dict[str, LocationTranslation]]: + results = ( FeedFilter( - stable_id=id, + stable_id=stable_id, status=None, provider__ilike=None, producer_url__ilike=None, ) - .filter(Database().get_query_model(Gtfsfeed)) - .first() - ) - if feed: - return GtfsFeedImpl.from_orm(feed) - else: - raise_http_error(404, gtfs_feed_not_found.format(id)) + .filter(Database().get_session().query(Gtfsfeed, t_location_with_translations_en)) + .outerjoin(Location, Feed.locations) + .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id) + .options( + joinedload(Gtfsfeed.gtfsdatasets) + .joinedload(Gtfsdataset.validation_reports) + .joinedload(Validationreport.notices), + *BasicFeedImpl.get_joinedload_options(), + ) + ).all() + if len(results) > 0 and results[0].Gtfsfeed: + translations = {result[1]: create_location_translation_object(result) for result in results} + return results[0].Gtfsfeed, translations + return None, {} def get_gtfs_feed_datasets( self, @@ -176,43 +195,54 @@ def get_gtfs_feeds( municipality__ilike=municipality, ), ) - gtfs_feed_query = gtfs_feed_filter.filter(Database().get_query_model(Gtfsfeed)) - - gtfs_feed_query = gtfs_feed_query.outerjoin(Location, Feed.locations).options( - joinedload(Gtfsfeed.gtfsdatasets) - .joinedload(Gtfsdataset.validation_reports) - .joinedload(Validationreport.notices), - *BasicFeedImpl.get_joinedload_options(), + gtfs_feed_query = gtfs_feed_filter.filter( + Database().get_session().query(Gtfsfeed, t_location_with_translations_en) + ) + gtfs_feed_query = ( + gtfs_feed_query.outerjoin(Location, Feed.locations) + .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id) + .options( + joinedload(Gtfsfeed.gtfsdatasets) + .joinedload(Gtfsdataset.validation_reports) + .joinedload(Validationreport.notices), + *BasicFeedImpl.get_joinedload_options(), + ) + .order_by(Gtfsfeed.provider, Gtfsfeed.stable_id) ) gtfs_feed_query = gtfs_feed_query.order_by(Gtfsfeed.provider, Gtfsfeed.stable_id) gtfs_feed_query = DatasetsApiImpl.apply_bounding_filtering( gtfs_feed_query, dataset_latitudes, dataset_longitudes, bounding_filter_method ) - if limit is not None: - gtfs_feed_query = gtfs_feed_query.limit(limit) - if offset is not None: - gtfs_feed_query = gtfs_feed_query.offset(offset) - results = gtfs_feed_query.all() - return [GtfsFeedImpl.from_orm(gtfs_feed) for gtfs_feed in results] + return self._get_response(gtfs_feed_query, limit, offset, GtfsFeedImpl) def get_gtfs_rt_feed( self, id: str, ) -> GtfsRTFeed: """Get the specified GTFS Realtime feed from the Mobility Database.""" - feed = ( - GtfsRtFeedFilter( - stable_id=id, - provider__ilike=None, - producer_url__ilike=None, - entity_types=None, - location=None, - ) - .filter(Database().get_query_model(Gtfsrealtimefeed)) - .first() + gtfs_rt_feed_filter = GtfsRtFeedFilter( + stable_id=id, + provider__ilike=None, + producer_url__ilike=None, + entity_types=None, + location=None, ) - if feed: - return GtfsRTFeedImpl.from_orm(feed) + results = gtfs_rt_feed_filter.filter( + Database() + .get_session() + .query(Gtfsrealtimefeed, t_location_with_translations_en) + .outerjoin(Location, Gtfsrealtimefeed.locations) + .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id) + .options( + joinedload(Gtfsrealtimefeed.entitytypes), + joinedload(Gtfsrealtimefeed.gtfs_feeds), + *BasicFeedImpl.get_joinedload_options(), + ) + ).all() + + if len(results) > 0 and results[0].Gtfsrealtimefeed: + translations = {result[1]: create_location_translation_object(result) for result in results} + return GtfsRTFeedImpl.from_orm(results[0].Gtfsrealtimefeed, translations) else: raise_http_error(404, gtfs_rt_feed_not_found.format(id)) @@ -251,42 +281,41 @@ def get_gtfs_rt_feeds( municipality__ilike=municipality, ), ) - gtfs_rt_feed_query = gtfs_rt_feed_filter.filter(Database().get_query_model(Gtfsrealtimefeed)).options( - *BasicFeedImpl.get_joinedload_options() + gtfs_rt_feed_query = gtfs_rt_feed_filter.filter( + Database().get_session().query(Gtfsrealtimefeed, t_location_with_translations_en) ) - gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Entitytype, Gtfsrealtimefeed.entitytypes).options( - joinedload(Gtfsrealtimefeed.entitytypes) - ) - gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Location, Feed.locations).options( - joinedload(Gtfsrealtimefeed.locations) - ) - gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Gtfsfeed, Gtfsrealtimefeed.gtfs_feeds).options( - joinedload(Gtfsrealtimefeed.gtfs_feeds) + gtfs_rt_feed_query = ( + gtfs_rt_feed_query.outerjoin(Location, Gtfsrealtimefeed.locations) + .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id) + .outerjoin(Entitytype, Gtfsrealtimefeed.entitytypes) + .options( + joinedload(Gtfsrealtimefeed.entitytypes), + joinedload(Gtfsrealtimefeed.gtfs_feeds), + *BasicFeedImpl.get_joinedload_options(), + ) + .order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id) ) - gtfs_rt_feed_query = gtfs_rt_feed_query.order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id) + return self._get_response(gtfs_rt_feed_query, limit, offset, GtfsRTFeedImpl) + + @staticmethod + def _get_response(feed_query: Query, limit: int, offset: int, impl_cls: type[T]) -> List[T]: + """Get the response for the feed query.""" if limit is not None: - gtfs_rt_feed_query = gtfs_rt_feed_query.limit(limit) + feed_query = feed_query.limit(limit) if offset is not None: - gtfs_rt_feed_query = gtfs_rt_feed_query.offset(offset) - results = gtfs_rt_feed_query.all() - return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed) for gtfs_rt_feed in results] + feed_query = feed_query.offset(offset) + results = feed_query.all() + location_translations = {row[1]: create_location_translation_object(row) for row in results} + response = [impl_cls.from_orm(feed[0], location_translations) for feed in results] + return list({feed.id: feed for feed in response}.values()) def get_gtfs_feed_gtfs_rt_feeds( self, id: str, ) -> List[GtfsRTFeed]: """Get a list of GTFS Realtime related to a GTFS feed.""" - feed = ( - FeedFilter( - stable_id=id, - status=None, - provider__ilike=None, - producer_url__ilike=None, - ) - .filter(Database().get_query_model(Gtfsfeed)) - .first() - ) + feed, translations = self._get_gtfs_feed(id) if feed: - return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed) for gtfs_rt_feed in feed.gtfs_rt_feeds] + return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed, translations) for gtfs_rt_feed in feed.gtfs_rt_feeds] else: raise_http_error(404, gtfs_feed_not_found.format(id)) diff --git a/api/src/feeds/impl/models/basic_feed_impl.py b/api/src/feeds/impl/models/basic_feed_impl.py index 3f3512b37..cf6bf87e3 100644 --- a/api/src/feeds/impl/models/basic_feed_impl.py +++ b/api/src/feeds/impl/models/basic_feed_impl.py @@ -20,7 +20,7 @@ class Config: from_attributes = True @classmethod - def from_orm(cls, feed: Feed | None) -> BasicFeed | None: + def from_orm(cls, feed: Feed | None, _=None) -> BasicFeed | None: if not feed: return None return cls( diff --git a/api/src/feeds/impl/models/gtfs_feed_impl.py b/api/src/feeds/impl/models/gtfs_feed_impl.py index c1d9fd825..ded4d00ff 100644 --- a/api/src/feeds/impl/models/gtfs_feed_impl.py +++ b/api/src/feeds/impl/models/gtfs_feed_impl.py @@ -1,8 +1,11 @@ +from typing import Dict + from database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm from feeds.impl.models.basic_feed_impl import BaseFeedImpl from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl from feeds.impl.models.location_impl import LocationImpl from feeds_gen.models.gtfs_feed import GtfsFeed +from utils.location_translation import LocationTranslation, translate_feed_locations class GtfsFeedImpl(BaseFeedImpl, GtfsFeed): @@ -17,12 +20,15 @@ class Config: from_attributes = True @classmethod - def from_orm(cls, feed: GtfsfeedOrm | None) -> GtfsFeed | None: - gtfs_feed = super().from_orm(feed) + def from_orm( + cls, feed: GtfsfeedOrm | None, location_translations: Dict[str, LocationTranslation] = None + ) -> GtfsFeed | None: + if location_translations is not None: + translate_feed_locations(feed, location_translations) + gtfs_feed: GtfsFeed = super().from_orm(feed) if not gtfs_feed: return None gtfs_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations] - latest_dataset = next( (dataset for dataset in feed.gtfsdatasets if dataset is not None and dataset.latest), None ) diff --git a/api/src/feeds/impl/models/gtfs_rt_feed_impl.py b/api/src/feeds/impl/models/gtfs_rt_feed_impl.py index a46169deb..d19956762 100644 --- a/api/src/feeds/impl/models/gtfs_rt_feed_impl.py +++ b/api/src/feeds/impl/models/gtfs_rt_feed_impl.py @@ -1,7 +1,10 @@ +from typing import Dict + from database_gen.sqlacodegen_models import Gtfsrealtimefeed as GtfsRTFeedOrm from feeds.impl.models.basic_feed_impl import BaseFeedImpl from feeds.impl.models.location_impl import LocationImpl from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed +from utils.location_translation import LocationTranslation, translate_feed_locations class GtfsRTFeedImpl(BaseFeedImpl, GtfsRTFeed): @@ -14,8 +17,12 @@ class Config: from_attributes = True @classmethod - def from_orm(cls, feed: GtfsRTFeedOrm | None) -> GtfsRTFeed | None: - gtfs_rt_feed = super().from_orm(feed) + def from_orm( + cls, feed: GtfsRTFeedOrm | None, location_translations: Dict[str, LocationTranslation] = None + ) -> GtfsRTFeed | None: + if location_translations is not None: + translate_feed_locations(feed, location_translations) + gtfs_rt_feed: GtfsRTFeed = super().from_orm(feed) if not gtfs_rt_feed: return None gtfs_rt_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations] diff --git a/api/src/feeds/impl/models/location_impl.py b/api/src/feeds/impl/models/location_impl.py index cb60df905..385aab593 100644 --- a/api/src/feeds/impl/models/location_impl.py +++ b/api/src/feeds/impl/models/location_impl.py @@ -16,6 +16,7 @@ def from_orm(cls, location: LocationOrm | None) -> Location | None: return None return cls( country_code=location.country_code, + country=location.country, subdivision_name=location.subdivision_name, municipality=location.municipality, ) diff --git a/api/src/feeds/impl/models/search_feed_item_result_impl.py b/api/src/feeds/impl/models/search_feed_item_result_impl.py index c0b218c95..3fa3eb2bb 100644 --- a/api/src/feeds/impl/models/search_feed_item_result_impl.py +++ b/api/src/feeds/impl/models/search_feed_item_result_impl.py @@ -1,6 +1,7 @@ from feeds_gen.models.latest_dataset import LatestDataset from feeds_gen.models.search_feed_item_result import SearchFeedItemResult from feeds_gen.models.source_info import SourceInfo +import pycountry class SearchFeedItemResultImpl(SearchFeedItemResult): @@ -74,11 +75,40 @@ def from_orm_gtfs_rt(cls, feed_search_row): feed_references=feed_search_row.feed_reference_ids, ) + @classmethod + def _translate_locations(cls, feed_search_row): + """Translate location information in the feed search row.""" + country_translations = cls._create_translation_dict(feed_search_row.country_translations) + subdivision_translations = cls._create_translation_dict(feed_search_row.subdivision_name_translations) + municipality_translations = cls._create_translation_dict(feed_search_row.municipality_translations) + + for location in feed_search_row.locations: + location["country"] = country_translations.get(location["country"], location["country"]) + if location["country"] is None: + location["country"] = pycountry.countries.get(alpha_2=location["country_code"]).name + location["subdivision_name"] = subdivision_translations.get( + location["subdivision_name"], location["subdivision_name"] + ) + location["municipality"] = municipality_translations.get(location["municipality"], location["municipality"]) + + @staticmethod + def _create_translation_dict(translations): + """Helper method to create a translation dictionary.""" + if translations: + return { + elem.get("key"): elem.get("value") for elem in translations if elem.get("key") and elem.get("value") + } + return {} + @classmethod def from_orm(cls, feed_search_row): """Create a model instance from a SQLAlchemy row object.""" if feed_search_row is None: return None + + # Translate location data + cls._translate_locations(feed_search_row) + match feed_search_row.data_type: case "gtfs": return cls.from_orm_gtfs(feed_search_row) diff --git a/api/src/utils/location_translation.py b/api/src/utils/location_translation.py new file mode 100644 index 000000000..68f94046b --- /dev/null +++ b/api/src/utils/location_translation.py @@ -0,0 +1,90 @@ +import pycountry +from sqlalchemy.engine.result import Row +from database_gen.sqlacodegen_models import Location as LocationOrm +from database_gen.sqlacodegen_models import Feed as FeedOrm + + +class LocationTranslation: + def __init__( + self, + location_id, + country_code, + country, + subdivision_name, + municipality, + country_translation, + subdivision_name_translation, + municipality_translation, + ): + self.location_id = location_id + self.country_code = country_code + self.country = country + self.subdivision_name = subdivision_name + self.municipality = municipality + self.country_translation = country_translation + self.subdivision_name_translation = subdivision_name_translation + self.municipality_translation = municipality_translation + + def __repr__(self): + return ( + f"LocationTranslation(location_id={self.location_id}, country_code={self.country_code}, " + f"country={self.country}, subdivision_name={self.subdivision_name}, municipality={self.municipality}, " + f"country_translation={self.country_translation}, subdivision_name_translation=" + f"{self.subdivision_name_translation}, municipality_translation={self.municipality_translation})" + ) + + +def create_location_translation_object(row: Row): + """Create a location translation object from a row.""" + return LocationTranslation( + location_id=row[1], + country_code=row[2], + country=row[3], + subdivision_name=row[4], + municipality=row[5], + country_translation=row[6], + subdivision_name_translation=row[7], + municipality_translation=row[8], + ) + + +def set_country_name(location: LocationOrm, country_name: str) -> LocationOrm: + """ + Set the country name of a location + :param location: The location object + :param country_name: The english translation of the country name + :return: Modified location object + """ + try: + if country_name is not None: + location.country = country_name + else: + location.country = pycountry.countries.get(alpha_2=location.country_code).name + except AttributeError: + pass + return location + + +def translate_feed_locations(feed: FeedOrm, location_translations: dict[str, LocationTranslation]): + """ + Translate the locations of a feed. + :param feed: The feed object + :param location_translations: The location translations + """ + for location in feed.locations: + location_translation = location_translations.get(location.id) + location = set_country_name( + location, location_translation.country_translation if location_translation else None + ) + if location_translation: + location.country_code = location_translation.country_code + location.subdivision_name = ( + location_translation.subdivision_name_translation + if location_translation.subdivision_name_translation + else location.subdivision_name + ) + location.municipality = ( + location_translation.municipality_translation + if location_translation.municipality_translation + else location.municipality + ) diff --git a/api/tests/test_utils/db_utils.py b/api/tests/test_utils/db_utils.py index 06a93e86d..633a0d414 100644 --- a/api/tests/test_utils/db_utils.py +++ b/api/tests/test_utils/db_utils.py @@ -184,8 +184,9 @@ def is_test_db(url): "geography_columns", "geometry_columns", "spatial_ref_sys", - # Excluding the materialized view + # Excluding the views "feedsearch", + "location_with_translations_en", ] diff --git a/api/tests/unittest/models/test_basic_feed_impl.py b/api/tests/unittest/models/test_basic_feed_impl.py index f5d567752..95881a866 100644 --- a/api/tests/unittest/models/test_basic_feed_impl.py +++ b/api/tests/unittest/models/test_basic_feed_impl.py @@ -41,7 +41,8 @@ locations=[ Location( id="id", - country_code="country_code", + country_code="CA", + country=None, subdivision_name="subdivision_name", municipality="municipality", ) diff --git a/api/tests/unittest/models/test_gtfs_feed_impl.py b/api/tests/unittest/models/test_gtfs_feed_impl.py index ea96ddd0a..7c902eb59 100644 --- a/api/tests/unittest/models/test_gtfs_feed_impl.py +++ b/api/tests/unittest/models/test_gtfs_feed_impl.py @@ -65,7 +65,8 @@ def create_test_notice(notice_code: str, total_notices: int, severity: str): locations=[ Location( id="id", - country_code="country_code", + country_code="CA", + country=None, subdivision_name="subdivision_name", municipality="municipality", ) @@ -146,7 +147,8 @@ def create_test_notice(notice_code: str, total_notices: int, severity: str): ], locations=[ LocationImpl( - country_code="country_code", + country_code="CA", + country="Canada", subdivision_name="subdivision_name", municipality="municipality", ) @@ -176,7 +178,7 @@ class TestGtfsFeedImpl(unittest.TestCase): def test_from_orm_all_fields(self): """Test the `from_orm` method with all fields.""" - result = GtfsFeedImpl.from_orm(gtfs_feed_orm) + result = GtfsFeedImpl.from_orm(gtfs_feed_orm, {}) assert result == expected_gtfs_feed_result def test_from_orm_empty_fields(self): diff --git a/api/tests/unittest/models/test_location_impl.py b/api/tests/unittest/models/test_location_impl.py index a0d39ed29..24ba78529 100644 --- a/api/tests/unittest/models/test_location_impl.py +++ b/api/tests/unittest/models/test_location_impl.py @@ -7,8 +7,12 @@ class TestLocationImpl(unittest.TestCase): def test_from_orm(self): result = LocationImpl.from_orm( - LocationOrm(country_code="US", subdivision_name="California", municipality="Los Angeles") + LocationOrm( + country_code="US", subdivision_name="California", municipality="Los Angeles", country="United States" + ) + ) + assert result == LocationImpl( + country_code="US", country="United States", subdivision_name="California", municipality="Los Angeles" ) - assert result == LocationImpl(country_code="US", subdivision_name="California", municipality="Los Angeles") assert LocationImpl.from_orm(None) is None diff --git a/api/tests/unittest/models/test_search_feed_item_result_impl.py b/api/tests/unittest/models/test_search_feed_item_result_impl.py index 729a13e8c..6ce31163e 100644 --- a/api/tests/unittest/models/test_search_feed_item_result_impl.py +++ b/api/tests/unittest/models/test_search_feed_item_result_impl.py @@ -43,6 +43,9 @@ def __init__(self, **kwargs): feed_reference_ids=[], entities=["sa"], locations=[], + country_translations=[], + subdivision_name_translations=[], + municipality_translations=[], ) diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 76850e63d..467c7dfd5 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -677,14 +677,18 @@ components: For a list of valid codes [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). type: string example: US + country: + description: The english name of the country where the system is located. + type: string + example: United States subdivision_name: description: > - ISO 3166-2 subdivision name designating the subdivision (e.g province, state, region) where the system is located. + ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). type: string example: California municipality: - description: Primary municipality in which the transit system is located. + description: Primary municipality in english in which the transit system is located. type: string example: Los Angeles diff --git a/functions-python/extract_location/src/reverse_geolocation/location_extractor.py b/functions-python/extract_location/src/reverse_geolocation/location_extractor.py index cfeda7640..14298b48b 100644 --- a/functions-python/extract_location/src/reverse_geolocation/location_extractor.py +++ b/functions-python/extract_location/src/reverse_geolocation/location_extractor.py @@ -9,6 +9,7 @@ Location, Translation, t_feedsearch, + Gtfsfeed, ) from helpers.database import refresh_materialized_view from .geocoded_location import GeocodedLocation @@ -192,15 +193,29 @@ def update_location( if len(locations) == 0: raise Exception("No locations found for the dataset.") + logging.info(f"Updating dataset with stable ID {dataset.stable_id}") dataset.locations.clear() dataset.locations = locations - # Update the location of the related feed as well + # Update the location of the related feeds as well + logging.info(f"Updating feed with stable ID {dataset.feed.stable_id}") dataset.feed.locations.clear() dataset.feed.locations = locations - session.add(dataset) + gtfs_feed: Gtfsfeed | None = ( + session.query(Gtfsfeed) + .filter(Gtfsfeed.stable_id == dataset.feed.stable_id) + .one_or_none() + ) + + for gtfs_rt_feed in gtfs_feed.gtfs_rt_feeds: + logging.info(f"Updating GTFS-RT feed with stable ID {gtfs_rt_feed.stable_id}") + gtfs_rt_feed.locations.clear() + gtfs_rt_feed.locations = locations + session.add(gtfs_rt_feed) + refresh_materialized_view(session, t_feedsearch.name) + session.add(dataset) session.commit() @@ -228,6 +243,7 @@ def get_or_create_location(location: GeocodedLocation, session: Session) -> Loca location_entity.country_code = location.country_code location_entity.municipality = location.municipality location_entity.subdivision_name = location.subdivision_name + session.add(location_entity) return location_entity diff --git a/functions-python/extract_location/tests/test_geocoding.py b/functions-python/extract_location/tests/test_geocoding.py index bd8ebfdcf..7316c9eae 100644 --- a/functions-python/extract_location/tests/test_geocoding.py +++ b/functions-python/extract_location/tests/test_geocoding.py @@ -186,7 +186,7 @@ def test_update_location(self): update_location(location_info, dataset_id, mock_session) - mock_session.add.assert_called_once_with(mock_dataset) + mock_session.add.assert_called_with(mock_dataset) mock_session.commit.assert_called_once() self.assertEqual(mock_dataset.locations[0].country, "日本") diff --git a/functions-python/test_utils/database_utils.py b/functions-python/test_utils/database_utils.py index 96aafa9be..69f4268d3 100644 --- a/functions-python/test_utils/database_utils.py +++ b/functions-python/test_utils/database_utils.py @@ -39,8 +39,9 @@ "geography_columns", "geometry_columns", "spatial_ref_sys", - # Excluding the materialized view + # Excluding the views "feedsearch", + "location_with_translations_en", ] diff --git a/liquibase/changelog.xml b/liquibase/changelog.xml index cdc725f36..e20df7c95 100644 --- a/liquibase/changelog.xml +++ b/liquibase/changelog.xml @@ -26,4 +26,5 @@ + \ No newline at end of file diff --git a/liquibase/changes/feat_622.sql b/liquibase/changes/feat_622.sql new file mode 100644 index 000000000..ba742223f --- /dev/null +++ b/liquibase/changes/feat_622.sql @@ -0,0 +1,29 @@ +DROP VIEW IF EXISTS location_with_translations; +DROP VIEW IF EXISTS location_with_translations_en; +CREATE VIEW location_with_translations_en AS +SELECT + l.id AS location_id, + l.country_code, + l.country, + l.subdivision_name, + l.municipality, + country_translation.value AS country_translation, + subdivision_name_translation.value AS subdivision_name_translation, + municipality_translation.value AS municipality_translation +FROM + location l +LEFT JOIN + translation AS country_translation + ON l.country = country_translation.key + AND country_translation.type = 'country' + AND country_translation.language_code = 'en' +LEFT JOIN + translation AS subdivision_name_translation + ON l.subdivision_name = subdivision_name_translation.key + AND subdivision_name_translation.type = 'subdivision_name' + AND subdivision_name_translation.language_code = 'en' +LEFT JOIN + translation AS municipality_translation + ON l.municipality = municipality_translation.key + AND municipality_translation.type = 'municipality' + AND municipality_translation.language_code = 'en'; \ No newline at end of file diff --git a/web-app/src/app/services/feeds/types.ts b/web-app/src/app/services/feeds/types.ts index 7ac0389d4..ddfd98b28 100644 --- a/web-app/src/app/services/feeds/types.ts +++ b/web-app/src/app/services/feeds/types.ts @@ -333,13 +333,18 @@ export interface components { */ country_code?: string; /** - * @description ISO 3166-2 subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). + * @description The english name of the country where the system is located. + * @example United States + */ + country?: string; + /** + * @description ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2). * * @example California */ subdivision_name?: string; /** - * @description Primary municipality in which the transit system is located. + * @description Primary municipality in english in which the transit system is located. * @example Los Angeles */ municipality?: string;