From daeef8851be8fa774a065edbc6ae6bcc233de602 Mon Sep 17 00:00:00 2001
From: cka-y <60586858+cka-y@users.noreply.github.com>
Date: Tue, 13 Aug 2024 10:44:05 -0400
Subject: [PATCH] feat: automate locations - modified API responses (#661)
---
api/requirements.txt | 3 +-
api/src/feeds/impl/feeds_api_impl.py | 153 +++++++++++-------
api/src/feeds/impl/models/basic_feed_impl.py | 2 +-
api/src/feeds/impl/models/gtfs_feed_impl.py | 12 +-
.../feeds/impl/models/gtfs_rt_feed_impl.py | 11 +-
api/src/feeds/impl/models/location_impl.py | 1 +
.../models/search_feed_item_result_impl.py | 30 ++++
api/src/utils/location_translation.py | 90 +++++++++++
api/tests/test_utils/db_utils.py | 3 +-
.../unittest/models/test_basic_feed_impl.py | 3 +-
.../unittest/models/test_gtfs_feed_impl.py | 8 +-
.../unittest/models/test_location_impl.py | 8 +-
.../test_search_feed_item_result_impl.py | 3 +
docs/DatabaseCatalogAPI.yaml | 8 +-
.../reverse_geolocation/location_extractor.py | 20 ++-
.../extract_location/tests/test_geocoding.py | 2 +-
functions-python/test_utils/database_utils.py | 3 +-
liquibase/changelog.xml | 1 +
liquibase/changes/feat_622.sql | 29 ++++
web-app/src/app/services/feeds/types.ts | 9 +-
20 files changed, 315 insertions(+), 84 deletions(-)
create mode 100644 api/src/utils/location_translation.py
create mode 100644 liquibase/changes/feat_622.sql
diff --git a/api/requirements.txt b/api/requirements.txt
index fd98f2b24..71c34089c 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -47,4 +47,5 @@ cloud-sql-python-connector[pg8000]
fastapi-filter[sqlalchemy]==1.0.0
PyJWT
shapely
-google-cloud-pubsub
\ No newline at end of file
+google-cloud-pubsub
+pycountry
\ No newline at end of file
diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py
index bb20d762b..c7213e75f 100644
--- a/api/src/feeds/impl/feeds_api_impl.py
+++ b/api/src/feeds/impl/feeds_api_impl.py
@@ -1,8 +1,8 @@
from datetime import datetime
-from typing import List, Union
+from typing import List, Union, TypeVar
from sqlalchemy.orm import joinedload
-
+from sqlalchemy.orm.query import Query
from database.database import Database
from database_gen.sqlacodegen_models import (
Feed,
@@ -12,6 +12,7 @@
Location,
Validationreport,
Entitytype,
+ t_location_with_translations_en,
)
from feeds.filters.feed_filter import FeedFilter
from feeds.filters.gtfs_dataset_filter import GtfsDatasetFilter
@@ -36,6 +37,9 @@
from feeds_gen.models.gtfs_feed import GtfsFeed
from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
from utils.date_utils import valid_iso_date
+from utils.location_translation import create_location_translation_object, LocationTranslation
+
+T = TypeVar("T", bound="BasicFeed")
class FeedsApiImpl(BaseFeedsApi):
@@ -91,20 +95,35 @@ def get_gtfs_feed(
id: str,
) -> GtfsFeed:
"""Get the specified gtfs feed from the Mobility Database."""
- feed = (
+ feed, translations = self._get_gtfs_feed(id)
+ if feed:
+ return GtfsFeedImpl.from_orm(feed, translations)
+ else:
+ raise_http_error(404, gtfs_feed_not_found.format(id))
+
+ @staticmethod
+ def _get_gtfs_feed(stable_id: str) -> tuple[Gtfsfeed | None, dict[str, LocationTranslation]]:
+ results = (
FeedFilter(
- stable_id=id,
+ stable_id=stable_id,
status=None,
provider__ilike=None,
producer_url__ilike=None,
)
- .filter(Database().get_query_model(Gtfsfeed))
- .first()
- )
- if feed:
- return GtfsFeedImpl.from_orm(feed)
- else:
- raise_http_error(404, gtfs_feed_not_found.format(id))
+ .filter(Database().get_session().query(Gtfsfeed, t_location_with_translations_en))
+ .outerjoin(Location, Feed.locations)
+ .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id)
+ .options(
+ joinedload(Gtfsfeed.gtfsdatasets)
+ .joinedload(Gtfsdataset.validation_reports)
+ .joinedload(Validationreport.notices),
+ *BasicFeedImpl.get_joinedload_options(),
+ )
+ ).all()
+ if len(results) > 0 and results[0].Gtfsfeed:
+ translations = {result[1]: create_location_translation_object(result) for result in results}
+ return results[0].Gtfsfeed, translations
+ return None, {}
def get_gtfs_feed_datasets(
self,
@@ -176,43 +195,54 @@ def get_gtfs_feeds(
municipality__ilike=municipality,
),
)
- gtfs_feed_query = gtfs_feed_filter.filter(Database().get_query_model(Gtfsfeed))
-
- gtfs_feed_query = gtfs_feed_query.outerjoin(Location, Feed.locations).options(
- joinedload(Gtfsfeed.gtfsdatasets)
- .joinedload(Gtfsdataset.validation_reports)
- .joinedload(Validationreport.notices),
- *BasicFeedImpl.get_joinedload_options(),
+ gtfs_feed_query = gtfs_feed_filter.filter(
+ Database().get_session().query(Gtfsfeed, t_location_with_translations_en)
+ )
+ gtfs_feed_query = (
+ gtfs_feed_query.outerjoin(Location, Feed.locations)
+ .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id)
+ .options(
+ joinedload(Gtfsfeed.gtfsdatasets)
+ .joinedload(Gtfsdataset.validation_reports)
+ .joinedload(Validationreport.notices),
+ *BasicFeedImpl.get_joinedload_options(),
+ )
+ .order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
)
gtfs_feed_query = gtfs_feed_query.order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
gtfs_feed_query = DatasetsApiImpl.apply_bounding_filtering(
gtfs_feed_query, dataset_latitudes, dataset_longitudes, bounding_filter_method
)
- if limit is not None:
- gtfs_feed_query = gtfs_feed_query.limit(limit)
- if offset is not None:
- gtfs_feed_query = gtfs_feed_query.offset(offset)
- results = gtfs_feed_query.all()
- return [GtfsFeedImpl.from_orm(gtfs_feed) for gtfs_feed in results]
+ return self._get_response(gtfs_feed_query, limit, offset, GtfsFeedImpl)
def get_gtfs_rt_feed(
self,
id: str,
) -> GtfsRTFeed:
"""Get the specified GTFS Realtime feed from the Mobility Database."""
- feed = (
- GtfsRtFeedFilter(
- stable_id=id,
- provider__ilike=None,
- producer_url__ilike=None,
- entity_types=None,
- location=None,
- )
- .filter(Database().get_query_model(Gtfsrealtimefeed))
- .first()
+ gtfs_rt_feed_filter = GtfsRtFeedFilter(
+ stable_id=id,
+ provider__ilike=None,
+ producer_url__ilike=None,
+ entity_types=None,
+ location=None,
)
- if feed:
- return GtfsRTFeedImpl.from_orm(feed)
+ results = gtfs_rt_feed_filter.filter(
+ Database()
+ .get_session()
+ .query(Gtfsrealtimefeed, t_location_with_translations_en)
+ .outerjoin(Location, Gtfsrealtimefeed.locations)
+ .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id)
+ .options(
+ joinedload(Gtfsrealtimefeed.entitytypes),
+ joinedload(Gtfsrealtimefeed.gtfs_feeds),
+ *BasicFeedImpl.get_joinedload_options(),
+ )
+ ).all()
+
+ if len(results) > 0 and results[0].Gtfsrealtimefeed:
+ translations = {result[1]: create_location_translation_object(result) for result in results}
+ return GtfsRTFeedImpl.from_orm(results[0].Gtfsrealtimefeed, translations)
else:
raise_http_error(404, gtfs_rt_feed_not_found.format(id))
@@ -251,42 +281,41 @@ def get_gtfs_rt_feeds(
municipality__ilike=municipality,
),
)
- gtfs_rt_feed_query = gtfs_rt_feed_filter.filter(Database().get_query_model(Gtfsrealtimefeed)).options(
- *BasicFeedImpl.get_joinedload_options()
+ gtfs_rt_feed_query = gtfs_rt_feed_filter.filter(
+ Database().get_session().query(Gtfsrealtimefeed, t_location_with_translations_en)
)
- gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Entitytype, Gtfsrealtimefeed.entitytypes).options(
- joinedload(Gtfsrealtimefeed.entitytypes)
- )
- gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Location, Feed.locations).options(
- joinedload(Gtfsrealtimefeed.locations)
- )
- gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Gtfsfeed, Gtfsrealtimefeed.gtfs_feeds).options(
- joinedload(Gtfsrealtimefeed.gtfs_feeds)
+ gtfs_rt_feed_query = (
+ gtfs_rt_feed_query.outerjoin(Location, Gtfsrealtimefeed.locations)
+ .outerjoin(t_location_with_translations_en, Location.id == t_location_with_translations_en.c.location_id)
+ .outerjoin(Entitytype, Gtfsrealtimefeed.entitytypes)
+ .options(
+ joinedload(Gtfsrealtimefeed.entitytypes),
+ joinedload(Gtfsrealtimefeed.gtfs_feeds),
+ *BasicFeedImpl.get_joinedload_options(),
+ )
+ .order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id)
)
- gtfs_rt_feed_query = gtfs_rt_feed_query.order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id)
+ return self._get_response(gtfs_rt_feed_query, limit, offset, GtfsRTFeedImpl)
+
+ @staticmethod
+ def _get_response(feed_query: Query, limit: int, offset: int, impl_cls: type[T]) -> List[T]:
+ """Get the response for the feed query."""
if limit is not None:
- gtfs_rt_feed_query = gtfs_rt_feed_query.limit(limit)
+ feed_query = feed_query.limit(limit)
if offset is not None:
- gtfs_rt_feed_query = gtfs_rt_feed_query.offset(offset)
- results = gtfs_rt_feed_query.all()
- return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed) for gtfs_rt_feed in results]
+ feed_query = feed_query.offset(offset)
+ results = feed_query.all()
+ location_translations = {row[1]: create_location_translation_object(row) for row in results}
+ response = [impl_cls.from_orm(feed[0], location_translations) for feed in results]
+ return list({feed.id: feed for feed in response}.values())
def get_gtfs_feed_gtfs_rt_feeds(
self,
id: str,
) -> List[GtfsRTFeed]:
"""Get a list of GTFS Realtime related to a GTFS feed."""
- feed = (
- FeedFilter(
- stable_id=id,
- status=None,
- provider__ilike=None,
- producer_url__ilike=None,
- )
- .filter(Database().get_query_model(Gtfsfeed))
- .first()
- )
+ feed, translations = self._get_gtfs_feed(id)
if feed:
- return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed) for gtfs_rt_feed in feed.gtfs_rt_feeds]
+ return [GtfsRTFeedImpl.from_orm(gtfs_rt_feed, translations) for gtfs_rt_feed in feed.gtfs_rt_feeds]
else:
raise_http_error(404, gtfs_feed_not_found.format(id))
diff --git a/api/src/feeds/impl/models/basic_feed_impl.py b/api/src/feeds/impl/models/basic_feed_impl.py
index 3f3512b37..cf6bf87e3 100644
--- a/api/src/feeds/impl/models/basic_feed_impl.py
+++ b/api/src/feeds/impl/models/basic_feed_impl.py
@@ -20,7 +20,7 @@ class Config:
from_attributes = True
@classmethod
- def from_orm(cls, feed: Feed | None) -> BasicFeed | None:
+ def from_orm(cls, feed: Feed | None, _=None) -> BasicFeed | None:
if not feed:
return None
return cls(
diff --git a/api/src/feeds/impl/models/gtfs_feed_impl.py b/api/src/feeds/impl/models/gtfs_feed_impl.py
index c1d9fd825..ded4d00ff 100644
--- a/api/src/feeds/impl/models/gtfs_feed_impl.py
+++ b/api/src/feeds/impl/models/gtfs_feed_impl.py
@@ -1,8 +1,11 @@
+from typing import Dict
+
from database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm
from feeds.impl.models.basic_feed_impl import BaseFeedImpl
from feeds.impl.models.latest_dataset_impl import LatestDatasetImpl
from feeds.impl.models.location_impl import LocationImpl
from feeds_gen.models.gtfs_feed import GtfsFeed
+from utils.location_translation import LocationTranslation, translate_feed_locations
class GtfsFeedImpl(BaseFeedImpl, GtfsFeed):
@@ -17,12 +20,15 @@ class Config:
from_attributes = True
@classmethod
- def from_orm(cls, feed: GtfsfeedOrm | None) -> GtfsFeed | None:
- gtfs_feed = super().from_orm(feed)
+ def from_orm(
+ cls, feed: GtfsfeedOrm | None, location_translations: Dict[str, LocationTranslation] = None
+ ) -> GtfsFeed | None:
+ if location_translations is not None:
+ translate_feed_locations(feed, location_translations)
+ gtfs_feed: GtfsFeed = super().from_orm(feed)
if not gtfs_feed:
return None
gtfs_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations]
-
latest_dataset = next(
(dataset for dataset in feed.gtfsdatasets if dataset is not None and dataset.latest), None
)
diff --git a/api/src/feeds/impl/models/gtfs_rt_feed_impl.py b/api/src/feeds/impl/models/gtfs_rt_feed_impl.py
index a46169deb..d19956762 100644
--- a/api/src/feeds/impl/models/gtfs_rt_feed_impl.py
+++ b/api/src/feeds/impl/models/gtfs_rt_feed_impl.py
@@ -1,7 +1,10 @@
+from typing import Dict
+
from database_gen.sqlacodegen_models import Gtfsrealtimefeed as GtfsRTFeedOrm
from feeds.impl.models.basic_feed_impl import BaseFeedImpl
from feeds.impl.models.location_impl import LocationImpl
from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
+from utils.location_translation import LocationTranslation, translate_feed_locations
class GtfsRTFeedImpl(BaseFeedImpl, GtfsRTFeed):
@@ -14,8 +17,12 @@ class Config:
from_attributes = True
@classmethod
- def from_orm(cls, feed: GtfsRTFeedOrm | None) -> GtfsRTFeed | None:
- gtfs_rt_feed = super().from_orm(feed)
+ def from_orm(
+ cls, feed: GtfsRTFeedOrm | None, location_translations: Dict[str, LocationTranslation] = None
+ ) -> GtfsRTFeed | None:
+ if location_translations is not None:
+ translate_feed_locations(feed, location_translations)
+ gtfs_rt_feed: GtfsRTFeed = super().from_orm(feed)
if not gtfs_rt_feed:
return None
gtfs_rt_feed.locations = [LocationImpl.from_orm(item) for item in feed.locations]
diff --git a/api/src/feeds/impl/models/location_impl.py b/api/src/feeds/impl/models/location_impl.py
index cb60df905..385aab593 100644
--- a/api/src/feeds/impl/models/location_impl.py
+++ b/api/src/feeds/impl/models/location_impl.py
@@ -16,6 +16,7 @@ def from_orm(cls, location: LocationOrm | None) -> Location | None:
return None
return cls(
country_code=location.country_code,
+ country=location.country,
subdivision_name=location.subdivision_name,
municipality=location.municipality,
)
diff --git a/api/src/feeds/impl/models/search_feed_item_result_impl.py b/api/src/feeds/impl/models/search_feed_item_result_impl.py
index c0b218c95..3fa3eb2bb 100644
--- a/api/src/feeds/impl/models/search_feed_item_result_impl.py
+++ b/api/src/feeds/impl/models/search_feed_item_result_impl.py
@@ -1,6 +1,7 @@
from feeds_gen.models.latest_dataset import LatestDataset
from feeds_gen.models.search_feed_item_result import SearchFeedItemResult
from feeds_gen.models.source_info import SourceInfo
+import pycountry
class SearchFeedItemResultImpl(SearchFeedItemResult):
@@ -74,11 +75,40 @@ def from_orm_gtfs_rt(cls, feed_search_row):
feed_references=feed_search_row.feed_reference_ids,
)
+ @classmethod
+ def _translate_locations(cls, feed_search_row):
+ """Translate location information in the feed search row."""
+ country_translations = cls._create_translation_dict(feed_search_row.country_translations)
+ subdivision_translations = cls._create_translation_dict(feed_search_row.subdivision_name_translations)
+ municipality_translations = cls._create_translation_dict(feed_search_row.municipality_translations)
+
+ for location in feed_search_row.locations:
+ location["country"] = country_translations.get(location["country"], location["country"])
+ if location["country"] is None:
+ location["country"] = pycountry.countries.get(alpha_2=location["country_code"]).name
+ location["subdivision_name"] = subdivision_translations.get(
+ location["subdivision_name"], location["subdivision_name"]
+ )
+ location["municipality"] = municipality_translations.get(location["municipality"], location["municipality"])
+
+ @staticmethod
+ def _create_translation_dict(translations):
+ """Helper method to create a translation dictionary."""
+ if translations:
+ return {
+ elem.get("key"): elem.get("value") for elem in translations if elem.get("key") and elem.get("value")
+ }
+ return {}
+
@classmethod
def from_orm(cls, feed_search_row):
"""Create a model instance from a SQLAlchemy row object."""
if feed_search_row is None:
return None
+
+ # Translate location data
+ cls._translate_locations(feed_search_row)
+
match feed_search_row.data_type:
case "gtfs":
return cls.from_orm_gtfs(feed_search_row)
diff --git a/api/src/utils/location_translation.py b/api/src/utils/location_translation.py
new file mode 100644
index 000000000..68f94046b
--- /dev/null
+++ b/api/src/utils/location_translation.py
@@ -0,0 +1,90 @@
+import pycountry
+from sqlalchemy.engine.result import Row
+from database_gen.sqlacodegen_models import Location as LocationOrm
+from database_gen.sqlacodegen_models import Feed as FeedOrm
+
+
+class LocationTranslation:
+ def __init__(
+ self,
+ location_id,
+ country_code,
+ country,
+ subdivision_name,
+ municipality,
+ country_translation,
+ subdivision_name_translation,
+ municipality_translation,
+ ):
+ self.location_id = location_id
+ self.country_code = country_code
+ self.country = country
+ self.subdivision_name = subdivision_name
+ self.municipality = municipality
+ self.country_translation = country_translation
+ self.subdivision_name_translation = subdivision_name_translation
+ self.municipality_translation = municipality_translation
+
+ def __repr__(self):
+ return (
+ f"LocationTranslation(location_id={self.location_id}, country_code={self.country_code}, "
+ f"country={self.country}, subdivision_name={self.subdivision_name}, municipality={self.municipality}, "
+ f"country_translation={self.country_translation}, subdivision_name_translation="
+ f"{self.subdivision_name_translation}, municipality_translation={self.municipality_translation})"
+ )
+
+
+def create_location_translation_object(row: Row):
+ """Create a location translation object from a row."""
+ return LocationTranslation(
+ location_id=row[1],
+ country_code=row[2],
+ country=row[3],
+ subdivision_name=row[4],
+ municipality=row[5],
+ country_translation=row[6],
+ subdivision_name_translation=row[7],
+ municipality_translation=row[8],
+ )
+
+
+def set_country_name(location: LocationOrm, country_name: str) -> LocationOrm:
+ """
+ Set the country name of a location
+ :param location: The location object
+ :param country_name: The english translation of the country name
+ :return: Modified location object
+ """
+ try:
+ if country_name is not None:
+ location.country = country_name
+ else:
+ location.country = pycountry.countries.get(alpha_2=location.country_code).name
+ except AttributeError:
+ pass
+ return location
+
+
+def translate_feed_locations(feed: FeedOrm, location_translations: dict[str, LocationTranslation]):
+ """
+ Translate the locations of a feed.
+ :param feed: The feed object
+ :param location_translations: The location translations
+ """
+ for location in feed.locations:
+ location_translation = location_translations.get(location.id)
+ location = set_country_name(
+ location, location_translation.country_translation if location_translation else None
+ )
+ if location_translation:
+ location.country_code = location_translation.country_code
+ location.subdivision_name = (
+ location_translation.subdivision_name_translation
+ if location_translation.subdivision_name_translation
+ else location.subdivision_name
+ )
+ location.municipality = (
+ location_translation.municipality_translation
+ if location_translation.municipality_translation
+ else location.municipality
+ )
diff --git a/api/tests/test_utils/db_utils.py b/api/tests/test_utils/db_utils.py
index 06a93e86d..633a0d414 100644
--- a/api/tests/test_utils/db_utils.py
+++ b/api/tests/test_utils/db_utils.py
@@ -184,8 +184,9 @@ def is_test_db(url):
"geography_columns",
"geometry_columns",
"spatial_ref_sys",
- # Excluding the materialized view
+ # Excluding the views
"feedsearch",
+ "location_with_translations_en",
]
diff --git a/api/tests/unittest/models/test_basic_feed_impl.py b/api/tests/unittest/models/test_basic_feed_impl.py
index f5d567752..95881a866 100644
--- a/api/tests/unittest/models/test_basic_feed_impl.py
+++ b/api/tests/unittest/models/test_basic_feed_impl.py
@@ -41,7 +41,8 @@
locations=[
Location(
id="id",
- country_code="country_code",
+ country_code="CA",
+ country=None,
subdivision_name="subdivision_name",
municipality="municipality",
)
diff --git a/api/tests/unittest/models/test_gtfs_feed_impl.py b/api/tests/unittest/models/test_gtfs_feed_impl.py
index ea96ddd0a..7c902eb59 100644
--- a/api/tests/unittest/models/test_gtfs_feed_impl.py
+++ b/api/tests/unittest/models/test_gtfs_feed_impl.py
@@ -65,7 +65,8 @@ def create_test_notice(notice_code: str, total_notices: int, severity: str):
locations=[
Location(
id="id",
- country_code="country_code",
+ country_code="CA",
+ country=None,
subdivision_name="subdivision_name",
municipality="municipality",
)
@@ -146,7 +147,8 @@ def create_test_notice(notice_code: str, total_notices: int, severity: str):
],
locations=[
LocationImpl(
- country_code="country_code",
+ country_code="CA",
+ country="Canada",
subdivision_name="subdivision_name",
municipality="municipality",
)
@@ -176,7 +178,7 @@ class TestGtfsFeedImpl(unittest.TestCase):
def test_from_orm_all_fields(self):
"""Test the `from_orm` method with all fields."""
- result = GtfsFeedImpl.from_orm(gtfs_feed_orm)
+ result = GtfsFeedImpl.from_orm(gtfs_feed_orm, {})
assert result == expected_gtfs_feed_result
def test_from_orm_empty_fields(self):
diff --git a/api/tests/unittest/models/test_location_impl.py b/api/tests/unittest/models/test_location_impl.py
index a0d39ed29..24ba78529 100644
--- a/api/tests/unittest/models/test_location_impl.py
+++ b/api/tests/unittest/models/test_location_impl.py
@@ -7,8 +7,12 @@
class TestLocationImpl(unittest.TestCase):
def test_from_orm(self):
result = LocationImpl.from_orm(
- LocationOrm(country_code="US", subdivision_name="California", municipality="Los Angeles")
+ LocationOrm(
+ country_code="US", subdivision_name="California", municipality="Los Angeles", country="United States"
+ )
+ )
+ assert result == LocationImpl(
+ country_code="US", country="United States", subdivision_name="California", municipality="Los Angeles"
)
- assert result == LocationImpl(country_code="US", subdivision_name="California", municipality="Los Angeles")
assert LocationImpl.from_orm(None) is None
diff --git a/api/tests/unittest/models/test_search_feed_item_result_impl.py b/api/tests/unittest/models/test_search_feed_item_result_impl.py
index 729a13e8c..6ce31163e 100644
--- a/api/tests/unittest/models/test_search_feed_item_result_impl.py
+++ b/api/tests/unittest/models/test_search_feed_item_result_impl.py
@@ -43,6 +43,9 @@ def __init__(self, **kwargs):
feed_reference_ids=[],
entities=["sa"],
locations=[],
+ country_translations=[],
+ subdivision_name_translations=[],
+ municipality_translations=[],
)
diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml
index 76850e63d..467c7dfd5 100644
--- a/docs/DatabaseCatalogAPI.yaml
+++ b/docs/DatabaseCatalogAPI.yaml
@@ -677,14 +677,18 @@ components:
For a list of valid codes [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2).
type: string
example: US
+ country:
+ description: The english name of the country where the system is located.
+ type: string
+ example: United States
subdivision_name:
description: >
- ISO 3166-2 subdivision name designating the subdivision (e.g province, state, region) where the system is located.
+ ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located.
For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2).
type: string
example: California
municipality:
- description: Primary municipality in which the transit system is located.
+ description: Primary municipality in english in which the transit system is located.
type: string
example: Los Angeles
diff --git a/functions-python/extract_location/src/reverse_geolocation/location_extractor.py b/functions-python/extract_location/src/reverse_geolocation/location_extractor.py
index cfeda7640..14298b48b 100644
--- a/functions-python/extract_location/src/reverse_geolocation/location_extractor.py
+++ b/functions-python/extract_location/src/reverse_geolocation/location_extractor.py
@@ -9,6 +9,7 @@
Location,
Translation,
t_feedsearch,
+ Gtfsfeed,
)
from helpers.database import refresh_materialized_view
from .geocoded_location import GeocodedLocation
@@ -192,15 +193,29 @@ def update_location(
if len(locations) == 0:
raise Exception("No locations found for the dataset.")
+ logging.info(f"Updating dataset with stable ID {dataset.stable_id}")
dataset.locations.clear()
dataset.locations = locations
- # Update the location of the related feed as well
+ # Update the location of the related feeds as well
+ logging.info(f"Updating feed with stable ID {dataset.feed.stable_id}")
dataset.feed.locations.clear()
dataset.feed.locations = locations
- session.add(dataset)
+ gtfs_feed: Gtfsfeed | None = (
+ session.query(Gtfsfeed)
+ .filter(Gtfsfeed.stable_id == dataset.feed.stable_id)
+ .one_or_none()
+ )
+
+ for gtfs_rt_feed in gtfs_feed.gtfs_rt_feeds:
+ logging.info(f"Updating GTFS-RT feed with stable ID {gtfs_rt_feed.stable_id}")
+ gtfs_rt_feed.locations.clear()
+ gtfs_rt_feed.locations = locations
+ session.add(gtfs_rt_feed)
+
refresh_materialized_view(session, t_feedsearch.name)
+ session.add(dataset)
session.commit()
@@ -228,6 +243,7 @@ def get_or_create_location(location: GeocodedLocation, session: Session) -> Loca
location_entity.country_code = location.country_code
location_entity.municipality = location.municipality
location_entity.subdivision_name = location.subdivision_name
+ session.add(location_entity)
return location_entity
diff --git a/functions-python/extract_location/tests/test_geocoding.py b/functions-python/extract_location/tests/test_geocoding.py
index bd8ebfdcf..7316c9eae 100644
--- a/functions-python/extract_location/tests/test_geocoding.py
+++ b/functions-python/extract_location/tests/test_geocoding.py
@@ -186,7 +186,7 @@ def test_update_location(self):
update_location(location_info, dataset_id, mock_session)
- mock_session.add.assert_called_once_with(mock_dataset)
+ mock_session.add.assert_called_with(mock_dataset)
mock_session.commit.assert_called_once()
self.assertEqual(mock_dataset.locations[0].country, "日本")
diff --git a/functions-python/test_utils/database_utils.py b/functions-python/test_utils/database_utils.py
index 96aafa9be..69f4268d3 100644
--- a/functions-python/test_utils/database_utils.py
+++ b/functions-python/test_utils/database_utils.py
@@ -39,8 +39,9 @@
"geography_columns",
"geometry_columns",
"spatial_ref_sys",
- # Excluding the materialized view
+ # Excluding the views
"feedsearch",
+ "location_with_translations_en",
]
diff --git a/liquibase/changelog.xml b/liquibase/changelog.xml
index cdc725f36..e20df7c95 100644
--- a/liquibase/changelog.xml
+++ b/liquibase/changelog.xml
@@ -26,4 +26,5 @@
+
\ No newline at end of file
diff --git a/liquibase/changes/feat_622.sql b/liquibase/changes/feat_622.sql
new file mode 100644
index 000000000..ba742223f
--- /dev/null
+++ b/liquibase/changes/feat_622.sql
@@ -0,0 +1,29 @@
+DROP VIEW IF EXISTS location_with_translations;
+DROP VIEW IF EXISTS location_with_translations_en;
+CREATE VIEW location_with_translations_en AS
+SELECT
+ l.id AS location_id,
+ l.country_code,
+ l.country,
+ l.subdivision_name,
+ l.municipality,
+ country_translation.value AS country_translation,
+ subdivision_name_translation.value AS subdivision_name_translation,
+ municipality_translation.value AS municipality_translation
+FROM
+ location l
+LEFT JOIN
+ translation AS country_translation
+ ON l.country = country_translation.key
+ AND country_translation.type = 'country'
+ AND country_translation.language_code = 'en'
+LEFT JOIN
+ translation AS subdivision_name_translation
+ ON l.subdivision_name = subdivision_name_translation.key
+ AND subdivision_name_translation.type = 'subdivision_name'
+ AND subdivision_name_translation.language_code = 'en'
+LEFT JOIN
+ translation AS municipality_translation
+ ON l.municipality = municipality_translation.key
+ AND municipality_translation.type = 'municipality'
+ AND municipality_translation.language_code = 'en';
\ No newline at end of file
diff --git a/web-app/src/app/services/feeds/types.ts b/web-app/src/app/services/feeds/types.ts
index 7ac0389d4..ddfd98b28 100644
--- a/web-app/src/app/services/feeds/types.ts
+++ b/web-app/src/app/services/feeds/types.ts
@@ -333,13 +333,18 @@ export interface components {
*/
country_code?: string;
/**
- * @description ISO 3166-2 subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2).
+ * @description The english name of the country where the system is located.
+ * @example United States
+ */
+ country?: string;
+ /**
+ * @description ISO 3166-2 english subdivision name designating the subdivision (e.g province, state, region) where the system is located. For a list of valid names [see here](https://unece.org/trade/uncefact/unlocode-country-subdivisions-iso-3166-2).
*
* @example California
*/
subdivision_name?: string;
/**
- * @description Primary municipality in which the transit system is located.
+ * @description Primary municipality in english in which the transit system is located.
* @example Los Angeles
*/
municipality?: string;