From c761c169eb9e4e9719b1e1b3dd302dc02af15c19 Mon Sep 17 00:00:00 2001 From: aronza Date: Sun, 20 Aug 2023 11:40:01 -0400 Subject: [PATCH] feat: implement feeds endpoint (#79) * Develop feeds endpoint #32 * Develop feeds/{id} endpoint #35 * Develop metadata endpoint #51 --- api/src/database/database.py | 11 +++++++- api/src/feeds/impl/feeds_api_impl.py | 34 ++++++++++++++++++++++--- api/src/feeds/impl/metadata_api_impl.py | 2 +- api/tests/test_datasets_api.py | 14 ++++------ api/tests/test_feeds_api.py | 21 ++++++--------- api/tests/test_metadata_api.py | 7 ++--- docs/DatabaseCatalogAPI.yaml | 8 +++--- scripts/api-start.sh | 3 +-- 8 files changed, 61 insertions(+), 39 deletions(-) diff --git a/api/src/database/database.py b/api/src/database/database.py index a9cb45b91..4b7f54548 100644 --- a/api/src/database/database.py +++ b/api/src/database/database.py @@ -1,5 +1,6 @@ import os import uuid +from typing import Type from google.cloud.sql.connector import Connector from sqlalchemy import create_engine, inspect @@ -88,13 +89,16 @@ def close_session(self): self.logger.error(f"Session closing failed with exception: \n {e}") return self.is_connected() - def select(self, model: Base, conditions: list = None, attributes: list = None, update_session: bool = True): + def select(self, model: Type[Base], conditions: list = None, attributes: list = None, update_session: bool = True, + limit: int = None, offset: int = None): """ Executes a query on the database :param model: the sqlalchemy model to query :param conditions: list of conditions (filters for the query) :param attributes: list of model's attribute names that you want to fetch. If not given, fetches all attributes. :param update_session: option to update session before running the query (defaults to True) + :param limit: the optional number of rows to limit the query with + :param offset: the optional number of rows to offset the query with :return: None if database is inaccessible, the results of the query otherwise """ try: @@ -106,6 +110,10 @@ def select(self, model: Base, conditions: list = None, attributes: list = None, query = query.filter(condition) if attributes: query = query.options(load_only(*attributes)) + if limit is not None: + query = query.limit(limit) + if offset is not None: + query = query.offset(offset) return query.all() except Exception as e: self.logger.error(f'SELECT query failed with exception: \n{e}') @@ -186,3 +194,4 @@ def merge_relationship( return False +DB_ENGINE = Database() diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py index 79b219d59..2178c65f2 100644 --- a/api/src/feeds/impl/feeds_api_impl.py +++ b/api/src/feeds/impl/feeds_api_impl.py @@ -1,9 +1,13 @@ from datetime import date from typing import List +from fastapi import HTTPException + +from database_gen.sqlacodegen_models import Feed, Externalid, t_redirectingid from feeds_gen.apis.feeds_api_base import BaseFeedsApi from feeds_gen.models.basic_feed import BasicFeed from feeds_gen.models.bounding_box import BoundingBox +from feeds_gen.models.external_id import ExternalId from feeds_gen.models.extra_models import TokenModel from feeds_gen.models.feed_log import FeedLog from feeds_gen.models.gtfs_dataset import GtfsDataset @@ -12,6 +16,8 @@ from feeds_gen.models.latest_dataset import LatestDataset from feeds_gen.models.source_info import SourceInfo +from database.database import DB_ENGINE + class FeedsApiImpl(BaseFeedsApi): """ @@ -20,13 +26,34 @@ class FeedsApiImpl(BaseFeedsApi): If a method is left blank the associated endpoint will return a 500 HTTP response. """ + @staticmethod + def map_feed(feed: Feed): + """ + Maps sqlalchemy data model Feed to API data model BasicFeed + """ + redirects = DB_ENGINE.select(t_redirectingid, conditions=[feed.id == t_redirectingid.c.source_id]) + external_ids = DB_ENGINE.select(Externalid, conditions=[feed.id == Externalid.feed_id]) + + return BasicFeed(id=feed.stable_id, data_type=feed.data_type, status=feed.status, + feed_name=feed.feed_name, note=feed.note, provider=feed.provider, + redirects=[redirect.target_id for redirect in redirects], + external_ids=[ExternalId(external_id=ext_id.associated_id, source=ext_id.source) + for ext_id in external_ids], + source_info=SourceInfo(producer_url=feed.producer_url, + authentication_type=feed.authentication_type, + authentication_info_url=feed.authentication_info_url, + api_key_parameter_name=feed.api_key_parameter_name, + license_url=feed.license_url)) + def get_feed( self, id: str, ) -> BasicFeed: """Get the specified feed from the Mobility Database.""" - return BasicFeed(id="gtfsFeedFoo", data_type=None, status=None, external_ids=[], provider="providerFoo", - feed_name="feedFoo", note="note", source_info=SourceInfo()) + feeds = DB_ENGINE.select(Feed, conditions=[Feed.stable_id == id]) + if len(feeds) == 1: + return self.map_feed(feeds[0]) + raise HTTPException(status_code=404, detail=f"Feed {id} not found") def get_feed_logs( id: str, @@ -47,7 +74,7 @@ def get_feeds( sort: str, ) -> List[BasicFeed]: """Get some (or all) feeds from the Mobility Database.""" - return [self.get_feed("gtfsFeedFoo")] + return [self.map_feed(feed) for feed in DB_ENGINE.select(Feed, limit=limit, offset=offset)] def get_gtfs_feed( self, @@ -85,6 +112,7 @@ def get_gtfs_feeds( bounding_filter_method: str, ) -> List[GtfsFeed]: """Get some (or all) GTFS feeds from the Mobility Database.""" + print("In get_gtfs_feeds endpoint") return [self.get_gtfs_feed("foo")] def get_gtfs_rt_feed( diff --git a/api/src/feeds/impl/metadata_api_impl.py b/api/src/feeds/impl/metadata_api_impl.py index f8f1a1952..67961b260 100644 --- a/api/src/feeds/impl/metadata_api_impl.py +++ b/api/src/feeds/impl/metadata_api_impl.py @@ -12,4 +12,4 @@ def get_metadata( self, ) -> Metadata: """Get metadata about this API.""" - return Metadata(version="0.0.0") \ No newline at end of file + return Metadata(version="1.0.0") diff --git a/api/tests/test_datasets_api.py b/api/tests/test_datasets_api.py index 6405e4f83..e2acde6f2 100644 --- a/api/tests/test_datasets_api.py +++ b/api/tests/test_datasets_api.py @@ -2,10 +2,6 @@ from fastapi.testclient import TestClient - -from feeds.models.dataset import Dataset # noqa: F401 - - def test_datasets_gtfs_get(client: TestClient): """Test case for datasets_gtfs_get @@ -17,7 +13,7 @@ def test_datasets_gtfs_get(client: TestClient): } response = client.request( "GET", - "/datasets/gtfs", + "/v1/datasets/gtfs", headers=headers, params=params, ) @@ -37,12 +33,12 @@ def test_datasets_gtfs_id_get(client: TestClient): } response = client.request( "GET", - "/datasets/gtfs/{id}".format(id='dataset_0'), + "/v1/datasets/gtfs/{id}".format(id='dataset_0'), headers=headers, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 def test_feeds_gtfs_id_datasets_get(client: TestClient): @@ -56,11 +52,11 @@ def test_feeds_gtfs_id_datasets_get(client: TestClient): } response = client.request( "GET", - "/feeds/gtfs/{id}/datasets".format(id='feed_0'), + "/v1/feeds/gtfs/{id}/datasets".format(id='feed_0'), headers=headers, params=params, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 diff --git a/api/tests/test_feeds_api.py b/api/tests/test_feeds_api.py index 355d504d4..1d5ec45ce 100644 --- a/api/tests/test_feeds_api.py +++ b/api/tests/test_feeds_api.py @@ -3,13 +3,8 @@ from fastapi.testclient import TestClient -from feeds.models.basic_feed import BasicFeed # noqa: F401 -from feeds.models.gtfs_feed import GtfsFeed # noqa: F401 - - def test_feeds_get(client: TestClient): """Test case for feeds_get - """ params = [("limit", 10), ("offset", 0), ("filter", 'status=active'), ("sort", '+provider')] @@ -18,13 +13,13 @@ def test_feeds_get(client: TestClient): } response = client.request( "GET", - "/feeds", + "/v1/feeds", headers=headers, params=params, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 def test_feeds_gtfs_get(client: TestClient): @@ -38,13 +33,13 @@ def test_feeds_gtfs_get(client: TestClient): } response = client.request( "GET", - "/feeds/gtfs", + "/v1/feeds/gtfs", headers=headers, params=params, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 def test_feeds_gtfs_id_get(client: TestClient): @@ -58,12 +53,12 @@ def test_feeds_gtfs_id_get(client: TestClient): } response = client.request( "GET", - "/feeds/gtfs/{id}".format(id='feed_0'), + "/v1/feeds/gtfs/{id}".format(id='feed_0'), headers=headers, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 def test_feeds_id_get(client: TestClient): @@ -77,10 +72,10 @@ def test_feeds_id_get(client: TestClient): } response = client.request( "GET", - "/feeds/{id}".format(id='feed_0'), + "/v1/feeds/{id}".format(id='feed_0'), headers=headers, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 diff --git a/api/tests/test_metadata_api.py b/api/tests/test_metadata_api.py index 98195799f..ba9d91f5b 100644 --- a/api/tests/test_metadata_api.py +++ b/api/tests/test_metadata_api.py @@ -3,9 +3,6 @@ from fastapi.testclient import TestClient -from feeds.models.metadata import Metadata # noqa: F401 - - def test_metadata_get(client: TestClient): """Test case for metadata_get @@ -17,10 +14,10 @@ def test_metadata_get(client: TestClient): } response = client.request( "GET", - "/metadata", + "/v1/metadata", headers=headers, ) # uncomment below to assert the status code of the HTTP response - #assert response.status_code == 200 + assert response.status_code == 200 diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 65c3cf8a1..61396790c 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -277,12 +277,10 @@ components: external_ids: $ref: "#/components/schemas/ExternalIds" - providers: + provider: description: A commonly used name for the transit provider included in the feed. - type: array - items: - type: string - example: London Transit Commission + type: string + example: London Transit Commission feed_name: description: > An optional description of the data feed, e.g to specify if the data feed is an aggregate of diff --git a/scripts/api-start.sh b/scripts/api-start.sh index 7a791d4d6..82300363a 100755 --- a/scripts/api-start.sh +++ b/scripts/api-start.sh @@ -5,5 +5,4 @@ # relative path SCRIPT_PATH="$(dirname -- "${BASH_SOURCE[0]}")" PORT=8080 - -(cd $SCRIPT_PATH/../api/src && uvicorn feeds_gen.main:app --host 0.0.0.0 --port $PORT) \ No newline at end of file +(cd $SCRIPT_PATH/../api/src && uvicorn feeds_gen.main:app --host 0.0.0.0 --port $PORT --env-file ../../config/.env.local) \ No newline at end of file