From c52325488bb88f80c8fca578716622846cec4932 Mon Sep 17 00:00:00 2001 From: remyvdwereld <86827854+remyvdwereld@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:57:27 +0100 Subject: [PATCH 1/2] Added api-key for BAG API v1 --- README.md | 4 +- app/apps/fines/api_queries_belastingen.py | 34 ++++-- app/apps/health/health_checks.py | 115 +++++++++++------- app/config/settings.py | 6 + app/utils/api_queries_bag.py | 4 + app/utils/api_queries_toeristische_verhuur.py | 38 +++--- 6 files changed, 130 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 0b20a666f..eea33fded 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Zakensysteem voor toezichthouders en handhavers van de Gemeente Amsterdam voor d - [Prerequisites](#prerequisites) - [Getting up and running (Local development only)](#getting-up-and-running-local-development-only) - [Running tests](#running-tests) - - [Accessing the API documentation](#accessing-the-api-documentation) + - [API documentation (Swagger)](#api-documentation-swagger) - [Generating an access token](#generating-an-access-token) - [Enabling local development environment variables](#enabling-local-development-environment-variables) - [Enabling Keycloak authentication for a locally run zaken-frontend](#enabling-keycloak-authentication-for-a-locally-run-zaken-frontend) @@ -88,7 +88,7 @@ To run tests for a specific module, add a path: docker compose run --rm zaak-gateway python manage.py test apps/cases ``` -## Accessing the API documentation +## API documentation (Swagger) You can access the documentation at: http://localhost:8080/api/v1/swagger/ diff --git a/app/apps/fines/api_queries_belastingen.py b/app/apps/fines/api_queries_belastingen.py index 8186ed6d4..f079b0327 100644 --- a/app/apps/fines/api_queries_belastingen.py +++ b/app/apps/fines/api_queries_belastingen.py @@ -8,24 +8,32 @@ logger = logging.getLogger(__name__) -@retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) -def get_fines(id): +def get_fines(id, use_retry=True): """ Search the Belasting API for fines with case_id identification """ - parameter = {"identificatienummer": id} - header = {"authorization": f"Bearer {settings.BELASTING_API_ACCESS_TOKEN}"} - response = requests.get( - url=settings.BELASTING_API_URL, - headers=header, - params=parameter, - verify="/usr/local/share/ca-certificates/adp_rootca.crt", - timeout=6, - ) - response.raise_for_status() + def _get_fines_internal(): + parameter = {"identificatienummer": id} + header = {"authorization": f"Bearer {settings.BELASTING_API_ACCESS_TOKEN}"} - return response.json() + response = requests.get( + url=settings.BELASTING_API_URL, + headers=header, + params=parameter, + verify="/usr/local/share/ca-certificates/adp_rootca.crt", + timeout=6, + ) + response.raise_for_status() + + return response.json() + + if use_retry: + _get_fines_internal = retry( + stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR) + )(_get_fines_internal) + + return _get_fines_internal() def get_mock_fines(case_id): diff --git a/app/apps/health/health_checks.py b/app/apps/health/health_checks.py index 8bff1f974..257513648 100644 --- a/app/apps/health/health_checks.py +++ b/app/apps/health/health_checks.py @@ -6,7 +6,8 @@ from django.conf import settings from health_check.backends import BaseHealthCheckBackend from health_check.exceptions import ServiceUnavailable -from requests.exceptions import HTTPError +from requests.exceptions import HTTPError, SSLError, Timeout +from utils.api_queries_bag import do_bag_search_nummeraanduiding_id_by_bag_id from utils.api_queries_toeristische_verhuur import ( get_bag_vakantieverhuur_registrations, get_bsn_vakantieverhuur_registrations, @@ -15,7 +16,7 @@ ) logger = logging.getLogger(__name__) -timeout_in_sec = 6 +timeout_in_sec = 10 class APIServiceCheckBackend(BaseHealthCheckBackend): @@ -31,19 +32,14 @@ def get_api_url(self): return self.api_url def check_status(self): - """Check service by opening and closing a broker channel.""" - logger.info("Checking status of API url...") api_url = self.get_api_url() - assert api_url, "The given api_url should be set" + if api_url is None: + self.add_error(ServiceUnavailable("API URL is not set.")) + return + try: response = requests.get(api_url, timeout=timeout_in_sec) response.raise_for_status() - except AssertionError as e: - logger.error(e) - self.add_error( - ServiceUnavailable("The given API endpoint has not been set"), - e, - ) except ConnectionRefusedError as e: logger.error(e) self.add_error( @@ -53,10 +49,13 @@ def check_status(self): except HTTPError as e: logger.error(e) self.add_error(ServiceUnavailable(f"Service not found. {api_url}")) - except requests.exceptions.Timeout: + except Timeout: self.add_error( - ServiceUnavailable(f"Exceeded timeout of {timeout_in_sec} seconds") + ServiceUnavailable(f"Exceeded timeout of {timeout_in_sec} seconds.") ) + except SSLError as e: + logger.error(e) + self.add_error(ServiceUnavailable("SSL error.")) except BaseException as e: logger.error(e) self.add_error(ServiceUnavailable("Unknown error"), e) @@ -80,16 +79,6 @@ class BAGAtlasServiceCheck(APIServiceCheckBackend): verbose_name = "BAG Atlas" -class BAGNummeraanduidingenServiceCheck(APIServiceCheckBackend): - """ - Endpoint for checking the BAG Nummeraanduidingen Service API - """ - - critical_service = True - api_url = settings.BAG_API_NUMMERAANDUIDING_SEARCH_URL - verbose_name = "BAG Nummeraanduidingen" - - class BAGVerblijfsobjectServiceCheck(APIServiceCheckBackend): """ Endpoint for checking the BAG Verblijfsobject Service API Endpoint @@ -110,6 +99,33 @@ class BRPServiceCheck(APIServiceCheckBackend): verbose_name = "BRP" +class BAGNummeraanduidingenServiceCheck(BaseHealthCheckBackend): + """ + Endpoint for checking the BAG Nummeraanduidingen API + """ + + critical_service = True + verbose_name = "BAG Nummeraanduidingen" + + def check_status(self): + try: + response = do_bag_search_nummeraanduiding_id_by_bag_id( + settings.BAG_ID_AMSTEL_1 + ) + message = response.get("message") + if message: + self.add_error(ServiceUnavailable(f"{message}"), message) + except HTTPError as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"HTTPError {e.response.status_code}.")) + except Exception as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"Failed {e}"), e) + + def identifier(self): + return self.verbose_name + + class CeleryExecuteTask(BaseHealthCheckBackend): def check_status(self): result = debug_task.apply_async(ignore_result=False) @@ -126,9 +142,12 @@ def check_status(self): try: # The id doesn't matter, as long an authenticated request is succesful. - get_fines("foo-id") + get_fines("foo-id", use_retry=False) + except SSLError as e: + logger.error(e) + self.add_error(ServiceUnavailable("SSL error.")) except Exception as e: - self.add_error(ServiceUnavailable("Failed"), e) + self.add_error(ServiceUnavailable(f"Failed {e}"), e) class DecosJoinCheck(BaseHealthCheckBackend): @@ -143,9 +162,9 @@ def check_status(self): # The address doesn't matter, as long an authenticated request is succesful. Amstel 1 ;) path = "items/90642DCCC2DB46469657C3D0DF0B1ED7/COBJECTS?filter=PHONE3 eq '0363010012143319'" response = DecosJoinRequest().get(path) - assert response, "Could not reach Decos Join" + assert response, "Could not reach Decos Join." except Exception as e: - self.add_error(ServiceUnavailable("Failed"), e) + self.add_error(ServiceUnavailable(f"{e}"), e) class KeycloakCheck(APIServiceCheckBackend): @@ -170,8 +189,11 @@ def get_api_url(self): from zgw_consumers.constants import APITypes from zgw_consumers.models import Service - zaken_service = Service.objects.filter(api_type=APITypes.zrc).get() - return zaken_service.api_root + try: + zaken_service = Service.objects.filter(api_type=APITypes.zrc).get() + return zaken_service.api_root + except Service.DoesNotExist: + return None class OpenZaakZakenCatalogus(APIServiceCheckBackend): @@ -186,8 +208,11 @@ def get_api_url(self): from zgw_consumers.constants import APITypes from zgw_consumers.models import Service - catalogi_service = Service.objects.filter(api_type=APITypes.ztc).get() - return catalogi_service.api_root + try: + catalogi_service = Service.objects.filter(api_type=APITypes.ztc).get() + return catalogi_service.api_root + except Service.DoesNotExist: + return None class OpenZaakZakenAlfresco(APIServiceCheckBackend): @@ -202,8 +227,11 @@ def get_api_url(self): from zgw_consumers.constants import APITypes from zgw_consumers.models import Service - documenten_service = Service.objects.filter(api_type=APITypes.drc).get() - return documenten_service.api_root + try: + documenten_service = Service.objects.filter(api_type=APITypes.drc).get() + return documenten_service.api_root + except Service.DoesNotExist: + return None class VakantieVerhuurRegistratieCheck(BaseHealthCheckBackend): @@ -262,15 +290,17 @@ def check_status(self): } try: - response = get_vakantieverhuur_meldingen( + get_vakantieverhuur_meldingen( settings.VAKANTIEVERHUUR_TOERISTISCHE_VERHUUR_API_HEALTH_CHECK_BAG_ID, query_params=params, + use_retry=False, ) - assert response, "Could not reach Toeristischeverhuur.nl" - + except HTTPError as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"HTTPError {e.response.status_code}.")) except Exception as e: logger.error(e) - self.add_error(ServiceUnavailable("Failed"), e) + self.add_error(ServiceUnavailable(f"Failed {e}"), e) else: logger.info( "Connection established. Toeristischeverhuur.nl API connection is healthy." @@ -284,9 +314,10 @@ class PowerBrowser(BaseHealthCheckBackend): def check_status(self): try: - response = PowerbrowserRequest().get_vergunningen_with_bag_id( - settings.BAG_ID_AMSTEL_1 - ) - assert response, "Could not reach PowerBrowser" + PowerbrowserRequest().get_vergunningen_with_bag_id(settings.BAG_ID_AMSTEL_1) + except HTTPError as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"HTTPError {e.response.status_code}.")) except Exception as e: - self.add_error(ServiceUnavailable("Failed"), e) + logger.error(e) + self.add_error(ServiceUnavailable(f"Failed {e}"), e) diff --git a/app/config/settings.py b/app/config/settings.py index 00de5d2b9..453f3ce46 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -302,6 +302,12 @@ "BAG_API_VERBLIJFSOBJECT_URL", "https://api.data.amsterdam.nl/bag/v1.1/verblijfsobject/", ) +# API key for the public Amsterdam API (api.data.amsterdam.nl). +# This key is NOT used for authorization, but to identify who is using the API for communication purposes. +BAG_API_PUBLIC_KEY = os.getenv( + "BAG_API_PUBLIC_KEY", + "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzdWIiOjMxNjQ2NDI4NzA1MzQ4NzI1NTEsImV4cCI6MTczODA3MDQ4N30.sGNs0EIRcdyUv76X1J1q46Y4kAIHSqHR1fca-srQlIQnV0aWduQn5xTlGQM1lvZCDk_F5qWf0__8u1jcYDMlDg", +) # Bag_id of Amstel 1 for testing purposes. BAG_ID_AMSTEL_1 = os.getenv( "BAG_ID_AMSTEL_1", diff --git a/app/utils/api_queries_bag.py b/app/utils/api_queries_bag.py index 131b90f8d..41caccaec 100644 --- a/app/utils/api_queries_bag.py +++ b/app/utils/api_queries_bag.py @@ -6,6 +6,8 @@ logger = logging.getLogger(__name__) +headers = {"x-api-key": settings.BAG_API_PUBLIC_KEY} + @retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) def do_bag_search_nummeraanduiding_id_by_bag_id(bag_id): @@ -15,6 +17,7 @@ def do_bag_search_nummeraanduiding_id_by_bag_id(bag_id): address_search = requests.get( settings.BAG_API_NUMMERAANDUIDING_SEARCH_URL, params={"adresseertVerblijfsobject.identificatie": bag_id}, + headers=headers, timeout=30, ) return address_search.json() @@ -34,6 +37,7 @@ def do_bag_search_nummeraanduiding_id_by_address(address): address_search = requests.get( settings.BAG_API_NUMMERAANDUIDING_SEARCH_URL, params=params, + headers=headers, timeout=30, ) return address_search.json() diff --git a/app/utils/api_queries_toeristische_verhuur.py b/app/utils/api_queries_toeristische_verhuur.py index d42a7b2a2..8e3e0f290 100644 --- a/app/utils/api_queries_toeristische_verhuur.py +++ b/app/utils/api_queries_toeristische_verhuur.py @@ -7,26 +7,36 @@ logger = logging.getLogger(__name__) -@retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) -def get_vakantieverhuur_meldingen(bag_id, query_params): +def get_vakantieverhuur_meldingen(bag_id, query_params, use_retry=True): """ Get the Vakantieverhuur meldingen from the Toeristische Verhuur register """ - header = { - "x-api-key": settings.VAKANTIEVERHUUR_TOERISTISCHE_VERHUUR_API_ACCESS_TOKEN - } - url = f"{settings.VAKANTIEVERHUUR_TOERISTISCHE_VERHUUR_API_URL}meldingen/{bag_id}" - response = requests.get( - url=url, - params=query_params, - headers=header, - timeout=30, - ) + def _get_vakantieverhuur_meldingen_internal(): + header = { + "x-api-key": settings.VAKANTIEVERHUUR_TOERISTISCHE_VERHUUR_API_ACCESS_TOKEN + } + url = ( + f"{settings.VAKANTIEVERHUUR_TOERISTISCHE_VERHUUR_API_URL}meldingen/{bag_id}" + ) - response.raise_for_status() + response = requests.get( + url=url, + params=query_params, + headers=header, + timeout=30, + ) + + response.raise_for_status() + + return response.json(), response.status_code + + if use_retry: + _get_vakantieverhuur_meldingen_internal = retry( + stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR) + )(_get_vakantieverhuur_meldingen_internal) - return response.json(), response.status_code + return _get_vakantieverhuur_meldingen_internal() @retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) From 434fb35c240ae23aaeb72ccb214fb10f30445e0d Mon Sep 17 00:00:00 2001 From: remyvdwereld <86827854+remyvdwereld@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:58:23 +0100 Subject: [PATCH 2/2] Removed mirror to ADO action --- .github/workflows/mirror-to-azure-devops.yml | 21 -------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/mirror-to-azure-devops.yml diff --git a/.github/workflows/mirror-to-azure-devops.yml b/.github/workflows/mirror-to-azure-devops.yml deleted file mode 100644 index 6b2480d4f..000000000 --- a/.github/workflows/mirror-to-azure-devops.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror to Azure DevOps - -on: - push: - branches: - - master - -jobs: - mirror: - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Push to Azure DevOps - env: - AZURE_DEVOPS_REPO_URL_PAT: ${{ secrets.AZURE_DEVOPS_REPO_URL_PAT }} - run: | - git remote add azure-devops $AZURE_DEVOPS_REPO_URL_PAT - git push azure-devops master