From 2f8cabcb48515639cb4c86a10a66018c9047646f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 14:45:23 -0500 Subject: [PATCH 01/13] logs --- examples/app.py | 3 + examples/models/cards.py | 6 +- examples/resources/base.py | 10 ++ examples/resources/cards.py | 21 +++- examples/validators.py | 4 + fast_agave/middlewares/loggers.py | 169 +++++++++++++++++++++++++++ fast_agave/version.py | 2 +- requirements.txt | 1 + setup.cfg | 2 +- setup.py | 1 + tests/blueprint/test_blueprint.py | 7 +- tests/helpers.py | 9 ++ tests/middlewares/__init__.py | 0 tests/middlewares/test_loggers.py | 187 ++++++++++++++++++++++++++++++ 14 files changed, 413 insertions(+), 9 deletions(-) create mode 100644 fast_agave/middlewares/loggers.py create mode 100644 tests/helpers.py create mode 100644 tests/middlewares/__init__.py create mode 100644 tests/middlewares/test_loggers.py diff --git a/examples/app.py b/examples/app.py index b903587..4486caf 100644 --- a/examples/app.py +++ b/examples/app.py @@ -5,6 +5,7 @@ from fastapi import FastAPI from fast_agave.middlewares import FastAgaveErrorHandler +from fast_agave.middlewares.loggers import OpenSearchLog from .resources import app as resources from .middlewares import AuthedMiddleware from .tasks.task_example import dummy_task @@ -13,7 +14,9 @@ app = FastAPI(title='example') app.include_router(resources) + app.add_middleware(AuthedMiddleware) +app.add_middleware(OpenSearchLog) app.add_middleware(FastAgaveErrorHandler) diff --git a/examples/models/cards.py b/examples/models/cards.py index fee553c..892fd88 100644 --- a/examples/models/cards.py +++ b/examples/models/cards.py @@ -1,4 +1,4 @@ -from mongoengine import DateTimeField, StringField +from mongoengine import DateTimeField, StringField, IntField from mongoengine_plus.aio import AsyncDocument from mongoengine_plus.models import BaseModel @@ -8,5 +8,9 @@ class Card(BaseModel, AsyncDocument): id = StringField(primary_key=True, default=uuid_field('CA')) number = StringField(required=True) + cvv = StringField(default='111') + exp_year = IntField(default=2040) user_id = StringField(required=True) + status = StringField(default='active') created_at = DateTimeField() + deactivated_at = DateTimeField() diff --git a/examples/resources/base.py b/examples/resources/base.py index ad8f541..2b31b45 100644 --- a/examples/resources/base.py +++ b/examples/resources/base.py @@ -8,6 +8,11 @@ app = RestApiBlueprint() +@app.get('/hello') +def hello() -> str: + return 'hello!' + + @app.get('/healthy_auth') def health_auth_check() -> Dict: return dict(greeting="I'm authenticated and healthy !!!") @@ -29,3 +34,8 @@ def you_shall_not_pass() -> None: # La prueba de este endpoint hace un mock a nivel middleware # para responder con un `UnauthorizedError` ... + + +@app.get('/buggy_endpoint') +def buggy_endpoint() -> NoReturn: + raise Exception('oh no!') diff --git a/examples/resources/cards.py b/examples/resources/cards.py index abcace1..7bd19db 100644 --- a/examples/resources/cards.py +++ b/examples/resources/cards.py @@ -1,10 +1,12 @@ +import datetime as dt from typing import Dict +from fastapi import Request from fastapi.responses import JSONResponse as Response from fast_agave.filters import generic_query from ..models import Card as CardModel -from ..validators import CardQuery +from ..validators import CardQuery, CardUpdateRequest from .base import app @@ -12,16 +14,29 @@ class Card: model = CardModel query_validator = CardQuery + update_validator = CardUpdateRequest get_query_filter = generic_query @staticmethod async def retrieve(card: CardModel) -> Response: data = card.to_dict() - data['number'] = '*' * 16 + data['last_four_digits'] = card.number[-4:] return Response(content=data) @staticmethod async def query(response: Dict): for item in response['items']: - item['number'] = '*' * 16 + item['last_four_digits'] = item['number'][-4:] return response + + @staticmethod + async def update(card: CardModel, request: CardUpdateRequest) -> Response: + card.status = request.status + await card.async_save() + return Response(content=card.to_dict(), status_code=200) + + @staticmethod + async def delete(card: CardModel, _: Request) -> Response: + card.deactivated_at = dt.datetime.utcnow() + await card.async_save() + return Response(content=card.to_dict(), status_code=200) diff --git a/examples/validators.py b/examples/validators.py index f1dafb4..b01d4b9 100644 --- a/examples/validators.py +++ b/examples/validators.py @@ -81,6 +81,10 @@ class CardQuery(QueryParams): number: Optional[str] = None +class CardUpdateRequest(BaseModel): + status: str + + class FileUploadValidator(BaseModel): file: bytes file_name: str diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py new file mode 100644 index 0000000..0b0758a --- /dev/null +++ b/fast_agave/middlewares/loggers.py @@ -0,0 +1,169 @@ +import asyncio +import base64 +import binascii +import json +import os +import re +from dataclasses import asdict +from typing import Any, Optional + +from aiohttp import ClientSession +from cuenca_validations.errors import CuencaError +from cuenca_validations.typing import DictStrAny +from fastapi import Request, Response +from starlette.datastructures import Headers +from starlette.middleware.base import ( + BaseHTTPMiddleware, + RequestResponseEndpoint, +) + +from fast_agave.exc import FastAgaveError + +LOGS_SERVER_URL = os.environ.get('LOGS_SERVER_URL', None) + + +AUTHED_REQUIRED_HEADERS = { + 'x-cuenca-token', + 'x-cuenca-logintoken', + 'x-cuenca-loginid', + 'x-cuenca-sessionid', +} + +SENSITIVE_DATA_FIELDS = { + 'number', + 'cvv2', + 'cvv', + 'icvv', + 'exp_month', + 'exp_year', + 'pin', + 'pin_block', + 'pin_block_switch', +} + +NON_VALID_SCOPE_VALUES = [ + 'headers', + 'app', + 'extensions', + 'fastapi_astack', + 'app', + 'route_handler', + 'router', + 'endpoint', +] + + +def basic_auth_decode(basic: str) -> str: + basic = re.sub(r'^Basic ', '', basic, flags=re.IGNORECASE) + try: + decoded = base64.b64decode(basic).decode('ascii') + ak, _ = decoded.split(':') + except (binascii.Error, UnicodeDecodeError, ValueError): + # If `basic` value is not valid then it is not a sensitive data + return basic + return ak + + +def mask_sensitive_headers(headers: Headers) -> DictStrAny: + masked_dict = dict() + for key, val in headers.items(): + if key == 'authorization': + masked_dict[key] = basic_auth_decode(val) + elif key in AUTHED_REQUIRED_HEADERS: + masked_dict[key] = val[0:5] + '*' * 5 if val else '' + else: + masked_dict[key] = val + return masked_dict + + +def mask_sensitive_data(data: Any) -> Any: + if type(data) is dict: + for key, val in data.items(): + if type(val) is list: + data[key] = [mask_sensitive_data(v) for v in val] + elif key == 'number': + data[key] = '*' * 12 + val[-4:] + elif key in SENSITIVE_DATA_FIELDS and type(val) is int: + data[key] = '***' + elif key in SENSITIVE_DATA_FIELDS: + data[key] = '*' * len(key) + + return data + + +class OpenSearchLog(BaseHTTPMiddleware): + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + if not LOGS_SERVER_URL: + return await call_next(request) + + request_data = dict(request.scope) + + request_headers = mask_sensitive_headers(request.headers) + request_data['client'] = f'{request.client.host}:{request.client.port}' + request_data['query_string'] = str(request.query_params) + request_data['server'] = ':'.join( + (str(v) for v in request.scope['server']) + ) + + for value in NON_VALID_SCOPE_VALUES: + request_data.pop(value, None) + + response_body = {} + + try: + response = await call_next(request) + except FastAgaveError as exc: + response_body = asdict(exc) + raise + except CuencaError as exc: + response_body = dict(status_code=exc.status_code, code=str(exc)) + raise + else: + # El objeto response es de tipo `starlette.responses.StreamingResponse` + # por lo que sus datos vienen en binario, se deben obtener manualmente + # y crear un custom Request + # https://stackoverflow.com/questions/71882419/fastapi-how-to-get-the-response-body-in-middleware + binary_response_body = b'' + async for chunk in response.body_iterator: # type: ignore + binary_response_body += chunk + + response_body = json.loads(binary_response_body.decode()) + response = Response( + content=binary_response_body, + status_code=response.status_code, + headers=dict(response.headers), + media_type=response.media_type, + ) + + response_body = mask_sensitive_data(response_body) + finally: + t = asyncio.create_task( + self.send_log_to_open_search( + request.app.title, + request_data, + request_headers, + response_body, + ) + ) + await t + return response + + @classmethod + async def send_log_to_open_search( + cls, + app_name: str, + request_data: DictStrAny, + request_headers: DictStrAny, + response_body: Optional[DictStrAny] = None, + ) -> None: + data = dict( + app=app_name, + request_data=request_data, + request_headers=request_headers, + response_body=response_body, + ) + async with ClientSession() as session: + async with session.put(LOGS_SERVER_URL, json=data): + pass diff --git a/fast_agave/version.py b/fast_agave/version.py index ef7eb44..06f971e 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.6.0' +__version__ = '0.7.0.dev0' diff --git a/requirements.txt b/requirements.txt index 6e0a5fa..3d2af30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ aiobotocore==2.1.0 +aiohttp==3.7.4.post0 cuenca-validations==0.10.7 fastapi==0.68.2 mongoengine-plus==0.0.3 diff --git a/setup.cfg b/setup.cfg index 6b110c0..c5bef90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ test=pytest [tool:pytest] -addopts = -p no:warnings -v --cov-report term-missing --cov=fast_agave +addopts = -p no:warnings -v --cov-report term-missing --cov-report=html --cov=fast_agave [flake8] inline-quotes = ' diff --git a/setup.py b/setup.py index 6d2ffad..7371237 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ python_requires='>=3.8', install_requires=[ 'aiobotocore>=2.1.0,<2.2.0', + 'aiohttp>=3.7.4.post0,<3.8.0', 'cuenca-validations>=0.9.4,<1.0.0', 'fastapi>=0.63.0,<0.69.0', 'mongoengine-plus>=0.0.2,<1.0.0', diff --git a/tests/blueprint/test_blueprint.py b/tests/blueprint/test_blueprint.py index 7bf236c..48ae912 100644 --- a/tests/blueprint/test_blueprint.py +++ b/tests/blueprint/test_blueprint.py @@ -87,7 +87,8 @@ def test_update_resource_with_invalid_params(client: TestClient) -> None: def test_retrieve_custom_method(client: TestClient, card: Card) -> None: resp = client.get(f'/cards/{card.id}') assert resp.status_code == 200 - assert resp.json()['number'] == '*' * 16 + assert resp.json()['number'] == card.number + assert resp.json()['last_four_digits'] == card.number[-4:] def test_update_resource_that_doesnt_exist(client: TestClient) -> None: @@ -229,13 +230,13 @@ def test_query_custom_method(client: TestClient) -> None: json_body = resp.json() assert resp.status_code == 200 assert len(json_body['items']) == 2 - assert all(card['number'] == '*' * 16 for card in json_body['items']) + assert all('last_four_digits' in card for card in json_body['items']) resp = client.get(json_body['next_page_uri']) json_body = resp.json() assert resp.status_code == 200 assert len(json_body['items']) == 2 - assert all(card['number'] == '*' * 16 for card in json_body['items']) + assert all('last_four_digits' in card for card in json_body['items']) def test_cannot_query_resource(client: TestClient) -> None: diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..ff1b58f --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,9 @@ +import base64 +from typing import Dict + + +def auth_header(username: str, password: str = '') -> Dict: + creds = base64.b64encode(f'{username}:{password}'.encode('ascii')).decode( + 'utf-8' + ) + return {'Authorization': f'Basic {creds}'} diff --git a/tests/middlewares/__init__.py b/tests/middlewares/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/middlewares/test_loggers.py b/tests/middlewares/test_loggers.py new file mode 100644 index 0000000..331d2b5 --- /dev/null +++ b/tests/middlewares/test_loggers.py @@ -0,0 +1,187 @@ +import json +from dataclasses import asdict +from typing import List +from unittest.mock import MagicMock, patch + +import pytest +from cuenca_validations.errors import WrongCredsError +from fastapi.testclient import TestClient + +from examples.models import Card +from fast_agave.exc import UnauthorizedError +from tests.helpers import auth_header + + +@pytest.fixture(autouse=True) +def patch_log_server_url(): + with patch( + 'fast_agave.middlewares.loggers.LOGS_SERVER_URL', 'http://logs.com' + ): + yield + + +@patch('aiohttp.client.ClientSession.put') +def test_doesnt_log_sensitive_data_in_headers( + mock_put: MagicMock, client: TestClient +) -> None: + token = 'TK123456789' + login_token = 'LT123456789' + login_id = 'LI123456789' + session_id = 'SS123456789' + auth = auth_header('AK123', 'secret') + + auth['X-Cuenca-Token'] = token + auth['X-Cuenca-LoginToken'] = login_token + auth['X-Cuenca-LoginId'] = login_id + auth['X-Cuenca-SessionId'] = session_id + + resp = client.get('/accounts', headers=auth) + assert resp.status_code == 200 + assert type(mock_put.call_args.kwargs['json']) is dict + + # makes it easy to search for specifics string instead + # of searching in each key and document level + data_as_str = json.dumps(mock_put.call_args.kwargs['json']) + + assert 'AK123' in data_as_str + assert 'secret' not in data_as_str + assert token not in data_as_str + assert login_token not in data_as_str + assert login_id not in data_as_str + + +@patch('aiohttp.client.ClientSession.put') +def test_log_fake_basic_auth_data_in_headers( + mock_put: MagicMock, client: TestClient +) -> None: + resp = client.get( + '/accounts', headers={'Authorization': 'Basic malformed-token'} + ) + assert resp.status_code == 200 + assert type(mock_put.call_args.kwargs['json']) is dict + + # makes it easy to search for specifics string instead + # of searching in each key and document level + data_as_str = json.dumps(mock_put.call_args.kwargs['json']) + + assert 'malformed-token' in data_as_str + + +@patch('aiohttp.client.ClientSession.put') +def test_doesnt_log_sensitive_data_in_query_response_body( + mock_put: MagicMock, client: TestClient, cards: List[Card] +) -> None: + resp = client.get('/cards') + assert resp.status_code == 200 + assert type(mock_put.call_args.kwargs['json']) is dict + + data_as_str = json.dumps(mock_put.call_args.kwargs['json']) + + assert not any(c.number in data_as_str for c in cards) + assert not any(str(c.exp_year) in data_as_str for c in cards) + assert all(c.user_id in data_as_str for c in cards) + + +@pytest.mark.parametrize('method', ['get', 'delete']) +@patch('aiohttp.client.ClientSession.put') +def test_doesnt_log_sensitive_data_in_response_body( + mock_put: MagicMock, client: TestClient, card: Card, method: str +) -> None: + client_method = getattr(client, method) + resp = client_method(f'/cards/{card.id}') + assert resp.status_code == 200 + + data_as_str = json.dumps(mock_put.call_args.kwargs['json']) + assert card.number not in data_as_str + assert card.user_id in data_as_str + + +@patch('aiohttp.client.ClientSession.put') +def test_doesnt_log_sensitive_data_in_patch_response_body( + mock_put: MagicMock, client: TestClient, card: Card +) -> None: + resp = client.patch(f'/cards/{card.id}', json=dict(status='deactivated')) + assert resp.status_code == 200 + + data_as_str = json.dumps(mock_put.call_args.kwargs['json']) + assert card.number not in data_as_str + assert card.user_id in data_as_str + + +@patch('aiohttp.client.ClientSession.put') +def test_logs_non_dict_response_bodies( + mock_put: MagicMock, client: TestClient +) -> None: + resp = client.get('/hello') + assert resp.status_code == 200 + log_data = mock_put.call_args.kwargs['json'] + assert log_data['response_body'] == 'hello!' + + +@patch('aiohttp.client.ClientSession.put') +def test_logs_agave_error_response_bodies( + mock_put: MagicMock, client: TestClient +) -> None: + resp = client.get('/raise_fast_agave_errors') + assert resp.status_code == 401 + expected_exc = UnauthorizedError('nice try!') + log_data = mock_put.call_args.kwargs['json'] + assert log_data['response_body'] == asdict(expected_exc) + + +@patch('aiohttp.client.ClientSession.put') +def test_logs_cuenca_error_response_bodies( + mock_put: MagicMock, client: TestClient +) -> None: + resp = client.get('/raise_cuenca_errors') + assert resp.status_code == 401 + expected_exc = WrongCredsError('you are not lucky enough!') + log_data = mock_put.call_args.kwargs['json'] + assert log_data['response_body'] == dict( + status_code=expected_exc.status_code, code='you are not lucky enough!' + ) + + +# +# +# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) +# def test_log_post_data_to_opensearch(client: TestClient) -> None: +# data = dict(name='Doroteo Arango') +# auth = auth_header('AK123', 'secret') +# auth['X-Cuenca-Token'] = 'TK123456789' +# auth['X-Cuenca-LoginToken'] = 'LT123456789' +# auth['X-Cuenca-LoginId'] = 'LI123456789' +# auth['X-Cuenca-SessionId'] = 'SS123456789' +# +# resp = client.post('/accounts', json=data, headers=auth) +# assert resp.status_code == 201 + + +# +# +# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) +# def test_log_get_data_to_opensearch( +# client: TestClient, account: Account +# ) -> None: +# auth = auth_header('AK123', 'secret') +# auth['X-Cuenca-Token'] = 'TK123456789' +# auth['X-Cuenca-LoginToken'] = 'LT123456789' +# auth['X-Cuenca-LoginId'] = 'LI123456789' +# auth['X-Cuenca-SessionId'] = 'SS123456789' +# +# resp = client.get(f'/accounts/{account.id}', headers=auth) +# assert resp.status_code == 200 +# +# +# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) +# def test_log_query_string_to_opensearch( +# client: TestClient, card: Card +# ) -> None: +# auth = auth_header('AK123', 'secret') +# auth['X-Cuenca-Token'] = 'TK123456789' +# auth['X-Cuenca-LoginToken'] = 'LT123456789' +# auth['X-Cuenca-LoginId'] = 'LI123456789' +# auth['X-Cuenca-SessionId'] = 'SS123456789' +# +# resp = client.get(f'/cards?number={card.number}', headers=auth) +# assert resp.status_code == 200 From 34afc5ee99a5d1db89f81ded20350081a787c474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 15:00:56 -0500 Subject: [PATCH 02/13] lint --- fast_agave/middlewares/loggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py index 0b0758a..d524758 100644 --- a/fast_agave/middlewares/loggers.py +++ b/fast_agave/middlewares/loggers.py @@ -19,7 +19,7 @@ from fast_agave.exc import FastAgaveError -LOGS_SERVER_URL = os.environ.get('LOGS_SERVER_URL', None) +LOGS_SERVER_URL = os.environ.get('LOGS_SERVER_URL', '') AUTHED_REQUIRED_HEADERS = { From e3044b6bf55ce769cad5285b8fe2c76028c9db9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 15:02:04 -0500 Subject: [PATCH 03/13] concurrency --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87f7af8..9b10992 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,10 @@ name: test on: push +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + jobs: lint: runs-on: ubuntu-latest From 0ccefc0cfde3eb1196ec571c48285d2120a8e8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 15:21:35 -0500 Subject: [PATCH 04/13] remove unused code --- examples/resources/base.py | 5 ---- tests/middlewares/test_loggers.py | 45 ------------------------------- 2 files changed, 50 deletions(-) diff --git a/examples/resources/base.py b/examples/resources/base.py index 2b31b45..5bcb437 100644 --- a/examples/resources/base.py +++ b/examples/resources/base.py @@ -34,8 +34,3 @@ def you_shall_not_pass() -> None: # La prueba de este endpoint hace un mock a nivel middleware # para responder con un `UnauthorizedError` ... - - -@app.get('/buggy_endpoint') -def buggy_endpoint() -> NoReturn: - raise Exception('oh no!') diff --git a/tests/middlewares/test_loggers.py b/tests/middlewares/test_loggers.py index 331d2b5..8df4881 100644 --- a/tests/middlewares/test_loggers.py +++ b/tests/middlewares/test_loggers.py @@ -140,48 +140,3 @@ def test_logs_cuenca_error_response_bodies( assert log_data['response_body'] == dict( status_code=expected_exc.status_code, code='you are not lucky enough!' ) - - -# -# -# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) -# def test_log_post_data_to_opensearch(client: TestClient) -> None: -# data = dict(name='Doroteo Arango') -# auth = auth_header('AK123', 'secret') -# auth['X-Cuenca-Token'] = 'TK123456789' -# auth['X-Cuenca-LoginToken'] = 'LT123456789' -# auth['X-Cuenca-LoginId'] = 'LI123456789' -# auth['X-Cuenca-SessionId'] = 'SS123456789' -# -# resp = client.post('/accounts', json=data, headers=auth) -# assert resp.status_code == 201 - - -# -# -# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) -# def test_log_get_data_to_opensearch( -# client: TestClient, account: Account -# ) -> None: -# auth = auth_header('AK123', 'secret') -# auth['X-Cuenca-Token'] = 'TK123456789' -# auth['X-Cuenca-LoginToken'] = 'LT123456789' -# auth['X-Cuenca-LoginId'] = 'LI123456789' -# auth['X-Cuenca-SessionId'] = 'SS123456789' -# -# resp = client.get(f'/accounts/{account.id}', headers=auth) -# assert resp.status_code == 200 -# -# -# @patch('fast_agave.middlewares.loggers.LOGS_SERVER_URL', True) -# def test_log_query_string_to_opensearch( -# client: TestClient, card: Card -# ) -> None: -# auth = auth_header('AK123', 'secret') -# auth['X-Cuenca-Token'] = 'TK123456789' -# auth['X-Cuenca-LoginToken'] = 'LT123456789' -# auth['X-Cuenca-LoginId'] = 'LI123456789' -# auth['X-Cuenca-SessionId'] = 'SS123456789' -# -# resp = client.get(f'/cards?number={card.number}', headers=auth) -# assert resp.status_code == 200 From d30ef91244c33cd1dbf12c0613c73a774136aa39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 16:38:23 -0500 Subject: [PATCH 05/13] dev.a --- fast_agave/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_agave/version.py b/fast_agave/version.py index 06f971e..a30103b 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev0' +__version__ = '0.7.0.dev0.a' From 16e04943070f894aecb8084549affc302fe31f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Fri, 15 Jul 2022 16:41:44 -0500 Subject: [PATCH 06/13] 0.7.0.dev0-a --- fast_agave/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_agave/version.py b/fast_agave/version.py index a30103b..8a0bf69 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev0.a' +__version__ = '0.7.0.dev0-a' From b9d63276c5a5a5c506553912e8f2f6d7f6ac4429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Mon, 18 Jul 2022 10:29:57 -0500 Subject: [PATCH 07/13] version --- fast_agave/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_agave/version.py b/fast_agave/version.py index 8a0bf69..42df73d 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev0-a' +__version__ = '0.7.0.dev100' From a5ff6c0d1d28353e4922d61c3dcf57bf77db8103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Mon, 18 Jul 2022 21:37:38 -0500 Subject: [PATCH 08/13] fix import --- fast_agave/middlewares/__init__.py | 3 ++- fast_agave/version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fast_agave/middlewares/__init__.py b/fast_agave/middlewares/__init__.py index d8037a9..8a73c88 100644 --- a/fast_agave/middlewares/__init__.py +++ b/fast_agave/middlewares/__init__.py @@ -1,3 +1,4 @@ from .error_handlers import FastAgaveErrorHandler +from .loggers import OpenSearchLog -__all__ = ['FastAgaveErrorHandler'] +__all__ = ['FastAgaveErrorHandler', 'OpenSearchLog'] diff --git a/fast_agave/version.py b/fast_agave/version.py index 42df73d..9ac1129 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev100' +__version__ = '0.7.0.dev101' From 3fda5205313314ce58b3b4607ab086a05f837714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Mon, 18 Jul 2022 22:28:39 -0500 Subject: [PATCH 09/13] version --- fast_agave/middlewares/loggers.py | 5 +---- fast_agave/version.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py index d524758..5d4cf90 100644 --- a/fast_agave/middlewares/loggers.py +++ b/fast_agave/middlewares/loggers.py @@ -19,9 +19,6 @@ from fast_agave.exc import FastAgaveError -LOGS_SERVER_URL = os.environ.get('LOGS_SERVER_URL', '') - - AUTHED_REQUIRED_HEADERS = { 'x-cuenca-token', 'x-cuenca-logintoken', @@ -95,7 +92,7 @@ class OpenSearchLog(BaseHTTPMiddleware): async def dispatch( self, request: Request, call_next: RequestResponseEndpoint ) -> Response: - if not LOGS_SERVER_URL: + if not os.environ.get('LOGS_SERVER_URL', ''): return await call_next(request) request_data = dict(request.scope) diff --git a/fast_agave/version.py b/fast_agave/version.py index 9ac1129..12e8e1a 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev101' +__version__ = '0.7.0.dev102' From 53dd2d651c07f373f1b0c48eca9840beced7e231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Mon, 18 Jul 2022 22:32:59 -0500 Subject: [PATCH 10/13] fix url --- fast_agave/middlewares/loggers.py | 3 ++- fast_agave/version.py | 2 +- tests/middlewares/test_loggers.py | 11 ++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py index 5d4cf90..de887c2 100644 --- a/fast_agave/middlewares/loggers.py +++ b/fast_agave/middlewares/loggers.py @@ -155,6 +155,7 @@ async def send_log_to_open_search( request_headers: DictStrAny, response_body: Optional[DictStrAny] = None, ) -> None: + log_server_url = os.environ.get('LOGS_SERVER_URL', '') data = dict( app=app_name, request_data=request_data, @@ -162,5 +163,5 @@ async def send_log_to_open_search( response_body=response_body, ) async with ClientSession() as session: - async with session.put(LOGS_SERVER_URL, json=data): + async with session.put(log_server_url, json=data): pass diff --git a/fast_agave/version.py b/fast_agave/version.py index 12e8e1a..6d1cc54 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev102' +__version__ = '0.7.0.dev103' diff --git a/tests/middlewares/test_loggers.py b/tests/middlewares/test_loggers.py index 8df4881..4fd9093 100644 --- a/tests/middlewares/test_loggers.py +++ b/tests/middlewares/test_loggers.py @@ -13,11 +13,12 @@ @pytest.fixture(autouse=True) -def patch_log_server_url(): - with patch( - 'fast_agave.middlewares.loggers.LOGS_SERVER_URL', 'http://logs.com' - ): - yield +def patch_log_server_url(monkeypatch): + monkeypatch.setenv('LOGS_SERVER_URL', 'http://logs.com') + # with patch( + # 'fast_agave.middlewares.loggers.LOGS_SERVER_URL', 'http://logs.com' + # ): + # yield @patch('aiohttp.client.ClientSession.put') From 83027a6ea97936652069a9aeb8b9b7e57f232393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Tue, 19 Jul 2022 12:51:42 -0500 Subject: [PATCH 11/13] remove binary data --- fast_agave/middlewares/loggers.py | 1 + fast_agave/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py index de887c2..21b4ea4 100644 --- a/fast_agave/middlewares/loggers.py +++ b/fast_agave/middlewares/loggers.py @@ -47,6 +47,7 @@ 'route_handler', 'router', 'endpoint', + 'raw_path', ] diff --git a/fast_agave/version.py b/fast_agave/version.py index 6d1cc54..af2a83f 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev103' +__version__ = '0.7.0.dev104' From 2768186181d08bf843f00b7f18bd3d4fba3e680d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felipe=20L=C3=B3pez?= Date: Tue, 19 Jul 2022 15:22:21 -0500 Subject: [PATCH 12/13] remove unnecesary await --- fast_agave/middlewares/loggers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fast_agave/middlewares/loggers.py b/fast_agave/middlewares/loggers.py index 21b4ea4..a5af360 100644 --- a/fast_agave/middlewares/loggers.py +++ b/fast_agave/middlewares/loggers.py @@ -137,7 +137,7 @@ async def dispatch( response_body = mask_sensitive_data(response_body) finally: - t = asyncio.create_task( + asyncio.create_task( self.send_log_to_open_search( request.app.title, request_data, @@ -145,7 +145,6 @@ async def dispatch( response_body, ) ) - await t return response @classmethod From 20e025fb7c7c5f4b822cee98ca1c1ebb43c24142 Mon Sep 17 00:00:00 2001 From: Pach Date: Mon, 1 Aug 2022 18:42:34 -0500 Subject: [PATCH 13/13] Update version.py --- fast_agave/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast_agave/version.py b/fast_agave/version.py index af2a83f..a71c5c7 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.7.0.dev104' +__version__ = '0.7.0'