From 8165b1cf771575772347f2d04f96c4c354598b30 Mon Sep 17 00:00:00 2001 From: David McKee Date: Mon, 6 Feb 2023 18:03:21 +0000 Subject: [PATCH 1/2] Make local variable so rollbar will contain AWS error message --- ds-caselaw-ingester/lambda_function.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ds-caselaw-ingester/lambda_function.py b/ds-caselaw-ingester/lambda_function.py index 918f63f..84c681a 100644 --- a/ds-caselaw-ingester/lambda_function.py +++ b/ds-caselaw-ingester/lambda_function.py @@ -22,6 +22,10 @@ rollbar.init(os.getenv("ROLLBAR_TOKEN"), environment=os.getenv("ROLLBAR_ENV")) +class S3HTTPError(Exception): + pass + + class FileNotFoundException(Exception): pass @@ -318,15 +322,19 @@ def handler(event, context): # Retrieve tar file from S3 http = urllib3.PoolManager() try: - file = http.request("GET", message["s3-folder-url"]) - except: + s3_response = http.request("GET", message["s3-folder-url"]) + tar_gz_contents = s3_response.data + if s3_response.status < 400: + raise S3HTTPError(tar_gz_contents) + except Exception: # Send retry message to sqs if the GET fails send_retry_message(message, sqs_client) - + raise # Store it in the /tmp directory filename = os.path.join("/tmp", f"{consignment_reference}.tar.gz") + with open(filename, "wb") as out: - out.write(file.data) + out.write(tar_gz_contents) out.close() tar = tarfile.open(filename, mode="r") From cf69ade756ede2d72101d39e44f9bd5fa8b50171 Mon Sep 17 00:00:00 2001 From: David McKee Date: Mon, 6 Feb 2023 18:26:17 +0000 Subject: [PATCH 2/2] Bump isort version in pre-commit config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52d4e70..4581bd3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: black - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort args: ["--profile", "black", "--filter-files"]