cq

lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib/views/files.py

@@ -13,7 +13,6 @@
 
 from aiohttp import web
 from aiohttp.multipart import BodyPartReader
-from dl_s3.utils import s3_file_exists
 from redis.asyncio.lock import Lock as RedisLock
 
 from dl_api_commons.aiohttp.aiohttp_wrappers import (
@@ -50,6 +49,7 @@
 )
 from dl_s3.data_sink import S3RawFileAsyncDataSink
 from dl_s3.stream import RawBytesAsyncDataStream
+from dl_s3.utils import s3_file_exists
 
 
 LOGGER = logging.getLogger(__name__)
@@ -136,17 +136,19 @@ async def _chunk_iter(chunk_size: int = 10 * 1024 * 1024) -> AsyncGenerator[byte
 class MakePresignedUrlView(FileUploaderBaseView):
     PRESIGNED_URL_EXPIRATION_SECONDS: ClassVar[int] = 60 * 60  # 1 hour
     PRESIGNED_URL_MIN_BYTES: ClassVar[int] = 1
-    PRESIGNED_URL_MAX_BYTES: ClassVar[int] = 200 * 1024 ** 2  # 200 MB
+    PRESIGNED_URL_MAX_BYTES: ClassVar[int] = 200 * 1024**2  # 200 MB
 
     async def post(self) -> web.StreamResponse:
         req_data = await self._load_post_request_schema_data(files_schemas.MakePresignedUrlRequestSchema)
         content_md5: str = req_data["content_md5"]
 
         s3 = self.dl_request.get_s3_service()
-        s3_key = S3_KEY_PARTS_SEPARATOR.join((
-            self.dl_request.rci.user_id or "unknown",
-            str(uuid.uuid4()),
-        ))
+        s3_key = S3_KEY_PARTS_SEPARATOR.join(
+            (
+                self.dl_request.rci.user_id or "unknown",
+                str(uuid.uuid4()),
+            )
+        )
 
         url = await s3.client.generate_presigned_post(
             Bucket=s3.tmp_bucket_name,

lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/db/conftest.py

@@ -4,12 +4,12 @@
 import logging
 import os
 
-from dl_s3.utils import upload_to_s3_by_presigned
 import pytest
 
 from dl_api_commons.client.common import Req
 from dl_file_uploader_api_lib_tests.req_builder import ReqBuilder
 from dl_file_uploader_lib.testing.data_gen import generate_sample_csv_data_str
+from dl_s3.utils import upload_to_s3_by_presigned
 
 
 LOGGER = logging.getLogger(__name__)
@@ -75,7 +75,9 @@ async def uploaded_file_id(s3_tmp_bucket, fu_client, csv_data) -> str:
     upload_resp = await upload_to_s3_by_presigned(presigned_url_resp.json, content_md5, csv_data)
     assert upload_resp.status == 204
 
-    download_resp = await fu_client.make_request(ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "csv_data.csv"))
+    download_resp = await fu_client.make_request(
+        ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "csv_data.csv")
+    )
     assert download_resp.status == 201, download_resp.json
 
     assert download_resp.status == 201
@@ -97,7 +99,9 @@ async def uploaded_excel_id(
     upload_resp = await upload_to_s3_by_presigned(presigned_url_resp.json, content_md5, excel_data)
     assert upload_resp.status == 204
 
-    download_resp = await fu_client.make_request(ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "data.xlsx"))
+    download_resp = await fu_client.make_request(
+        ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "data.xlsx")
+    )
     assert download_resp.status == 201, download_resp.json
 
     assert download_resp.status == 201

lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib_tests/db/test_files_api.py

@@ -1,25 +1,26 @@
 import asyncio
+import base64
+import hashlib
 import http
-import io
 import json
 import uuid
-import hashlib
-from urllib.parse import urlparse
-import base64
 
 import aiohttp
 import attr
-from dl_file_uploader_api_lib.views.files import S3_KEY_PARTS_SEPARATOR, MakePresignedUrlView
-from dl_s3.utils import upload_to_s3_by_presigned
 import pytest
 
 from dl_api_commons.base_models import RequestContextInfo
 from dl_configs.crypto_keys import get_dummy_crypto_keys_config
 from dl_constants.enums import FileProcessingStatus
+from dl_file_uploader_api_lib.views.files import (
+    S3_KEY_PARTS_SEPARATOR,
+    MakePresignedUrlView,
+)
 from dl_file_uploader_api_lib_tests.req_builder import ReqBuilder
 from dl_file_uploader_lib.redis_model.base import RedisModelManager
 from dl_file_uploader_lib.redis_model.models import DataFile
 from dl_s3.data_sink import S3RawFileAsyncDataSink
+from dl_s3.utils import upload_to_s3_by_presigned
 
 
 @pytest.mark.asyncio
@@ -60,7 +61,9 @@ async def test_download_presigned_url(fu_client, s3_tmp_bucket, rci, csv_data):
     upload_resp_data = await upload_resp.read()
     assert upload_resp.status == 204, upload_resp_data
 
-    download_resp = await fu_client.make_request(ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "csv_data.csv"))
+    download_resp = await fu_client.make_request(
+        ReqBuilder.presigned_url_download(presigned_url_resp.json["fields"]["key"], "csv_data.csv")
+    )
     assert download_resp.status == 201, download_resp.json
 
 

lib/dl_file_uploader_worker_lib/dl_file_uploader_worker_lib_tests/ext/test_yadocs.py

@@ -7,8 +7,8 @@
     YaDocsUserSourceProperties,
 )
 from dl_file_uploader_task_interface.tasks import DownloadYaDocsTask
-from dl_task_processor.state import wait_task
 from dl_s3.utils import s3_file_exists
+from dl_task_processor.state import wait_task
 
 
 @pytest.mark.asyncio

lib/dl_s3/dl_s3/s3_service.py

@@ -60,7 +60,8 @@ async def initialize(self) -> None:
             aws_access_key_id=self._access_key_id,
             aws_secret_access_key=self._secret_access_key,
             endpoint_url=self._endpoint_url,
-            config=AioConfig(signature_version="s3v4"),  # v4 signature is required to generate presigned URLs with restriction policies
+            config=AioConfig(signature_version="s3v4"),
+            # ^ v4 signature is required to generate presigned URLs with restriction policies
         )
 
         session = get_session()

lib/dl_s3/dl_s3/utils.py

@@ -5,22 +5,29 @@
 import aiohttp
 import botocore.exceptions
 
+
 if typing.TYPE_CHECKING:
     from types_aiobotocore_s3 import S3Client as AsyncS3Client
 
 
-async def upload_to_s3_by_presigned(presigned_url: dict[str, typing.Any], content_md5: str, data: str) -> aiohttp.ClientResponse:
+async def upload_to_s3_by_presigned(
+    presigned_url: dict[str, typing.Any], content_md5: str, data: str
+) -> aiohttp.ClientResponse:
     upload_url = presigned_url["url"]
     upload_url_fields = presigned_url["fields"]
     upload_url_fields["content-md5"] = content_md5
 
     async with aiohttp.ClientSession() as session:
         with aiohttp.MultipartWriter("form-data") as mpwriter:
             for k, v in upload_url_fields.items():
-                part = mpwriter.append(v, {'Content-Type': 'text/plain', 'Content-Disposition': f'attachment; name="{k}"'})
+                part = mpwriter.append(
+                    v, {"Content-Type": "text/plain", "Content-Disposition": f'attachment; name="{k}"'}
+                )
                 part.set_content_disposition("form-data", name=k)
 
-            part = mpwriter.append(data, {'Content-Type': 'text/plain', 'Content-Disposition': f'attachment; filename="mydata"'})
+            part = mpwriter.append(
+                data, {"Content-Type": "text/plain", "Content-Disposition": 'attachment; filename="mydata"'}
+            )
             part.set_content_disposition("form-data", name="file")
 
             async with session.post(