modify S3 settings to work with non-aws s3

src/pyflexplot/config/service_settings.py

@@ -7,6 +7,9 @@
 class Bucket(BaseModel):
     region: str
     name: str
+    endpoint_url: str
+    s3_access_key: str
+    s3_secret_key: str
     retries: int
 
 class S3(BaseModel):

src/pyflexplot/config/settings.yaml

@@ -5,10 +5,16 @@ main:
       input:
         region: eu-central-2
         name: meteoswiss-flexpart-flexpart-output-dev
+        endpoint_url: ""
+        s3_access_key: ""
+        s3_secret_key: ""
         retries: 10
       output:
         region: eu-central-2
         name: meteoswiss-flexpart-pyflexplot-output-dev
+        endpoint_url: ""
+        s3_access_key: ""
+        s3_secret_key: ""
         retries: 10
   local:
     paths:

src/pyflexplot/s3.py

@@ -58,6 +58,7 @@ def split_s3_uri(infile: str) -> tuple[str, str, str]:
         filename = ''
     return bucket_name, key, filename
 
+
 def download_key_from_bucket(key: str,
                              dest: Path,
                              bucket: Bucket = CONFIG.main.aws.s3.input,
@@ -72,13 +73,8 @@ def download_key_from_bucket(key: str,
         bucket: S3 bucket from where data will be fetched.
     """
 
-    client = boto3.Session().client('s3', config=Config(
-                                            region_name=bucket.region,
-                                            retries={
-                                                'max_attempts': int(bucket.retries),
-                                                'mode': 'standard'
-                                            })
-                                        )
+    client = _configure_s3_client(bucket)
+
     # Make directory if not existing
     if not os.path.exists( dest.parent ):
         os.makedirs( dest.parent )
@@ -106,13 +102,7 @@ def upload_outpaths_to_s3(upload_outpaths: list[str],
         raise ValueError("Model object must be provided to upload to S3, \
                          model name and base time are used in the object key.")
     try:
-        client = boto3.Session().client('s3', config=Config(
-                                            region_name=bucket.region,
-                                            retries={
-                                                'max_attempts': int(bucket.retries),
-                                                'mode': 'standard'
-                                            })
-                                        )
+        client = _configure_s3_client(bucket)
 
         for outpath in upload_outpaths:
             key = f"{model.name}/{model.base_time}/{Path(outpath).name}"
@@ -154,3 +144,44 @@ def _retry_with_backoff(fn: Callable,
             logging.info("Sleep: %.2f seconds", sleep)
             time.sleep(sleep)
             x += 1
+
+def _configure_s3_client(bucket):
+    """
+    Configures the S3 client based on the bucket's platform.
+
+    Args:
+        bucket: An object containing bucket configuration attributes.
+                Must include 'name', 'retries' and optionally 'endpoint_url', 's3_access_key', and 's3_secret_key'.
+
+    Returns:
+        A configured boto3 S3 client.
+    """
+    common_config = Config(
+        retries={
+            'max_attempts': int(bucket.retries),
+            'mode': 'standard'
+        }
+    )
+
+    # Check if endpoint_url is present to differentiate between AWS and other platforms
+    if hasattr(bucket, 'endpoint_url') and bucket.endpoint_url.strip():
+        # Non-AWS configuration
+        return boto3.Session().client(
+            's3',
+            endpoint_url=bucket.endpoint_url,
+            aws_access_key_id=bucket.s3_access_key,
+            aws_secret_access_key=bucket.s3_secret_key,
+            config=common_config
+        )
+    else:
+        # AWS S3 configuration
+        return boto3.Session().client(
+            's3',
+            config=Config(
+                region_name=bucket.region,
+                retries={
+                    'max_attempts': int(bucket.retries),
+                    'mode': 'standard'
+                }
+            )
+        )