lint, main docker compose file

PeerDB-io · Apr 26, 2024 · 4d1e250 · 4d1e250
1 parent 6910055
commit 4d1e250
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 6 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,12 @@
 version: "3.9"
 
+x-minio-config: &minio-config
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
+  PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001
+  PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket
+
 x-catalog-config: &catalog-config
   PEERDB_CATALOG_HOST: catalog
   PEERDB_CATALOG_PORT: 5432
@@ -103,7 +110,7 @@ services:
       - 8112:8112
       - 8113:8113
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -112,7 +119,7 @@ services:
     container_name: flow-snapshot-worker
     image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -121,7 +128,7 @@ services:
     container_name: flow-worker1
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     depends_on:
       temporal-admin-tools:
         condition: service_healthy
@@ -130,7 +137,7 @@ services:
     container_name: flow-worker2
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -141,7 +148,7 @@ services:
     container_name: flow-worker3
     image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
-      <<: [*catalog-config, *flow-worker-env]
+      <<: [*catalog-config, *flow-worker-env, *minio-config]
     profiles:
       - multi
     depends_on:
@@ -179,6 +186,27 @@ services:
     depends_on:
       - flow-api
 
+  minio:
+    image: minio/minio
+    volumes:
+      - minio-data:/data
+    ports:
+      - "9001:9000"
+      - "9002:36987"
+    environment:
+      <<: *minio-config
+    entrypoint: >
+      /bin/sh -c "
+      export MINIO_ROOT_USER=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID;
+      export MINIO_ROOT_PASSWORD=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY;
+      minio server /data --console-address=":36987" &
+      sleep 2;
+      /usr/bin/mc alias set myminiopeerdb http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD;
+      /usr/bin/mc mb myminiopeerdb/$$PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME;
+      wait
+      "
+
 volumes:
   pgdata:
   prometheusdata:
+  minio-data:
diff --git a/flow/connectors/utils/aws.go b/flow/connectors/utils/aws.go
@@ -162,7 +162,8 @@ func LoadPeerDBAWSEnvConfigProvider(connectorName string) AWSCredentialsProvider
 
 func GetAWSCredentialsProvider(ctx context.Context, connectorName string, peerCredentials PeerAWSCredentials) (AWSCredentialsProvider, error) {
 	if !(peerCredentials.Credentials.AccessKeyID == "" && peerCredentials.Credentials.SecretAccessKey == "" &&
-		peerCredentials.Region == "" && peerCredentials.RoleArn == nil && (peerCredentials.EndpointUrl == nil || *peerCredentials.EndpointUrl == "")) {
+		peerCredentials.Region == "" && peerCredentials.RoleArn == nil &&
+		(peerCredentials.EndpointUrl == nil || *peerCredentials.EndpointUrl == "")) {
 		staticProvider := NewStaticAWSCredentialsProvider(AWSCredentials{
 			AWS:         peerCredentials.Credentials,
 			EndpointUrl: peerCredentials.EndpointUrl,