diff --git a/docker-compose.yml b/docker-compose.yml index 7ac7e19bb5..5b1ed78653 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,12 @@ version: "3.9" +x-minio-config: &minio-config + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: _peerdb_minioadmin + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: _peerdb_minioadmin + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1 + PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://host.docker.internal:9001 + PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdbbucket + x-catalog-config: &catalog-config PEERDB_CATALOG_HOST: catalog PEERDB_CATALOG_PORT: 5432 @@ -103,7 +110,7 @@ services: - 8112:8112 - 8113:8113 environment: - <<: [*catalog-config, *flow-worker-env] + <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: temporal-admin-tools: condition: service_healthy @@ -112,7 +119,7 @@ services: container_name: flow-snapshot-worker image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev environment: - <<: [*catalog-config, *flow-worker-env] + <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: temporal-admin-tools: condition: service_healthy @@ -121,7 +128,7 @@ services: container_name: flow-worker1 image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: - <<: [*catalog-config, *flow-worker-env] + <<: [*catalog-config, *flow-worker-env, *minio-config] depends_on: temporal-admin-tools: condition: service_healthy @@ -130,7 +137,7 @@ services: container_name: flow-worker2 image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: - <<: [*catalog-config, *flow-worker-env] + <<: [*catalog-config, *flow-worker-env, *minio-config] profiles: - multi depends_on: @@ -141,7 +148,7 @@ services: container_name: flow-worker3 image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: - <<: [*catalog-config, *flow-worker-env] + <<: [*catalog-config, *flow-worker-env, *minio-config] profiles: - multi depends_on: @@ -179,6 +186,27 @@ services: depends_on: - flow-api + minio: + image: minio/minio + volumes: + - minio-data:/data + ports: + - "9001:9000" + - "9002:36987" + environment: + <<: *minio-config + entrypoint: > + /bin/sh -c " + export MINIO_ROOT_USER=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID; + export MINIO_ROOT_PASSWORD=$$PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY; + minio server /data --console-address=":36987" & + sleep 2; + /usr/bin/mc alias set myminiopeerdb http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD; + /usr/bin/mc mb myminiopeerdb/$$PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME; + wait + " + volumes: pgdata: prometheusdata: + minio-data: \ No newline at end of file diff --git a/flow/connectors/utils/aws.go b/flow/connectors/utils/aws.go index 7ff82beba2..29e7a7bf19 100644 --- a/flow/connectors/utils/aws.go +++ b/flow/connectors/utils/aws.go @@ -162,7 +162,8 @@ func LoadPeerDBAWSEnvConfigProvider(connectorName string) AWSCredentialsProvider func GetAWSCredentialsProvider(ctx context.Context, connectorName string, peerCredentials PeerAWSCredentials) (AWSCredentialsProvider, error) { if !(peerCredentials.Credentials.AccessKeyID == "" && peerCredentials.Credentials.SecretAccessKey == "" && - peerCredentials.Region == "" && peerCredentials.RoleArn == nil && (peerCredentials.EndpointUrl == nil || *peerCredentials.EndpointUrl == "")) { + peerCredentials.Region == "" && peerCredentials.RoleArn == nil && + (peerCredentials.EndpointUrl == nil || *peerCredentials.EndpointUrl == "")) { staticProvider := NewStaticAWSCredentialsProvider(AWSCredentials{ AWS: peerCredentials.Credentials, EndpointUrl: peerCredentials.EndpointUrl,