diff --git a/README.md b/README.md index 560f9c542e..bca6cc82dc 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,10 @@ You can use PeerDB for any of the below use-cases: git clone --recursive git@github.com:PeerDB-io/peerdb.git cd peerdb -# Run docker containers: peerdb-server, postgres as catalog, temporal -# Make sure to have at least 5GB of RAM for building PeerDB docker images -export COMPOSE_PROJECT_NAME=peerdb-stack -docker compose up +# Run docker containers: postgres as catalog, temporal, PeerDB server, PeerDB flow API + workers, PeerDB UI +bash ./run-peerdb.sh +# OR for local development, images will be built locally: +bash ./dev-peerdb.sh # connect to peerdb and query away (Use psql version >=14.0, <16.0) psql "port=9900 host=localhost password=peerdb" diff --git a/dev-peerdb.sh b/dev-peerdb.sh new file mode 100644 index 0000000000..53050655f7 --- /dev/null +++ b/dev-peerdb.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -Eeuo pipefail + +if ! command -v docker &> /dev/null +then + echo "docker could not be found on PATH" + exit 1 +fi + +docker compose -f docker-compose-dev.yml up --build diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000000..c6e4de2faa --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,243 @@ +version: "3.9" + +x-catalog-config: &catalog-config + PEERDB_CATALOG_HOST: catalog + PEERDB_CATALOG_PORT: 5432 + PEERDB_CATALOG_USER: postgres + PEERDB_CATALOG_PASSWORD: postgres + PEERDB_CATALOG_DATABASE: postgres + +x-flow-worker-env: &flow-worker-env + TEMPORAL_HOST_PORT: temporal:7233 + # For GCS, these will be your HMAC keys instead + # For more information: + # https://cloud.google.com/storage/docs/authentication/managing-hmackeys + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + # For GCS, set this to "auto" without the quotes + AWS_REGION: ${AWS_REGION:-} + # For GCS, set this as: https://storage.googleapis.com + AWS_ENDPOINT: ${AWS_ENDPOINT:-} + # enables worker profiling using Grafana Pyroscope + ENABLE_PROFILING: "true" + # enables exporting of mirror metrics to Prometheus for visualization using Grafana + ENABLE_METRICS: "true" + PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040 + +services: + catalog: + container_name: catalog + image: debezium/postgres:16-alpine + ports: + - 9901:5432 + environment: + PGUSER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready", "-d", "postgres", "-U", "postgres"] + interval: 10s + timeout: 30s + retries: 5 + start_period: 60s + + temporal: + container_name: temporal + depends_on: + catalog: + condition: service_healthy + environment: + - DB=postgresql + - DB_PORT=5432 + - POSTGRES_USER=postgres + - POSTGRES_PWD=postgres + - POSTGRES_SEEDS=catalog + - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml + image: temporalio/auto-setup:1.22.1 + ports: + - 7233:7233 + volumes: + - ./temporal-dynamicconfig:/etc/temporal/config/dynamicconfig + labels: + kompose.volume.type: configMap + + pyroscope: + container_name: pyroscope + image: grafana/pyroscope:latest + ports: + - 4040:4040 + + temporal-admin-tools: + container_name: temporal-admin-tools + depends_on: + - temporal + environment: + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_CLI_ADDRESS=temporal:7233 + image: temporalio/admin-tools:1.22.1 + stdin_open: true + tty: true + healthcheck: + test: ["CMD", "tctl", "workflow", "list"] + interval: 1s + timeout: 5s + retries: 30 + + temporal-ui: + container_name: temporal-ui + depends_on: + - temporal + environment: + - TEMPORAL_ADDRESS=temporal:7233 + - TEMPORAL_CORS_ORIGINS=http://localhost:3000 + - TEMPORAL_CSRF_COOKIE_INSECURE=true + image: temporalio/ui:2.17.2 + ports: + - 8085:8080 + + flow-api: + container_name: flow_api + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-api + ports: + - 8112:8112 + - 8113:8113 + environment: + <<: [*catalog-config] + TEMPORAL_HOST_PORT: temporal:7233 + depends_on: + temporal-admin-tools: + condition: service_healthy + + flow-snapshot-worker: + container_name: flow-snapshot-worker + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-snapshot-worker + environment: + TEMPORAL_HOST_PORT: temporal:7233 + depends_on: + temporal-admin-tools: + condition: service_healthy + + flow-worker1: + container_name: flow-worker1 + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-worker + environment: + <<: [*catalog-config, *flow-worker-env] + METRICS_SERVER: 0.0.0.0:6061 + ports: + - 6060:6060 + - 6061:6061 + depends_on: + temporal-admin-tools: + condition: service_healthy + + flow-worker2: + container_name: flow-worker2 + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-worker + environment: + <<: [*catalog-config, *flow-worker-env] + METRICS_SERVER: 0.0.0.0:6063 + ports: + - 6062:6062 + - 6063:6063 + profiles: + - multi + - multi-metrics + depends_on: + temporal-admin-tools: + condition: service_healthy + + flow-worker3: + container_name: flow-worker3 + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-worker + environment: + <<: [*catalog-config, *flow-worker-env] + METRICS_SERVER: 0.0.0.0:6065 + ports: + - 6064:6064 + - 6065:6065 + profiles: + - multi + - multi-metrics + depends_on: + temporal-admin-tools: + condition: service_healthy + + peerdb: + container_name: peerdb-server + build: + context: . + dockerfile: stacks/peerdb-server.Dockerfile + environment: + <<: *catalog-config + PEERDB_LOG_DIR: /var/log/peerdb + PEERDB_PASSWORD: peerdb + PEERDB_FLOW_SERVER_ADDRESS: grpc://flow_api:8112 + RUST_LOG: info + RUST_BACKTRACE: 1 + ports: + - 9900:9900 + depends_on: + catalog: + condition: service_healthy + + peerdb-prometheus: + container_name: peerdb-prometheus + build: + context: . + dockerfile: stacks/prometheus.Dockerfile + volumes: + - prometheusdata:/prometheus + ports: + - 9090:9090 + profiles: + - multi-metrics + - metrics + + peerdb-grafana: + container_name: peerdb-grafana + build: + context: . + dockerfile: stacks/grafana.Dockerfile + ports: + - 3000:3000 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: peerdb + profiles: + - multi-metrics + - metrics + + peerdb-ui: + container_name: peerdb-ui + build: + context: . + dockerfile: stacks/peerdb-ui.Dockerfile + ports: + - 3001:3000 + environment: + <<: *catalog-config + DATABASE_URL: postgres://postgres:postgres@catalog:5432/postgres + PEERDB_FLOW_SERVER_HTTP: http://flow_api:8113 + +volumes: + pgdata: + prometheusdata: diff --git a/docker-compose.yml b/docker-compose.yml index 1bad765a58..d5d4dbbff6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,11 +18,8 @@ x-flow-worker-env: &flow-worker-env AWS_REGION: ${AWS_REGION:-} # For GCS, set this as: https://storage.googleapis.com AWS_ENDPOINT: ${AWS_ENDPOINT:-} - # enables worker profiling using Go's pprof - ENABLE_PROFILING: "true" # enables exporting of mirror metrics to Prometheus for visualization using Grafana ENABLE_METRICS: "true" - PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040 x-peerdb-temporal-namespace: &peerdb-temporal-namespace PEERDB_TEMPORAL_NAMESPACE: default @@ -60,7 +57,7 @@ services: - POSTGRES_PWD=postgres - POSTGRES_SEEDS=catalog - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml - image: temporalio/auto-setup:1.22.2 + image: temporalio/auto-setup:1.22.1 ports: - 7233:7233 volumes: @@ -68,12 +65,6 @@ services: labels: kompose.volume.type: configMap - pyroscope: - container_name: pyroscope - image: grafana/pyroscope:latest - ports: - - 4040:4040 - temporal-admin-tools: container_name: temporal-admin-tools depends_on: @@ -81,7 +72,7 @@ services: environment: - TEMPORAL_ADDRESS=temporal:7233 - TEMPORAL_CLI_ADDRESS=temporal:7233 - image: temporalio/admin-tools:1.22.2 + image: temporalio/admin-tools:1.22.1 stdin_open: true tty: true healthcheck: @@ -104,10 +95,7 @@ services: flow-api: container_name: flow_api - build: - context: . - dockerfile: stacks/flow.Dockerfile - target: flow-api + image: ghcr.io/peerdb-io/flow-api:latest-dev ports: - 8112:8112 - 8113:8113 @@ -120,10 +108,7 @@ services: flow-snapshot-worker: container_name: flow-snapshot-worker - build: - context: . - dockerfile: stacks/flow.Dockerfile - target: flow-snapshot-worker + image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev environment: <<: [*peerdb-temporal-namespace] TEMPORAL_HOST_PORT: temporal:7233 @@ -133,10 +118,7 @@ services: flow-worker1: container_name: flow-worker1 - build: - context: . - dockerfile: stacks/flow.Dockerfile - target: flow-worker + image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace] METRICS_SERVER: 0.0.0.0:6061 @@ -149,9 +131,7 @@ services: flow-worker2: container_name: flow-worker2 - build: - context: . - dockerfile: stacks/flow-worker.Dockerfile + image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace] METRICS_SERVER: 0.0.0.0:6063 @@ -167,9 +147,7 @@ services: flow-worker3: container_name: flow-worker3 - build: - context: . - dockerfile: stacks/flow-worker.Dockerfile + image: ghcr.io/peerdb-io/flow-worker:latest-dev environment: <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace] METRICS_SERVER: 0.0.0.0:6065 @@ -185,9 +163,7 @@ services: peerdb: container_name: peerdb-server - build: - context: . - dockerfile: stacks/peerdb-server.Dockerfile + image: ghcr.io/peerdb-io/peerdb-server:latest-dev environment: <<: *catalog-config PEERDB_LOG_DIR: /var/log/peerdb @@ -230,9 +206,7 @@ services: peerdb-ui: container_name: peerdb-ui - build: - context: . - dockerfile: stacks/peerdb-ui.Dockerfile + image: ghcr.io/peerdb-io/peerdb-ui:latest-dev ports: - 3001:3000 environment: diff --git a/run-peerdb.sh b/run-peerdb.sh new file mode 100644 index 0000000000..74b1dfda71 --- /dev/null +++ b/run-peerdb.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -Eeuo pipefail + +if ! command -v docker &> /dev/null +then + echo "docker could not be found on PATH" + exit 1 +fi + +docker compose pull +docker compose -f docker-compose.yml up --no-attach catalog --no-attach temporal --no-attach temporal-ui --no-attach temporal-admin-tools diff --git a/stacks/peerdb-ui.Dockerfile b/stacks/peerdb-ui.Dockerfile index d3f827a68a..2496361465 100644 --- a/stacks/peerdb-ui.Dockerfile +++ b/stacks/peerdb-ui.Dockerfile @@ -42,7 +42,7 @@ RUN chown nextjs:nodejs .next # https://nextjs.org/docs/advanced-features/output-file-tracing COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static -COPY stacks/ui-entrypoint.sh /app/entrypoint.sh +COPY stacks/ui/ui-entrypoint.sh /app/entrypoint.sh # allow permissions for nextjs user to do anything in /app RUN chown -R nextjs:nodejs /app diff --git a/stacks/ui-entrypoint.sh b/stacks/ui/ui-entrypoint.sh similarity index 100% rename from stacks/ui-entrypoint.sh rename to stacks/ui/ui-entrypoint.sh