add scripts for running PeerDB using prebuilt artifacts and development

README.md

@@ -27,10 +27,10 @@ You can use PeerDB for any of the below use-cases:
 git clone --recursive [email protected]:PeerDB-io/peerdb.git
 cd peerdb
 
-# Run docker containers: peerdb-server, postgres as catalog, temporal
-# Make sure to have at least 5GB of RAM for building PeerDB docker images
-export COMPOSE_PROJECT_NAME=peerdb-stack
-docker compose up
+# Run docker containers: postgres as catalog, temporal, PeerDB server, PeerDB flow API + workers, PeerDB UI
+bash ./run-peerdb.sh
+# OR for local development, images will be built locally:
+bash ./dev-peerdb.sh
 
 # connect to peerdb and query away (Use psql version >=14.0, <16.0)
 psql "port=9900 host=localhost password=peerdb"

dev-peerdb.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+if ! command -v docker &> /dev/null
+then
+    echo "docker could not be found on PATH"
+    exit 1
+fi
+
+docker compose -f docker-compose-dev.yml up --build

docker-compose-dev.yml

@@ -0,0 +1,243 @@
+version: "3.9"
+
+x-catalog-config: &catalog-config
+  PEERDB_CATALOG_HOST: catalog
+  PEERDB_CATALOG_PORT: 5432
+  PEERDB_CATALOG_USER: postgres
+  PEERDB_CATALOG_PASSWORD: postgres
+  PEERDB_CATALOG_DATABASE: postgres
+
+x-flow-worker-env: &flow-worker-env
+  TEMPORAL_HOST_PORT: temporal:7233
+  # For GCS, these will be your HMAC keys instead
+  # For more information:
+  # https://cloud.google.com/storage/docs/authentication/managing-hmackeys
+  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
+  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
+  # For GCS, set this to "auto" without the quotes
+  AWS_REGION: ${AWS_REGION:-}
+  # For GCS, set this as: https://storage.googleapis.com
+  AWS_ENDPOINT: ${AWS_ENDPOINT:-}
+  # enables worker profiling using Grafana Pyroscope
+  ENABLE_PROFILING: "true"
+  # enables exporting of mirror metrics to Prometheus for visualization using Grafana
+  ENABLE_METRICS: "true"
+  PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040
+
+services:
+  catalog:
+    container_name: catalog
+    image: debezium/postgres:16-alpine
+    ports:
+      - 9901:5432
+    environment:
+      PGUSER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready", "-d", "postgres", "-U", "postgres"]
+      interval: 10s
+      timeout: 30s
+      retries: 5
+      start_period: 60s
+
+  temporal:
+    container_name: temporal
+    depends_on:
+      catalog:
+        condition: service_healthy
+    environment:
+      - DB=postgresql
+      - DB_PORT=5432
+      - POSTGRES_USER=postgres
+      - POSTGRES_PWD=postgres
+      - POSTGRES_SEEDS=catalog
+      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
+    image: temporalio/auto-setup:1.22.1
+    ports:
+      - 7233:7233
+    volumes:
+      - ./temporal-dynamicconfig:/etc/temporal/config/dynamicconfig
+    labels:
+      kompose.volume.type: configMap
+
+  pyroscope:
+    container_name: pyroscope
+    image: grafana/pyroscope:latest
+    ports:
+      - 4040:4040
+
+  temporal-admin-tools:
+    container_name: temporal-admin-tools
+    depends_on:
+      - temporal
+    environment:
+      - TEMPORAL_ADDRESS=temporal:7233
+      - TEMPORAL_CLI_ADDRESS=temporal:7233
+    image: temporalio/admin-tools:1.22.1
+    stdin_open: true
+    tty: true
+    healthcheck:
+      test: ["CMD", "tctl", "workflow", "list"]
+      interval: 1s
+      timeout: 5s
+      retries: 30
+
+  temporal-ui:
+    container_name: temporal-ui
+    depends_on:
+      - temporal
+    environment:
+      - TEMPORAL_ADDRESS=temporal:7233
+      - TEMPORAL_CORS_ORIGINS=http://localhost:3000
+      - TEMPORAL_CSRF_COOKIE_INSECURE=true
+    image: temporalio/ui:2.17.2
+    ports:
+      - 8085:8080
+
+  flow-api:
+    container_name: flow_api
+    build:
+      context: .
+      dockerfile: stacks/flow.Dockerfile
+      target: flow-api
+    ports:
+      - 8112:8112
+      - 8113:8113
+    environment:
+      <<: [*catalog-config]
+      TEMPORAL_HOST_PORT: temporal:7233
+    depends_on:
+      temporal-admin-tools:
+        condition: service_healthy
+
+  flow-snapshot-worker:
+    container_name: flow-snapshot-worker
+    build:
+      context: .
+      dockerfile: stacks/flow.Dockerfile
+      target: flow-snapshot-worker
+    environment:
+      TEMPORAL_HOST_PORT: temporal:7233
+    depends_on:
+      temporal-admin-tools:
+        condition: service_healthy
+
+  flow-worker1:
+    container_name: flow-worker1
+    build:
+      context: .
+      dockerfile: stacks/flow.Dockerfile
+      target: flow-worker
+    environment:
+      <<: [*catalog-config, *flow-worker-env]
+      METRICS_SERVER: 0.0.0.0:6061
+    ports:
+      - 6060:6060
+      - 6061:6061
+    depends_on:
+      temporal-admin-tools:
+        condition: service_healthy
+
+  flow-worker2:
+    container_name: flow-worker2
+    build:
+      context: .
+      dockerfile: stacks/flow.Dockerfile
+      target: flow-worker
+    environment:
+      <<: [*catalog-config, *flow-worker-env]
+      METRICS_SERVER: 0.0.0.0:6063
+    ports:
+      - 6062:6062
+      - 6063:6063
+    profiles:
+      - multi
+      - multi-metrics
+    depends_on:
+      temporal-admin-tools:
+        condition: service_healthy
+
+  flow-worker3:
+    container_name: flow-worker3
+    build:
+      context: .
+      dockerfile: stacks/flow.Dockerfile
+      target: flow-worker
+    environment:
+      <<: [*catalog-config, *flow-worker-env]
+      METRICS_SERVER: 0.0.0.0:6065
+    ports:
+      - 6064:6064
+      - 6065:6065
+    profiles:
+      - multi
+      - multi-metrics
+    depends_on:
+      temporal-admin-tools:
+        condition: service_healthy
+
+  peerdb:
+    container_name: peerdb-server
+    build:
+      context: .
+      dockerfile: stacks/peerdb-server.Dockerfile
+    environment:
+      <<: *catalog-config
+      PEERDB_LOG_DIR: /var/log/peerdb
+      PEERDB_PASSWORD: peerdb
+      PEERDB_FLOW_SERVER_ADDRESS: grpc://flow_api:8112
+      RUST_LOG: info
+      RUST_BACKTRACE: 1
+    ports:
+      - 9900:9900
+    depends_on:
+      catalog:
+        condition: service_healthy
+
+  peerdb-prometheus:
+    container_name: peerdb-prometheus
+    build:
+      context: .
+      dockerfile: stacks/prometheus.Dockerfile
+    volumes:
+      - prometheusdata:/prometheus
+    ports:
+      - 9090:9090
+    profiles:
+      - multi-metrics
+      - metrics
+
+  peerdb-grafana:
+    container_name: peerdb-grafana
+    build:
+      context: .
+      dockerfile: stacks/grafana.Dockerfile
+    ports:
+      - 3000:3000
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: peerdb
+    profiles:
+      - multi-metrics
+      - metrics
+
+  peerdb-ui:
+    container_name: peerdb-ui
+    build:
+      context: .
+      dockerfile: stacks/peerdb-ui.Dockerfile
+    ports:
+      - 3001:3000
+    environment:
+      <<: *catalog-config
+      DATABASE_URL: postgres://postgres:postgres@catalog:5432/postgres
+      PEERDB_FLOW_SERVER_HTTP: http://flow_api:8113
+
+volumes:
+  pgdata:
+  prometheusdata:

docker-compose.yml

@@ -18,11 +18,8 @@ x-flow-worker-env: &flow-worker-env
   AWS_REGION: ${AWS_REGION:-}
   # For GCS, set this as: https://storage.googleapis.com
   AWS_ENDPOINT: ${AWS_ENDPOINT:-}
-  # enables worker profiling using Go's pprof
-  ENABLE_PROFILING: "true"
   # enables exporting of mirror metrics to Prometheus for visualization using Grafana
   ENABLE_METRICS: "true"
-  PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040
 
 x-peerdb-temporal-namespace: &peerdb-temporal-namespace
   PEERDB_TEMPORAL_NAMESPACE: default
@@ -60,28 +57,22 @@ services:
       - POSTGRES_PWD=postgres
       - POSTGRES_SEEDS=catalog
       - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
-    image: temporalio/auto-setup:1.22.2
+    image: temporalio/auto-setup:1.22.1
     ports:
       - 7233:7233
     volumes:
       - ./temporal-dynamicconfig:/etc/temporal/config/dynamicconfig
     labels:
       kompose.volume.type: configMap
 
-  pyroscope:
-    container_name: pyroscope
-    image: grafana/pyroscope:latest
-    ports:
-      - 4040:4040
-
   temporal-admin-tools:
     container_name: temporal-admin-tools
     depends_on:
       - temporal
     environment:
       - TEMPORAL_ADDRESS=temporal:7233
       - TEMPORAL_CLI_ADDRESS=temporal:7233
-    image: temporalio/admin-tools:1.22.2
+    image: temporalio/admin-tools:1.22.1
     stdin_open: true
     tty: true
     healthcheck:
@@ -104,10 +95,7 @@ services:
 
   flow-api:
     container_name: flow_api
-    build:
-      context: .
-      dockerfile: stacks/flow.Dockerfile
-      target: flow-api
+    image: ghcr.io/peerdb-io/flow-api:latest-dev
     ports:
       - 8112:8112
       - 8113:8113
@@ -120,10 +108,7 @@ services:
 
   flow-snapshot-worker:
     container_name: flow-snapshot-worker
-    build:
-      context: .
-      dockerfile: stacks/flow.Dockerfile
-      target: flow-snapshot-worker
+    image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
     environment:
       <<: [*peerdb-temporal-namespace]
       TEMPORAL_HOST_PORT: temporal:7233
@@ -133,10 +118,7 @@ services:
 
   flow-worker1:
     container_name: flow-worker1
-    build:
-      context: .
-      dockerfile: stacks/flow.Dockerfile
-      target: flow-worker
+    image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
       <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace]
       METRICS_SERVER: 0.0.0.0:6061
@@ -149,9 +131,7 @@ services:
 
   flow-worker2:
     container_name: flow-worker2
-    build:
-      context: .
-      dockerfile: stacks/flow-worker.Dockerfile
+    image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
       <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace]
       METRICS_SERVER: 0.0.0.0:6063
@@ -167,9 +147,7 @@ services:
 
   flow-worker3:
     container_name: flow-worker3
-    build:
-      context: .
-      dockerfile: stacks/flow-worker.Dockerfile
+    image: ghcr.io/peerdb-io/flow-worker:latest-dev
     environment:
       <<: [*catalog-config, *flow-worker-env, *peerdb-temporal-namespace]
       METRICS_SERVER: 0.0.0.0:6065
@@ -185,9 +163,7 @@ services:
 
   peerdb:
     container_name: peerdb-server
-    build:
-      context: .
-      dockerfile: stacks/peerdb-server.Dockerfile
+    image: ghcr.io/peerdb-io/peerdb-server:latest-dev
     environment:
       <<: *catalog-config
       PEERDB_LOG_DIR: /var/log/peerdb
@@ -230,9 +206,7 @@ services:
 
   peerdb-ui:
     container_name: peerdb-ui
-    build:
-      context: .
-      dockerfile: stacks/peerdb-ui.Dockerfile
+    image: ghcr.io/peerdb-io/peerdb-ui:latest-dev
     ports:
       - 3001:3000
     environment: