Merge pull request #5190 from mnaamani/docker-compose-no-more

Docker-Compose no more
Joystream · Sep 18, 2024 · f98ab20 · f98ab20
2 parents e4f3204 + a2f1769
commit f98ab20
Show file tree

Hide file tree

Showing 21 changed files with 71 additions and 112 deletions.
diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ The following tools are required for building, testing and contributing to this
 - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
 - [nodejs](https://nodejs.org/) >= v14.18.x - _required_ (However volta will try to use v18.6)
 - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
-- [docker](https://www.docker.com/get-started) and docker-compose v2.20.x or higher - _required_
+- [docker](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) v2.20.x or higher - _required_
 - [ansible](https://www.ansible.com/) - _optional_
 
 If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.

diff --git a/devops/ansible/roles/common/tasks/install-tools.yml b/devops/ansible/roles/common/tasks/install-tools.yml
@@ -43,7 +43,6 @@
       - docker-compose
       - docker.io
       - docker-doc
-      - docker-compose
       - podman-docker
       - containerd
       - runc
@@ -74,17 +73,6 @@
     state: latest
     update_cache: true
 
-- name: Install docker-compose
-  become: true
-  shell:
-    cmd: |
-      bash -c "
-        curl -L 'https://github.com/docker/compose/releases/download/v2.20.1/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose
-        chmod +x /usr/local/bin/docker-compose
-        ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
-      "
-    creates: /usr/local/bin/docker-compose
-
 - name: Install Docker module for Python
   pip:
     name: docker

diff --git a/docker-compose-no-bind-volumes.yml b/docker-compose-no-bind-volumes.yml
@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -30,7 +29,6 @@ services:
     ports:
       - 3333:3333
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
@@ -65,7 +63,6 @@ services:
       - 3334:3334
       - 127.0.0.1:4334:4334
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     # Node configuration can be overriden via env, for exampe:
     environment:
@@ -101,7 +98,6 @@ services:
     ports:
       - 3335:3333
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
@@ -131,7 +127,6 @@ services:
       - 3336:3334
       - 127.0.0.1:4336:4334
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     # Node configuration can be overriden via env, for exampe:
     environment:
@@ -163,7 +158,6 @@ services:
     volumes:
       - query-node-data:/var/lib/postgresql/data
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       POSTGRES_USER: ${DB_USER}
@@ -179,7 +173,6 @@ services:
     container_name: graphql-server
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - DB_HOST=db
@@ -203,7 +196,6 @@ services:
     container_name: processor
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - INDEXER_ENDPOINT_URL=${PROCESSOR_INDEXER_GATEWAY}
@@ -233,7 +225,6 @@ services:
     container_name: indexer
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - DB_NAME=${INDEXER_DB_NAME}
@@ -255,7 +246,6 @@ services:
     container_name: hydra-indexer-gateway
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - WARTHOG_STARTER_DB_DATABASE=${INDEXER_DB_NAME}

diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml
@@ -1,5 +1,3 @@
-version: '3.4'
-
 services:
   # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/8.7/docker.html
   elasticsearch:

diff --git a/docker-compose.storage-squid.yml b/docker-compose.storage-squid.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   squid_db:
     container_name: squid_db

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -30,7 +29,6 @@ services:
     ports:
       - 3333:3333
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
@@ -65,7 +63,6 @@ services:
       - 3334:3334
       - 127.0.0.1:4334:4334
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     # Node configuration can be overriden via env, for exampe:
     environment:
@@ -104,7 +101,6 @@ services:
     ports:
       - 3335:3333
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
@@ -137,7 +133,6 @@ services:
       - 3336:3334
       - 127.0.0.1:4336:4334
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     # Node configuration can be overriden via env, for exampe:
     environment:
@@ -169,7 +164,6 @@ services:
     volumes:
       - query-node-data:/var/lib/postgresql/data
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       POSTGRES_USER: ${DB_USER}
@@ -182,7 +176,6 @@ services:
     container_name: graphql-server
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - DB_HOST=db
@@ -209,7 +202,6 @@ services:
     container_name: processor
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - INDEXER_ENDPOINT_URL=${PROCESSOR_INDEXER_GATEWAY}
@@ -244,7 +236,6 @@ services:
     container_name: indexer
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - DB_NAME=${INDEXER_DB_NAME}
@@ -266,7 +257,6 @@ services:
     container_name: hydra-indexer-gateway
     restart: unless-stopped
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       - WARTHOG_STARTER_DB_DATABASE=${INDEXER_DB_NAME}

diff --git a/query-node/README.md b/query-node/README.md
@@ -37,8 +37,8 @@ This script script will:
 - Bring up `db` service (query node is using a PostgreSQL database to store the data)
 - Configure the database (`yarn workspace query-node config:dev`)
 - Create the database and initialize schema (`yarn workspace query-node-root db:prepare && yarn workspace query-node-root db:processor:migrate`)
-- Start the GraphQL server service (`docker-compose up -d graphql-server`)
-- Start the Hydra Processor responsible for processing the runtime events and running the mappings' functions (`docker-compose up -d processor`)
+- Start the GraphQL server service (`docker compose up -d graphql-server`)
+- Start the Hydra Processor responsible for processing the runtime events and running the mappings' functions (`docker compose up -d processor`)
 
 ## 4. Stopping the node and removing all associated containers
 
@@ -102,26 +102,26 @@ It's useful when you want to interact with Joystream node via Pioneer or Atlas a
 processed by the processor.
 
 ```
-docker-compose up -d joystream-node indexer hydra-indexer-gateway processor
+docker compose up -d joystream-node indexer hydra-indexer-gateway processor
 
 # start the GraphQL server and Playground if needed via
-docker-compose up -d graphql-server
+docker compose up -d graphql-server
 ```
 
 **Running processor with remote Joystream node and local indexer.**
 It's useful when you want to synchronize the indexer and processor with Joystream node hosted remotely from scratch.
 You can analyze any errors in docker logs and tweak mappings.
 
 ```
-JOYSTREAM_NODE_WS=wss://target-domain.tmp/ws-rpc docker-compose up -d indexer hydra-indexer-gateway processor
+JOYSTREAM_NODE_WS=wss://target-domain.tmp/ws-rpc docker compose up -d indexer hydra-indexer-gateway processor
 ```
 
 **Running processor with remote Joystream node and remote indexer.**
 When debugging an error that happened in processor mappings on a remote server that has its own indexer, you can use it
 and skip potentially time-consuming indexer synchronization
 
 ```
-PROCESSOR_INDEXER_GATEWAY=https://target-domain.tmp/query-node/indexer/graphql docker-compose up -d processor
+PROCESSOR_INDEXER_GATEWAY=https://target-domain.tmp/query-node/indexer/graphql docker compose up -d processor
 ```
 
 ### Restart processor from the beginning
@@ -134,10 +134,10 @@ when the said `Member` created a forum post trying to create. The only way to cr
 is to start processing events all over.
 
 ```
-docker-compose stop graphql-server # ensure graphql server is disconnected from db
-docker-compose rm -vfs processor # turn off processor
+docker compose stop graphql-server # ensure graphql server is disconnected from db
+docker compose rm -vfs processor # turn off processor
 WARTHOG_DB_OVERRIDE=true yarn workspace query-node-root db:reset # reset processor database
-docker-compose up -d processor # start processor again
+docker compose up -d processor # start processor again
 ```
 
 ### Debugging Hydra errors

diff --git a/query-node/kill.sh b/query-node/kill.sh
@@ -5,10 +5,10 @@ SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
 # Only remove query-node related services
-docker-compose -f ../docker-compose.yml rm -vsf processor
-docker-compose -f ../docker-compose.yml rm -vsf graphql-server
-docker-compose -f ../docker-compose.yml rm -vsf indexer
-docker-compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway
-docker-compose -f ../docker-compose.yml rm -vsf redis
-docker-compose -f ../docker-compose.yml rm -vsf db
+docker compose -f ../docker-compose.yml rm -vsf processor
+docker compose -f ../docker-compose.yml rm -vsf graphql-server
+docker compose -f ../docker-compose.yml rm -vsf indexer
+docker compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway
+docker compose -f ../docker-compose.yml rm -vsf redis
+docker compose -f ../docker-compose.yml rm -vsf db
 docker volume rm joystream_query-node-data
diff --git a/query-node/reset-processor.sh b/query-node/reset-processor.sh
@@ -4,7 +4,7 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
-docker-compose -f ../docker-compose.yml rm -vsf processor
-docker-compose -f ../docker-compose.yml rm -vsf graphql-server
+docker compose -f ../docker-compose.yml rm -vsf processor
+docker compose -f ../docker-compose.yml rm -vsf graphql-server
 docker exec db psql -U postgres -c "DROP DATABASE query_node_processor;"
 ./start.sh
diff --git a/query-node/start.sh b/query-node/start.sh
@@ -8,22 +8,22 @@ cd $SCRIPT_PATH
 [ ! -d "generated/" ] && yarn build
 
 # Bring up db
-docker-compose -f ../docker-compose.yml up -d db
+docker compose -f ../docker-compose.yml up -d db
 echo "Waiting for the db to be ready..."
 sleep 5
 
 # Start indexer and gateway
-docker-compose -f ../docker-compose.yml up -d indexer
-docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway
+docker compose -f ../docker-compose.yml up -d indexer
+docker compose -f ../docker-compose.yml up -d hydra-indexer-gateway
 
 # Start processor
-docker-compose -f ../docker-compose.yml up -d processor
+docker compose -f ../docker-compose.yml up -d processor
 echo "Waiting for processor to be ready..." && sleep 30
 if [[ "$OSTYPE" == "darwin"* ]]; then
     # On Docker Desktop things take a bit longer to startup
     sleep 150
 fi
 
 # Start graphql-server
-docker-compose -f ../docker-compose.yml up -d graphql-server
+docker compose -f ../docker-compose.yml up -d graphql-server
 echo "Waiting for graphql-server to be ready..." && sleep 30
diff --git a/scripts/save-to-docker-images.sh b/scripts/save-to-docker-images.sh
@@ -2,10 +2,10 @@
 set -e
 
 # clean start
-docker-compose -f ../docker-compose.yml down -v
+docker compose -f ../docker-compose.yml down -v
 
 function cleanup() {
-    docker-compose -f ../docker-compose.yml down -v
+    docker compose -f ../docker-compose.yml down -v
 }
 
 trap cleanup EXIT
@@ -21,7 +21,7 @@ if [[ -z $JOYSTREAM_NODE_TAG ]]; then
 fi
 
 # start node image, network and volume
-docker-compose -f ../docker-compose.yml up -d joystream-node
+docker compose -f ../docker-compose.yml up -d joystream-node
 
 # copy native runtime
 docker cp ../target/release/joystream-node joystream-node:/joystream/node

diff --git a/setup.sh b/setup.sh
@@ -14,13 +14,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     if ! command -v docker &> /dev/null
     then
       # Install Docker from linux distro maintaners
-      sudo apt-get install -y docker.io containerd runc
+      echo "docker not found. You will need to install it. Visit https://www.docker.com/get-started"
     fi
-    # Install latest version of docker-compose
-    COMPOSE_VERSION=`curl -sL https://api.github.com/repos/docker/compose/releases/latest | jq -r ".tag_name"`
-    sudo curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-    sudo chmod +x /usr/local/bin/docker-compose
-    sudo ln -sf /usr/local/bin/docker-compose /usr/bin/docker-compose
 elif [[ "$OSTYPE" == "darwin"* ]]; then
     # install brew package manager
     if ! which brew >/dev/null 2>&1; then

diff --git a/start-elasticsearch-stack.sh b/start-elasticsearch-stack.sh
@@ -15,10 +15,10 @@ ELASTIC_USERNAME=${ELASTIC_USERNAME:="elastic"}
 ELASTIC_PASSWORD=${ELASTIC_PASSWORD:="password"}
 
 # Remove elasticsearch stack containers & volumes
-docker-compose -f ./docker-compose.elasticsearch.yml down -v
+docker compose -f ./docker-compose.elasticsearch.yml down -v
 
-# Run docker-compose to start elasticsearch container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch
+# Run docker compose to start elasticsearch container
+docker compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch
 
 echo 'Waiting for Elasticsearch...'
 
@@ -46,10 +46,10 @@ export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$(echo $response_body | jq -r '.token.v
 
 echo 'Starting for Kibana...'
 
-## Run docker-compose to start kibana container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d kibana
+## Run docker compose to start kibana container
+docker compose -f ./docker-compose.elasticsearch.yml up -d kibana
 
 echo 'Starting APM Server...'
 
-## Run docker-compose to start apm-server container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d apm-server
+## Run docker compose to start apm-server container
+docker compose -f ./docker-compose.elasticsearch.yml up -d apm-server