From d20c905fe9c0859749b490278b2549a561c2cb86 Mon Sep 17 00:00:00 2001 From: PascalEgn Date: Mon, 5 Aug 2024 10:52:13 +0200 Subject: [PATCH] docker: update to compose V2 --- .github/workflows/backend-tests-on-docker.yml | 32 ++++++++-------- INSTALL.rst | 2 +- docker-compose.test.yml | 4 +- docker/wait-for-services.sh | 4 +- docs/docker.rst | 38 +++++++++---------- docs/e2e_tutorial.rst | 10 ++--- docs/harvesting.rst | 6 +-- docs/postgresql.rst | 4 +- 8 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/backend-tests-on-docker.yml b/.github/workflows/backend-tests-on-docker.yml index aa72b5b202..fc74184e6b 100644 --- a/.github/workflows/backend-tests-on-docker.yml +++ b/.github/workflows/backend-tests-on-docker.yml @@ -28,7 +28,7 @@ jobs: key: docker-images-cache-${{ hashFiles('.github/workflows/backend-tests-on-docker', 'services.yml', 'docker-compose.test.yml', 'Dockerfile.dev', 'poetry.lock', 'setup.py', 'setup.cfg') }} - name: Build images if: steps.cache.outputs.cache-hit != 'true' - run: docker-compose -f services.yml build --force-rm base + run: docker compose -f services.yml build --force-rm base - name: Install lrzip if: steps.cache.outputs.cache-hit != 'true' run: | @@ -59,13 +59,13 @@ jobs: uses: ./.github/actions/load-image-from-cache - name: Prepare docker run: | - docker-compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-rabbitmq test-redis - docker-compose -f docker-compose.test.yml build --parallel workflows test-web - docker-compose -f docker-compose.test.yml up -d test-database test-indexer test-rabbitmq test-redis test-web + docker compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-rabbitmq test-redis + docker compose -f docker-compose.test.yml build --parallel workflows test-web + docker compose -f docker-compose.test.yml up -d test-database test-indexer test-rabbitmq test-redis test-web sh docker/wait-for-services.sh - name: Run tests run: | - docker-compose -f docker-compose.test.yml run --rm workflows py.test tests/integration/workflows \ + docker compose -f docker-compose.test.yml run --rm workflows py.test tests/integration/workflows \ --ignore tests/integration/workflows/test_audit.py \ --ignore tests/integration/workflows/test_hep_approval.py \ --ignore tests/integration/workflows/test_submissions_views.py \ @@ -95,13 +95,13 @@ jobs: uses: ./.github/actions/load-image-from-cache - name: Prepare docker run: | - docker-compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-rabbitmq test-redis - docker-compose -f docker-compose.test.yml build --parallel workflows test-web - docker-compose -f docker-compose.test.yml up -d test-database test-indexer test-rabbitmq test-redis test-web + docker compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-rabbitmq test-redis + docker compose -f docker-compose.test.yml build --parallel workflows test-web + docker compose -f docker-compose.test.yml up -d test-database test-indexer test-rabbitmq test-redis test-web sh docker/wait-for-services.sh - name: Run tests run: | - docker-compose -f docker-compose.test.yml run --rm workflows py.test \ + docker compose -f docker-compose.test.yml run --rm workflows py.test \ tests/integration/workflows/test_views.py \ tests/integration/workflows \ --ignore tests/integration/workflows/test_arxiv_merge.py \ @@ -133,11 +133,11 @@ jobs: uses: ./.github/actions/load-image-from-cache - name: Prepare docker run: | - docker-compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-redis - docker-compose -f docker-compose.test.yml up -d test-database test-indexer test-redis + docker compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-redis + docker compose -f docker-compose.test.yml up -d test-database test-indexer test-redis sh docker/wait-for-services.sh - name: Run tests - run: docker-compose -f docker-compose.test.yml run --rm integration + run: docker compose -f docker-compose.test.yml run --rm integration Integration-async: runs-on: ubuntu-latest @@ -158,11 +158,11 @@ jobs: uses: ./.github/actions/load-image-from-cache - name: Prepare docker run: | - docker-compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-redis test-rabbitmq - docker-compose -f docker-compose.test.yml up -d test-database test-indexer test-redis test-rabbitmq + docker compose -f docker-compose.test.yml pull --parallel test-database test-indexer test-redis test-rabbitmq + docker compose -f docker-compose.test.yml up -d test-database test-indexer test-redis test-rabbitmq sh docker/wait-for-services.sh - name: Run tests - run: docker-compose -f docker-compose.test.yml run --rm integration_async + run: docker compose -f docker-compose.test.yml run --rm integration_async Unit: needs: Build @@ -183,7 +183,7 @@ jobs: uses: ./.github/actions/load-image-from-cache - name: Run tests run: | - docker-compose -f docker-compose.test.yml run --rm unit + docker compose -f docker-compose.test.yml run --rm unit Release: if: ${{ github.event_name == 'push' || github.event_name == 'release'}} diff --git a/INSTALL.rst b/INSTALL.rst index 9d97db001d..e6cfd75c7b 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -215,7 +215,7 @@ Then create the database and database tables if you haven't already done so: .. code-block:: console - (inspirehep)$ docker-compose up database indexer redis rabbitmq + (inspirehep)$ docker compose up database indexer redis rabbitmq Everything else: Honcho diff --git a/docker-compose.test.yml b/docker-compose.test.yml index b13fbdd8ff..3fb891f780 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -326,8 +326,8 @@ services: - APP_LEGACY_PID_PROVIDER="http://inspirehep.net//batchuploader/allocaterecord" - APP_CRAWLER_HOST_URL=http://test-scrapyd:6800 - # This is needed as docker-compose does not wait for the entry services to - # be actually healthy when doing `docker-compose run e2e`, but only for + # This is needed as docker compose does not wait for the entry services to + # be actually healthy when doing `docker compose run e2e`, but only for # derived dependent services. e2e-deps-placeholder: extends: diff --git a/docker/wait-for-services.sh b/docker/wait-for-services.sh index 9d2d8fb320..39128eba41 100755 --- a/docker/wait-for-services.sh +++ b/docker/wait-for-services.sh @@ -22,11 +22,11 @@ check_ready() { fi done } -_db_check(){ docker-compose -f docker-compose.test.yml exec --user postgres test-database bash -c "pg_isready" &>/dev/null; } +_db_check(){ docker compose -f docker-compose.test.yml exec --user postgres test-database bash -c "pg_isready" &>/dev/null; } check_ready "postgres" _db_check _es_check(){ curl -sL -w "%{http_code}\\n" "http://localhost:9200/" -o /dev/null | grep '200' &> /dev/null; } check_ready "Elasticsearch" _es_check -_redis_check(){ docker-compose -f docker-compose.test.yml exec test-redis bash -c 'redis-cli ping' | grep 'PONG' &> /dev/null; } +_redis_check(){ docker compose -f docker-compose.test.yml exec test-redis bash -c 'redis-cli ping' | grep 'PONG' &> /dev/null; } check_ready "redis" _redis_check diff --git a/docs/docker.rst b/docs/docker.rst index f4b6a3d266..882e0ec824 100644 --- a/docs/docker.rst +++ b/docs/docker.rst @@ -36,7 +36,7 @@ Get the latest Docker appropriate to your operationg system, by visiting `Docker *Get Docker* section. .. note:: If you are using Mac, please build a simple box with ``docker-engine`` above ``1.10`` and - ``docker-compose`` above ``1.6.0``. + ``docker compose V2``. Make sure you can run docker without ``sudo``. @@ -48,7 +48,7 @@ Make sure you can run docker without ``sudo``. - ``sudo usermod -a -G docker $USER`` -Get the latest `docker-compose +Get the latest `docker compose `_: .. code-block:: console @@ -68,7 +68,7 @@ By default the virtualenv and everything else will be kept on ``/tmp`` and they .. Note:: - From now on all the docker-compose commands must be run at the root of the + From now on all the docker compose commands must be run at the root of the inspire-next repository, you can get a local copy with: .. code-block:: console @@ -78,8 +78,8 @@ By default the virtualenv and everything else will be kept on ``/tmp`` and they .. code-block:: console - $ docker-compose pull - $ docker-compose -f docker-compose.deps.yml run --rm pip + $ docker compose pull + $ docker compose -f docker-compose.deps.yml run --rm pip .. note:: If you have trouble with internet connection inside docker probably you are facing known DNS issue. Please follow `this solution @@ -88,19 +88,19 @@ By default the virtualenv and everything else will be kept on ``/tmp`` and they .. code-block:: console - $ docker-compose -f docker-compose.deps.yml run --rm assets + $ docker compose -f docker-compose.deps.yml run --rm assets - Run the service locally .. code-block:: console - $ docker-compose up + $ docker compose up - Populate database .. code-block:: console - $ docker-compose run --rm web scripts/recreate_records + $ docker compose run --rm web scripts/recreate_records Once you have the database populated with the tables and demo records, you can @@ -112,10 +112,10 @@ go to `localhost:5000 `_ .. Note:: - The tests use a different set of containers than the default ``docker-compose + The tests use a different set of containers than the default ``docker compose up``, so if you run both at the same time you might start having ram/load - issues, if so, you can stop all the containers started by ``docker-compose - up`` with ``docker-compose kill -f`` + issues, if so, you can stop all the containers started by ``docker compose + up`` with ``docker compose kill -f`` You can choose one of the following tests types: @@ -126,8 +126,8 @@ You can choose one of the following tests types: .. code-block:: console - $ docker-compose -f docker-compose.test.yml run --rm - $ docker-compose -f docker-compose.test.yml down + $ docker compose -f docker-compose.test.yml run --rm + $ docker compose -f docker-compose.test.yml down .. tip:: - cleanup all the containers: @@ -148,7 +148,7 @@ Extra useful tips .. code-block:: console - $ docker-compose run --rm web inspirehep shell + $ docker compose run --rm web inspirehep shell - Run *virtualenv* bash shell for running scripts manually (e.g. recreating records or `building documentation`_) @@ -156,30 +156,30 @@ Extra useful tips .. code-block:: console - $ docker-compose run --rm web bash + $ docker compose run --rm web bash - Reload code in a worker .. code-block:: console - $ docker-compose restart worker + $ docker compose restart worker - Quick and safe reindex .. code-block:: console - $ docker-compose restart worker && docker-compose run --rm web scripts/recreate_records + $ docker compose restart worker && docker compose run --rm web scripts/recreate_records - Recreate all static assets. Will download all dependencies from npm and copy all static files to ``${DOCKER_DATA}/tmp/virtualenv/var/inspirehep-instance/static``. .. code-block:: console - $ docker-compose -f docker-compose.deps.yml run --rm assets + $ docker compose -f docker-compose.deps.yml run --rm assets - Monitor the output from all the services (elasticsearch, web, celery workers, database, flower, rabbitmq, scrapyd, redis) via the following command: .. code-block:: console - $ docker-compose up + $ docker compose up diff --git a/docs/e2e_tutorial.rst b/docs/e2e_tutorial.rst index f881bda381..7dbf815d85 100644 --- a/docs/e2e_tutorial.rst +++ b/docs/e2e_tutorial.rst @@ -81,7 +81,7 @@ Let us now run this "test" and see what happens: .. code-block:: bash - docker-compose -f docker-compose.test.yml run --rm e2e pytest tests/e2e/test_arxiv_in_hp.py + docker compose -f docker-compose.test.yml run --rm e2e pytest tests/e2e/test_arxiv_in_hp.py Proxy Web UI ++++++++++++ @@ -254,7 +254,7 @@ In order to start the web container (don't forget the ``.local`` at the end!): .. code-block:: bash - docker-compose -f docker-compose.test.yml up test-web-e2e.local + docker compose -f docker-compose.test.yml up test-web-e2e.local For any other container, change the ``test-web-e2e.local`` to the suitable name; other containers don't end in ``.local``, this is needed only for inspire-next node as it has to be a domain name. @@ -266,15 +266,15 @@ To view the logs of a container: .. code-block:: bash - docker-compose -f docker-compose.test.yml logs test-worker-e2e + docker compose -f docker-compose.test.yml logs test-worker-e2e In order to run a shell in an already running container (e.g. to investigate errors): .. code-block:: bash # E.g. for INSPIRE - docker-compose -f docker-compose.test.yml exec test-web-e2e.local bash + docker compose -f docker-compose.test.yml exec test-web-e2e.local bash # For MITM-Proxy we use `ash`, as it runs on Alpine Linux base, which doesn't ship with `bash` - docker-compose -f docker-compose.test.yml exec mitm-proxy ash + docker compose -f docker-compose.test.yml exec mitm-proxy ash diff --git a/docs/harvesting.rst b/docs/harvesting.rst index c3b69743f0..aa23ad3d7e 100644 --- a/docs/harvesting.rst +++ b/docs/harvesting.rst @@ -76,7 +76,7 @@ using docker: .. code-block:: console - docker-compose -f docker-compose.deps.yml run --rm scrapyd-deploy + docker compose -f docker-compose.deps.yml run --rm scrapyd-deploy The simplest way to get records into your system is to harvest from arXiv.org using OAI-PMH. @@ -86,7 +86,7 @@ See `the diagram in hepcrawl documentation