Skip to content

Commit

Permalink
Replace docker-compose with docker compose (#648)
Browse files Browse the repository at this point in the history
  • Loading branch information
jacobtomlinson authored Aug 2, 2024
1 parent 3ee6171 commit 5a4b39b
Show file tree
Hide file tree
Showing 11 changed files with 32 additions and 32 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/build-docker-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ jobs:
- name: Check versions
run: |
docker version
docker-compose version
docker compose version
- name: Building Image
shell: bash -l {0}
run: |
cd ./ci/${{ matrix.jobqueue }}
cp ../environment.yml environment.yml
docker-compose build
docker compose build
- name: List images
run: |
docker ps -a
Expand Down
24 changes: 12 additions & 12 deletions ci/htcondor.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,41 @@

function jobqueue_before_install {
docker version
docker-compose version
docker compose version

# start htcondor cluster
cd ./ci/htcondor
docker-compose pull
docker compose pull
./start-htcondor.sh
docker-compose exec -T submit /bin/bash -c "condor_status"
docker-compose exec -T submit /bin/bash -c "condor_q"
docker compose exec -T submit /bin/bash -c "condor_status"
docker compose exec -T submit /bin/bash -c "condor_q"
cd -

#Set shared space permissions
docker-compose exec -T submit /bin/bash -c "chmod -R 777 /shared_space"
docker compose exec -T submit /bin/bash -c "chmod -R 777 /shared_space"

docker ps -a
docker images
}

function jobqueue_install {
cd ./ci/htcondor
docker-compose exec -T submit conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip3 install -e .;chown -R submituser ."
docker compose exec -T submit conda run -n dask-jobqueue /bin/bash -c "cd /dask-jobqueue; pip3 install -e .;chown -R submituser ."
cd -
}

function jobqueue_script {
cd ./ci/htcondor
docker-compose exec -T --user submituser submit conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --log-cli-level DEBUG --capture=tee-sys --verbose -E htcondor "
docker compose exec -T --user submituser submit conda run -n dask-jobqueue /bin/bash -c "cd; pytest /dask-jobqueue/dask_jobqueue --log-cli-level DEBUG --capture=tee-sys --verbose -E htcondor "
cd -
}

function jobqueue_after_script {
cd ./ci/htcondor
docker-compose exec -T --user submituser submit /bin/bash -c "condor_q"
docker-compose exec -T submit /bin/bash -c "condor_status"
docker-compose exec -T --user submituser submit /bin/bash -c "condor_history"
docker-compose exec -T --user submituser submit /bin/bash -c "cd; cat logs/*"
docker-compose exec -T cm /bin/bash -c " grep -R \"\" /var/log/condor/ "
docker compose exec -T --user submituser submit /bin/bash -c "condor_q"
docker compose exec -T submit /bin/bash -c "condor_status"
docker compose exec -T --user submituser submit /bin/bash -c "condor_history"
docker compose exec -T --user submituser submit /bin/bash -c "cd; cat logs/*"
docker compose exec -T cm /bin/bash -c " grep -R \"\" /var/log/condor/ "
cd -
}
4 changes: 2 additions & 2 deletions ci/htcondor/start-htcondor.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/bin/bash

docker-compose up -d --no-build
docker compose up -d --no-build

while [ `docker-compose exec -T submit condor_status -af activity|grep Idle|wc -l` -ne 2 ]
while [ `docker compose exec -T submit condor_status -af activity|grep Idle|wc -l` -ne 2 ]
do
echo "Waiting for cluster to become ready";
sleep 2
Expand Down
4 changes: 2 additions & 2 deletions ci/pbs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

function jobqueue_before_install {
docker version
docker-compose version
docker compose version

# start pbs cluster
cd ./ci/pbs
docker-compose pull
docker compose pull
./start-pbs.sh
cd -

Expand Down
2 changes: 1 addition & 1 deletion ci/pbs/start-pbs.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

docker-compose up -d --no-build
docker compose up -d --no-build
while [ `docker exec -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ]
do
echo "Waiting for PBS slave nodes to become available";
Expand Down
4 changes: 2 additions & 2 deletions ci/sge.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

function jobqueue_before_install {
docker version
docker-compose version
docker compose version

# start sge cluster
cd ./ci/sge
docker-compose pull
docker compose pull
./start-sge.sh
cd -

Expand Down
2 changes: 1 addition & 1 deletion ci/sge/start-sge.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

docker-compose up -d --no-build
docker compose up -d --no-build

START=$(date +%s)
MAX_WAIT_SECONDS=300
Expand Down
4 changes: 2 additions & 2 deletions ci/slurm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

function jobqueue_before_install {
docker version
docker-compose version
docker compose version

# start slurm cluster
cd ./ci/slurm
docker-compose pull
docker compose pull
./start-slurm.sh
cd -

Expand Down
2 changes: 1 addition & 1 deletion ci/slurm/register_cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
set -e

docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && \
docker-compose restart slurmdbd slurmctld
docker compose restart slurmdbd slurmctld
2 changes: 1 addition & 1 deletion ci/slurm/start-slurm.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

docker-compose up -d --no-build
docker compose up -d --no-build

while [ `./register_cluster.sh 2>&1 | grep "sacctmgr: error" | wc -l` -ne 0 ]
do
Expand Down
12 changes: 6 additions & 6 deletions docs/source/develop.rst
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,17 @@ This is basically a simplified version of what is in the ci/*.sh files.
For example with Slurm::

cd ci/slurm
docker-compose pull
docker compose pull
# Start a Slurm dockerized cluster
./start-slurm.sh #which is doing docker-compose up -d --no-build
./start-slurm.sh #which is doing docker compose up -d --no-build
# Install dask-jobqueue in Docker container
docker exec slurmctld /bin/bash -c "cd /dask-jobqueue; pip install -e ."
# Run the tests for slurm
docker exec slurmctld /bin/bash -c "pytest /dask-jobqueue/dask_jobqueue --verbose -E slurm -s"

You can then shutdown the dockerized cluster and remove all the containers from your computer::

docker-compose down
docker compose down

Test on a real Job queuing system
---------------------------------
Expand Down Expand Up @@ -107,12 +107,12 @@ For Slurm for example::

cd ci/slurm
cp ../environment.yml environment.yml #The Dockerfile needs the reference Conda environment file in its context to build
docker-compose build
docker compose build

You might want to stop your dockerized cluster and refresh the build if you have done this previously::

docker-compose down
docker-compose build --no-cache
docker compose down
docker compose build --no-cache

Update Docker images for CI tests
---------------------------------
Expand Down

0 comments on commit 5a4b39b

Please sign in to comment.