From d7ecbb8a4503186d7cba12c2af40e57543e694ea Mon Sep 17 00:00:00 2001 From: ll7 Date: Mon, 17 Jun 2024 12:31:59 +0200 Subject: [PATCH 01/10] feat: Add Docker Compose file for Carla Simulator in Swarm mode --- build/docker-compose.swarm.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 build/docker-compose.swarm.yml diff --git a/build/docker-compose.swarm.yml b/build/docker-compose.swarm.yml new file mode 100644 index 00000000..47d9d694 --- /dev/null +++ b/build/docker-compose.swarm.yml @@ -0,0 +1,26 @@ +version: "3.8" + +services: + carla-simulator: + # command: /bin/bash CarlaUE4.sh -quality-level=High -world-port=2000 -resx=800 -resy=600 -nosound -carla-settings="/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini" + command: /bin/bash CarlaUE4.sh -quality-level=High -world-port=2000 -resx=800 -resy=600 -nosound + image: ghcr.io/una-auxme/paf23:leaderboard-2.0 + init: true + deploy: + resources: + limits: + memory: 16G + expose: + - 2000 + - 2001 + - 2002 + environment: + - XDG_RUNTIME_DIR=/tmp/runtime-carla + networks: + - carla + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix + # - ./CustomCarlaSettings.ini:/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini + +networks: + carla: From aa6b9546bd972b97faa4d8106c2c98e606963052 Mon Sep 17 00:00:00 2001 From: ll7 Date: Mon, 17 Jun 2024 13:16:32 +0200 Subject: [PATCH 02/10] feat: start with a simple carla image --- build/docker-compose.carla-server.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 build/docker-compose.carla-server.yaml diff --git a/build/docker-compose.carla-server.yaml b/build/docker-compose.carla-server.yaml new file mode 100644 index 00000000..245f9694 --- /dev/null +++ b/build/docker-compose.carla-server.yaml @@ -0,0 +1,14 @@ +version: '3.8' +services: + carla-simulator: + image: carlasim/carla:latest + command: /bin/bash ./CarlaUE4.sh + environment: + - DISPLAY=${DISPLAY} + privileged: true + network_mode: host + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] \ No newline at end of file From a70435ec3a5c0bac2d079c9c194e667b7a4ffe39 Mon Sep 17 00:00:00 2001 From: ll7 Date: Mon, 17 Jun 2024 13:22:10 +0200 Subject: [PATCH 03/10] feat: Update Docker Compose file for Carla Simulator --- build/docker-compose.carla-server.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/docker-compose.carla-server.yaml b/build/docker-compose.carla-server.yaml index 245f9694..8ca8803c 100644 --- a/build/docker-compose.carla-server.yaml +++ b/build/docker-compose.carla-server.yaml @@ -1,8 +1,7 @@ -version: '3.8' services: carla-simulator: - image: carlasim/carla:latest - command: /bin/bash ./CarlaUE4.sh + image: ghcr.io/una-auxme/paf23:leaderboard-2.0 + command: /bin/bash ./CarlaUE4.sh quality-level=Low -resx=800 -resy=600 -nosound -world-port=2000 environment: - DISPLAY=${DISPLAY} privileged: true From 9ae302f3ee08904ba96d2ef4dfe1daf8c06f3c18 Mon Sep 17 00:00:00 2001 From: ll7 Date: Tue, 18 Jun 2024 17:14:54 +0200 Subject: [PATCH 04/10] feat: Add Docker Swarm notes for GPU configuration --- build/docker-swarm-notes.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 build/docker-swarm-notes.md diff --git a/build/docker-swarm-notes.md b/build/docker-swarm-notes.md new file mode 100644 index 00000000..237645db --- /dev/null +++ b/build/docker-swarm-notes.md @@ -0,0 +1,33 @@ +# Docker Swarm Notes + +```bash +GPU-0f76ba56 + +sudo docker service create --replicas 1 \ + --cap-add all \ + --network host \ + --name carla-server \ + --generic-resource "NVIDIA-GPU=0" \ + -e DISPLAY=$DISPLAY \ + carlasim/carla:latest \ + /bin/bash ./CarlaUE4.sh + + +sudo docker run --privileged --gpus all --net=host -e DISPLAY=$DISPLAY carlasim/carla:latest /bin/bash ./CarlaUE4.sh + + docker service create --replicas 1 \ + --name tensor-qs \ + --generic-resource "gpu=1" \ + tomlankhorst/tensorflow-quickstart + + + -e XDG_RUNTIME_DIR=/tmp/runtime-carla \ + + +sudo docker service create \ + --name carla-server \ + --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock,ro \ + docker \ + docker run --privileged --gpus all --net=host -e DISPLAY=$DISPLAY ghcr.io/una-auxme/paf23:leaderboard-2.0 /bin/bash ./CarlaUE4.sh + +``` From 2f98ae4e826294d4140d0e937a182b1513aad713 Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 28 Jun 2024 13:34:34 +0200 Subject: [PATCH 05/10] feat: Add documentation for distributed simulation setup --- .../14_distributed_simulation.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 doc/02_development/14_distributed_simulation.md diff --git a/doc/02_development/14_distributed_simulation.md b/doc/02_development/14_distributed_simulation.md new file mode 100644 index 00000000..ba0319e7 --- /dev/null +++ b/doc/02_development/14_distributed_simulation.md @@ -0,0 +1,27 @@ +# Distributed Simulation + +If you have not enough compute resources, start the `carla-simulator-server' on a remote machine and execute the agent on your local machine. + +## Author + +Julian Trommer and Lennart Luttkus + +## Date + +2024-06-28 + +## Remote Machine Setup + +- Gain `ssh` access to the remote machine. +- Download the [Carla leaderboard release](https://leaderboard.carla.org/get_started/#11-download-the-carla-leaderboard-package) to the remote PC +- Extract the `.zip` file. +- start the server per ssh without rendering the spectator view: + - `/CARLA_Leaderboard_20/CarlaUE4.sh -RenderOffScreen` + +## Local Machine Setup + +- get access to the remote machine via `ssh` + - Start the server as described above +- set the host ip address from the remote machine as the new carla-ip address +- start the agent on your local machine + \ No newline at end of file From a9ee519f9475d301feeb39727c3bd19527f8e241 Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 28 Jun 2024 13:39:48 +0200 Subject: [PATCH 06/10] feat: Add command to monitor GPU usage --- doc/02_development/14_distributed_simulation.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/02_development/14_distributed_simulation.md b/doc/02_development/14_distributed_simulation.md index ba0319e7..5fa139f9 100644 --- a/doc/02_development/14_distributed_simulation.md +++ b/doc/02_development/14_distributed_simulation.md @@ -24,4 +24,9 @@ Julian Trommer and Lennart Luttkus - Start the server as described above - set the host ip address from the remote machine as the new carla-ip address - start the agent on your local machine - \ No newline at end of file + +## How do you know that you do not have enough compute resources? + +```bash +watch -n 1 nvidia-smi +``` From 913a2a20b131dc45546bf8057933953987864923 Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 28 Jun 2024 13:53:26 +0200 Subject: [PATCH 07/10] feat: Update distributed simulation documentation --- .../14_distributed_simulation.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/doc/02_development/14_distributed_simulation.md b/doc/02_development/14_distributed_simulation.md index 5fa139f9..75a981d7 100644 --- a/doc/02_development/14_distributed_simulation.md +++ b/doc/02_development/14_distributed_simulation.md @@ -1,6 +1,7 @@ # Distributed Simulation -If you have not enough compute resources, start the `carla-simulator-server' on a remote machine and execute the agent on your local machine. +If you have not enough compute resources, start the `carla-simulator-server` on a remote machine and execute the agent on your local machine. +As far as we know, you need more than **10 GB of VRAM** to run the server and the agent on the same machine. ## Author @@ -25,6 +26,35 @@ Julian Trommer and Lennart Luttkus - set the host ip address from the remote machine as the new carla-ip address - start the agent on your local machine +### Ensure similarity between normal docker-compose and distributed docker-compose files + +Carefully compare that their are no major differences between the `docker-compose.yml` and `docker-compose.distributed.yml` files. +Mainly, the `carla-simulator` service will not be executed in the non-distributed version. + +### Set the `` of the carla simulator in docker-compose distributed files + +Replace the argument `` with the ip address of the remote machine. +You can find the ip address of the remote machine by executing the following command on the remote machine: + +```bash +hostname -I +``` + +Typically, the ip address is the first one in the list. +`172.xxx.xxx.xxx` is the localhost address and not the relevant address. + +Replace the ip-address in the following files: + +- `docker-compose.distributed.yml` +- `docker-compose.dev.distributed.yml` + +### Start the agent on your local machine + +```bash +b5 run_distributed +b5 run_dev_distributed +``` + ## How do you know that you do not have enough compute resources? ```bash From a0f45333196fcaab58b50b854070e98fd0f75e75 Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 28 Jun 2024 13:57:12 +0200 Subject: [PATCH 08/10] feat: remove swarm notes as it is easier to use ssh --- build/docker-compose.carla-server.yaml | 13 ---------- build/docker-compose.swarm.yml | 26 -------------------- build/docker-swarm-notes.md | 33 -------------------------- 3 files changed, 72 deletions(-) delete mode 100644 build/docker-compose.carla-server.yaml delete mode 100644 build/docker-compose.swarm.yml delete mode 100644 build/docker-swarm-notes.md diff --git a/build/docker-compose.carla-server.yaml b/build/docker-compose.carla-server.yaml deleted file mode 100644 index 8ca8803c..00000000 --- a/build/docker-compose.carla-server.yaml +++ /dev/null @@ -1,13 +0,0 @@ -services: - carla-simulator: - image: ghcr.io/una-auxme/paf23:leaderboard-2.0 - command: /bin/bash ./CarlaUE4.sh quality-level=Low -resx=800 -resy=600 -nosound -world-port=2000 - environment: - - DISPLAY=${DISPLAY} - privileged: true - network_mode: host - deploy: - resources: - reservations: - devices: - - capabilities: ["gpu"] \ No newline at end of file diff --git a/build/docker-compose.swarm.yml b/build/docker-compose.swarm.yml deleted file mode 100644 index 47d9d694..00000000 --- a/build/docker-compose.swarm.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: "3.8" - -services: - carla-simulator: - # command: /bin/bash CarlaUE4.sh -quality-level=High -world-port=2000 -resx=800 -resy=600 -nosound -carla-settings="/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini" - command: /bin/bash CarlaUE4.sh -quality-level=High -world-port=2000 -resx=800 -resy=600 -nosound - image: ghcr.io/una-auxme/paf23:leaderboard-2.0 - init: true - deploy: - resources: - limits: - memory: 16G - expose: - - 2000 - - 2001 - - 2002 - environment: - - XDG_RUNTIME_DIR=/tmp/runtime-carla - networks: - - carla - volumes: - - /tmp/.X11-unix:/tmp/.X11-unix - # - ./CustomCarlaSettings.ini:/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini - -networks: - carla: diff --git a/build/docker-swarm-notes.md b/build/docker-swarm-notes.md deleted file mode 100644 index 237645db..00000000 --- a/build/docker-swarm-notes.md +++ /dev/null @@ -1,33 +0,0 @@ -# Docker Swarm Notes - -```bash -GPU-0f76ba56 - -sudo docker service create --replicas 1 \ - --cap-add all \ - --network host \ - --name carla-server \ - --generic-resource "NVIDIA-GPU=0" \ - -e DISPLAY=$DISPLAY \ - carlasim/carla:latest \ - /bin/bash ./CarlaUE4.sh - - -sudo docker run --privileged --gpus all --net=host -e DISPLAY=$DISPLAY carlasim/carla:latest /bin/bash ./CarlaUE4.sh - - docker service create --replicas 1 \ - --name tensor-qs \ - --generic-resource "gpu=1" \ - tomlankhorst/tensorflow-quickstart - - - -e XDG_RUNTIME_DIR=/tmp/runtime-carla \ - - -sudo docker service create \ - --name carla-server \ - --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock,ro \ - docker \ - docker run --privileged --gpus all --net=host -e DISPLAY=$DISPLAY ghcr.io/una-auxme/paf23:leaderboard-2.0 /bin/bash ./CarlaUE4.sh - -``` From a890f26e780a202779ac72a4ac80ed263351dec7 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Fri, 28 Jun 2024 14:49:53 +0200 Subject: [PATCH 09/10] feat: added distributed docker compose files --- build/Taskfile | 10 +++ build/docker-compose.dev.distributed.yaml | 77 +++++++++++++++++++++++ build/docker-compose.dev.yaml | 15 +++++ build/docker-compose.distributed.yaml | 77 +++++++++++++++++++++++ build/docker-compose.yml | 17 ++++- 5 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 build/docker-compose.dev.distributed.yaml create mode 100644 build/docker-compose.distributed.yaml diff --git a/build/Taskfile b/build/Taskfile index b152bc75..580d3ad0 100644 --- a/build/Taskfile +++ b/build/Taskfile @@ -16,6 +16,16 @@ task:run_dev() { docker:compose -f docker-compose.dev.yaml up } +task:run_distributed() { + xhost +local:docker + docker:compose -f docker-compose.distributed.yaml up +} + +task:run_dev_distributed() { + xhost +local:docker + docker:compose -f docker-compose.dev.distributed.yaml up +} + task:restart() { container="$1" docker:compose restart "${container:-agent}" diff --git a/build/docker-compose.dev.distributed.yaml b/build/docker-compose.dev.distributed.yaml new file mode 100644 index 00000000..5eba6099 --- /dev/null +++ b/build/docker-compose.dev.distributed.yaml @@ -0,0 +1,77 @@ +version: "3" + +services: + flake8: + image: alpine/flake8 + command: . + volumes: + - ../:/apps + + comlipy: + build: docker/comlipy + command: . + volumes: + - ../:/apps + + mdlint: + image: peterdavehello/markdownlint:0.32.2 + command: markdownlint . + volumes: + - ../:/md + + roscore: + image: ros:noetic + command: roscore + environment: + - ROS_MASTER_URI=http://roscore:11311 + - ROS_HOSTNAME=roscore + expose: + - 11311 + networks: + - ros + + agent: + build: + dockerfile: build/docker/agent/Dockerfile + args: + - USER_UID=${DOCKER_HOST_UNIX_UID:-1000} + - USER_GID=${DOCKER_HOST_UNIX_GID:-1000} + context: ../ + init: true + tty: true + shm_size: 2gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] + #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS" + command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch" + # command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP" + + logging: + driver: "local" + environment: + - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all + - ROS_MASTER_URI=http://roscore:11311 + - CARLA_SIM_HOST= + - ROS_HOSTNAME=agent + - XDG_RUNTIME_DIR=/tmp/runtime-carla + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix + # if you change the volume here also change the copy command + # in ``build/docker/build/Dockerfile + - ../:/workspace + # mount git config for dvc + - ../.gitconfig:/home/carla/.gitconfig + - ../:/workspace/ + networks: + - carla + - ros + +networks: + carla: + ros: diff --git a/build/docker-compose.dev.yaml b/build/docker-compose.dev.yaml index 6ba17e3d..3e9980f5 100644 --- a/build/docker-compose.dev.yaml +++ b/build/docker-compose.dev.yaml @@ -28,11 +28,18 @@ services: resources: limits: memory: 16G + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] expose: - 2000 - 2001 - 2002 environment: + - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all - XDG_RUNTIME_DIR=/tmp/runtime-carla networks: - carla @@ -61,6 +68,12 @@ services: init: true tty: true shm_size: 2gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS" command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch" # command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP" @@ -69,6 +82,8 @@ services: driver: "local" environment: - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all - ROS_MASTER_URI=http://roscore:11311 - CARLA_SIM_HOST=carla-simulator - ROS_HOSTNAME=agent diff --git a/build/docker-compose.distributed.yaml b/build/docker-compose.distributed.yaml new file mode 100644 index 00000000..ecffb8eb --- /dev/null +++ b/build/docker-compose.distributed.yaml @@ -0,0 +1,77 @@ +version: "3" + +services: + flake8: + image: alpine/flake8 + command: . + volumes: + - ../:/apps + + comlipy: + build: docker/comlipy + command: . + volumes: + - ../:/apps + + mdlint: + image: peterdavehello/markdownlint:0.32.2 + command: markdownlint . + volumes: + - ../:/md + + roscore: + image: ros:noetic + command: roscore + environment: + - ROS_MASTER_URI=http://roscore:11311 + - ROS_HOSTNAME=roscore + expose: + - 11311 + networks: + - ros + + agent: + build: + dockerfile: build/docker/agent/Dockerfile + args: + - USER_UID=${DOCKER_HOST_UNIX_UID:-1000} + - USER_GID=${DOCKER_HOST_UNIX_GID:-1000} + context: ../ + init: true + tty: true + shm_size: 2gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] + #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS" + #command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch" + command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host= --track=MAP" + + logging: + driver: "local" + environment: + - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all + - ROS_MASTER_URI=http://roscore:11311 + - CARLA_SIM_HOST= + - ROS_HOSTNAME=agent + - XDG_RUNTIME_DIR=/tmp/runtime-carla + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix + # if you change the volume here also change the copy command + # in ``build/docker/build/Dockerfile + - ../:/workspace + # mount git config for dvc + - ../.gitconfig:/home/carla/.gitconfig + - ../:/workspace/ + networks: + - carla + - ros + +networks: + carla: + ros: diff --git a/build/docker-compose.yml b/build/docker-compose.yml index 6c3506be..eeece9ed 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -28,11 +28,18 @@ services: resources: limits: memory: 16G + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] expose: - 2000 - 2001 - 2002 environment: + - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all - XDG_RUNTIME_DIR=/tmp/runtime-carla networks: - carla @@ -61,14 +68,22 @@ services: init: true tty: true shm_size: 2gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + capabilities: [ gpu ] #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS" #command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch" - command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP" + command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=carla-simulator --track=MAP" logging: driver: "local" environment: - DISPLAY + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=all - ROS_MASTER_URI=http://roscore:11311 - CARLA_SIM_HOST=carla-simulator - ROS_HOSTNAME=agent From e0b7d116c6a8e7ee03c5f267dc7fa7e5a92678ae Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 28 Jun 2024 15:06:29 +0200 Subject: [PATCH 10/10] feat: Specify a specific 'b5' workflow for GPU --- doc/01_general/03_commands.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/01_general/03_commands.md b/doc/01_general/03_commands.md index 3c7f8478..3a731218 100644 --- a/doc/01_general/03_commands.md +++ b/doc/01_general/03_commands.md @@ -1,5 +1,7 @@ # ⌨️ Available commands +A specific `b5` workflow for gpu installation in this project is specified in an issue comment: + ## General commands ### `b5 run`