From 8c043bb7b2dc2795ae4efaf0700d2a54e1ab0762 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Wed, 9 Oct 2024 09:43:31 +0200 Subject: [PATCH 1/4] Renamed compose files for execution --- .vscode/extensions.json | 3 ++- .vscode/settings.json | 9 ++++++++- ..._service.yaml => docker-compose.carla-simulator.yaml} | 0 ...docker-compose_cicd.yaml => docker-compose.cicd.yaml} | 2 +- ...-compose_dev_offline.yaml => docker-compose.dev.yaml} | 0 ...ted.yaml => docker-compose.devroute-distributed.yaml} | 2 +- ...ker-compose_dev.yaml => docker-compose.devroute.yaml} | 4 ++-- ....yaml => docker-compose.leaderboard-distributed.yaml} | 2 +- ...cker-compose.yaml => docker-compose.leaderboard.yaml} | 4 ++-- .../{linter_services.yaml => docker-compose.linter.yaml} | 0 10 files changed, 17 insertions(+), 9 deletions(-) rename build/{carla-simulator_service.yaml => docker-compose.carla-simulator.yaml} (100%) rename build/{docker-compose_cicd.yaml => docker-compose.cicd.yaml} (94%) rename build/{docker-compose_dev_offline.yaml => docker-compose.dev.yaml} (100%) rename build/{docker-compose_dev_distributed.yaml => docker-compose.devroute-distributed.yaml} (92%) rename build/{docker-compose_dev.yaml => docker-compose.devroute.yaml} (88%) rename build/{docker-compose_distributed.yaml => docker-compose.leaderboard-distributed.yaml} (93%) rename build/{docker-compose.yaml => docker-compose.leaderboard.yaml} (84%) rename build/{linter_services.yaml => docker-compose.linter.yaml} (100%) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index ed5f4b16..6e4fc554 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -10,6 +10,7 @@ "ms-python.flake8", "bierner.markdown-mermaid", "richardkotze.git-mob", - "ms-vscode-remote.remote-containers" + "ms-vscode-remote.remote-containers", + "valentjn.vscode-ltex" ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index b7873462..8d42024f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,5 +19,12 @@ "query": "state:open repo:${owner}/${repository} sort:created-desc" } ], - "ltex.language": "en-US" + "ltex.language": "en-US", + "docker.commands.composeUp": [ + { + "label": "Compose Up", + "template": "xhost +local: && ${composeCommand} ${configurationFile} up" + } + ], + "workbench.iconTheme": "vscode-icons" } \ No newline at end of file diff --git a/build/carla-simulator_service.yaml b/build/docker-compose.carla-simulator.yaml similarity index 100% rename from build/carla-simulator_service.yaml rename to build/docker-compose.carla-simulator.yaml diff --git a/build/docker-compose_cicd.yaml b/build/docker-compose.cicd.yaml similarity index 94% rename from build/docker-compose_cicd.yaml rename to build/docker-compose.cicd.yaml index 7fd57177..2d7a47d5 100644 --- a/build/docker-compose_cicd.yaml +++ b/build/docker-compose.cicd.yaml @@ -2,7 +2,7 @@ include: # linter runs in a seperate workflow - roscore_service.yaml - - carla-simulator_service.yaml + - docker-compose.carla-simulator.yaml services: agent: diff --git a/build/docker-compose_dev_offline.yaml b/build/docker-compose.dev.yaml similarity index 100% rename from build/docker-compose_dev_offline.yaml rename to build/docker-compose.dev.yaml diff --git a/build/docker-compose_dev_distributed.yaml b/build/docker-compose.devroute-distributed.yaml similarity index 92% rename from build/docker-compose_dev_distributed.yaml rename to build/docker-compose.devroute-distributed.yaml index e8f8b127..e8ed7e9f 100644 --- a/build/docker-compose_dev_distributed.yaml +++ b/build/docker-compose.devroute-distributed.yaml @@ -1,6 +1,6 @@ # compose file for the development environment with distributed mode include: - - linter_services.yaml + - docker-compose.linter.yaml - roscore_service.yaml services: diff --git a/build/docker-compose_dev.yaml b/build/docker-compose.devroute.yaml similarity index 88% rename from build/docker-compose_dev.yaml rename to build/docker-compose.devroute.yaml index 1b9129de..6f04d601 100644 --- a/build/docker-compose_dev.yaml +++ b/build/docker-compose.devroute.yaml @@ -1,9 +1,9 @@ # compose file for the development environment # routes_simple.xml include: - - linter_services.yaml + - docker-compose.linter.yaml - roscore_service.yaml - - carla-simulator_service.yaml + - docker-compose.carla-simulator.yaml services: agent: diff --git a/build/docker-compose_distributed.yaml b/build/docker-compose.leaderboard-distributed.yaml similarity index 93% rename from build/docker-compose_distributed.yaml rename to build/docker-compose.leaderboard-distributed.yaml index cd600a02..464c1aa6 100644 --- a/build/docker-compose_distributed.yaml +++ b/build/docker-compose.leaderboard-distributed.yaml @@ -1,5 +1,5 @@ include: - - linter_services.yaml + - docker-compose.linter.yaml - roscore_service.yaml services: diff --git a/build/docker-compose.yaml b/build/docker-compose.leaderboard.yaml similarity index 84% rename from build/docker-compose.yaml rename to build/docker-compose.leaderboard.yaml index 44acaf62..93a669fc 100644 --- a/build/docker-compose.yaml +++ b/build/docker-compose.leaderboard.yaml @@ -1,7 +1,7 @@ include: - - linter_services.yaml + - docker-compose.linter.yaml - roscore_service.yaml - - carla-simulator_service.yaml + - docker-compose.carla-simulator.yaml services: agent: diff --git a/build/linter_services.yaml b/build/docker-compose.linter.yaml similarity index 100% rename from build/linter_services.yaml rename to build/docker-compose.linter.yaml From 0aece19578fff54f76aac175e5ece7afa7d5161a Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Wed, 9 Oct 2024 09:46:29 +0200 Subject: [PATCH 2/4] Removed obsolete shell scripts --- dc-run-file.sh | 13 ------------- pc_setup_user.sh | 3 --- xhost_enable.sh | 4 ---- 3 files changed, 20 deletions(-) delete mode 100755 dc-run-file.sh delete mode 100755 xhost_enable.sh diff --git a/dc-run-file.sh b/dc-run-file.sh deleted file mode 100755 index 8f182b83..00000000 --- a/dc-run-file.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# run docker compose file specified as argument and located in the build directory - -# enable xhost for the current user to allow docker to display graphics -./xhost_enable.sh - -# run docker compose -if [ $# -eq 0 ]; then - echo "Usage: $0 " - exit 1 -fi - -docker compose -f "$1" up diff --git a/pc_setup_user.sh b/pc_setup_user.sh index 7efd778d..23b931f9 100755 --- a/pc_setup_user.sh +++ b/pc_setup_user.sh @@ -2,6 +2,3 @@ cd mkdir git cd git git clone https://github.com/una-auxme/paf.git - -cd paf -./dc-run-file.sh build/docker-compose.yaml \ No newline at end of file diff --git a/xhost_enable.sh b/xhost_enable.sh deleted file mode 100755 index e80e040d..00000000 --- a/xhost_enable.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# enable xhost for the current user to allow docker to display graphics -xhost +local: \ No newline at end of file From ca4fdaee113ab9d4c0c603015dc64bbe36dabbb9 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Wed, 9 Oct 2024 10:09:25 +0200 Subject: [PATCH 3/4] Updated build documentation --- build/README.md | 94 +++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/build/README.md b/build/README.md index 7665c425..6a4e7861 100644 --- a/build/README.md +++ b/build/README.md @@ -12,13 +12,14 @@ facilitating both normal and distributed execution modes. - [Directory Structure](#directory-structure) - [Base Service Files](#base-service-files) - [`agent_service.yaml`](#agent_serviceyaml) - - [`carla-simulator_service.yaml`](#carla-simulator_serviceyaml) - - [`linter_services.yaml`](#linter_servicesyaml) - [`roscore_service.yaml`](#roscore_serviceyaml) - [Docker Compose Files](#docker-compose-files) - - [`docker-compose.yaml`](#docker-composeyaml) - - [`docker-compose_dev.yaml`](#docker-compose_devyaml) - - [`docker-compose_cicd.yaml`](#docker-compose_cicdyaml) + - [`docker-compose.carla-simulator.yaml`](#docker-composecarla-simulatoryaml) + - [`docker-compose.linter.yaml`](#docker-composelinteryaml) + - [`docker-compose.leaderboard.yaml`](#docker-composeleaderboardyaml) + - [`docker-compose.devroute.yaml`](#docker-composedevrouteyaml) + - [`docker-compose.dev.yaml`](#docker-composedevyaml) + - [`docker-compose.cicd.yaml`](#docker-composecicdyaml) - [Execution Modes](#execution-modes) - [Normal Execution](#normal-execution) - [Distributed Execution](#distributed-execution) @@ -32,13 +33,14 @@ The `build` directory contains the necessary configuration and setup files for b - **Base Service Files** - `agent_service.yaml` - - `carla-simulator_service.yaml` - - `linter_services.yaml` - `roscore_service.yaml` - **Docker Compose Files** - - `docker-compose.yaml` - - `docker-compose_dev.yaml` - - `docker-compose_cicd.yaml` + - `docker-compose.carla-simulator.yaml` + - `docker-compose.linter.yaml` + - `docker-compose.leaderboard.yaml` + - `docker-compose.devroute.yaml` + - `docker-compose.dev.yaml` + - `docker-compose.cicd.yaml` ## Base Service Files @@ -53,7 +55,20 @@ Defines the configuration for the `agent` service, which represents the autonomo - **Volumes**: Mounts directories like `/workspace` to share code and data between the host and the container. - **Networks**: Connects the agent to the `carla` and `ros` networks. -### `carla-simulator_service.yaml` +### `roscore_service.yaml` + +Defines the `roscore` service for running the ROS master node. Key configurations include: + +- **Image**: Uses the official ROS Noetic image. +- **Command**: Starts `roscore`. +- **Environment Variables**: Sets up ROS networking variables. +- **Networks**: Connects to the `ros` network. + +## Docker Compose Files + +The Docker Compose files allow the execution of different components or whole scenarios that include multiple services. + +### `docker-compose.carla-simulator.yaml` Defines the configuration for the `carla-simulator` service, which runs the CARLA simulator. Key configurations include: @@ -63,7 +78,7 @@ Defines the configuration for the `carla-simulator` service, which runs the CARL - **Volumes**: Shares the X11 UNIX socket and custom CARLA settings. - **Networks**: Connects to the `carla` network. -### `linter_services.yaml` +### `docker-compose.linter.yaml` Defines services for code linting and static analysis. Includes: @@ -71,35 +86,22 @@ Defines services for code linting and static analysis. Includes: - **mdlint**: For Markdown file linting. - **Volumes**: Mounts the project directory for linting files within the container. -### `roscore_service.yaml` - -Defines the `roscore` service for running the ROS master node. Key configurations include: - -- **Image**: Uses the official ROS Noetic image. -- **Command**: Starts `roscore`. -- **Environment Variables**: Sets up ROS networking variables. -- **Networks**: Connects to the `ros` network. - -## Docker Compose Files - -The Docker Compose files orchestrate multiple services defined in the base service files, allowing for different execution scenarios. - -### `docker-compose.yaml` +### `docker-compose.leaderboard.yaml` - **Includes**: - - `linter_services.yaml` + - `docker-compose.linter.yaml` + - `docker-compose.carla-simulator.yaml` - `roscore_service.yaml` - - `carla-simulator_service.yaml` - **Services**: - Extends the `agent` service from `agent_service.yaml`. - **Purpose**: Runs the agent with special scenarios included. Solving these scenarios is the primary goal of the project. -### `docker-compose_dev.yaml` +### `docker-compose.devroute.yaml` - **Includes**: - - `linter_services.yaml` + - `docker-compose.linter.yaml` + - `docker-compose.carla-simulator.yaml` - `roscore_service.yaml` - - `carla-simulator_service.yaml` - **Services**: - Extends the `agent` service from `agent_service.yaml`. - **Environment Overrides**: @@ -108,11 +110,17 @@ The Docker Compose files orchestrate multiple services defined in the base servi - Runs the agent with simplified settings suitable for development and testing. - **Purpose**: Provides a minimal setup for development without special scenarios. -### `docker-compose_cicd.yaml` +### `docker-compose.dev.yaml` + +- **Services**: + - Defines an `agent-dev` service using the corresponding Dockerfile. +- **Purpose**: Provides a container for attaching a VS Code instance for development. + +### `docker-compose.cicd.yaml` - **Includes**: + - `docker-compose.carla-simulator.yaml` - `roscore_service.yaml` - - `carla-simulator_service.yaml` - **Services**: - Defines an `agent` service using a prebuilt image from the project's container registry. - **Dependencies**: @@ -137,34 +145,22 @@ Distributed execution separates the agent and the CARLA simulator onto different - Running large vision models that require extensive VRAM. - The single machine's resources are insufficient to handle both the agent and simulator. -**Note**: In distributed execution, the CARLA simulator must be running on a second desktop PC, and the `CARLA_SIM_HOST` environment variable should be set accordingly. +**Note**: In distributed execution, the CARLA simulator must be running on a second desktop PC, and the `CARLA_SIM_HOST` environment variable should be set accordingly. Further information can be found in [here](../doc/02_development/14_distributed_simulation.md). ## Usage -To run the project using the provided Docker Compose files: - -- **Standard Execution with Special Scenarios**: - - ```bash - docker-compose -f build/docker-compose.yaml up - ``` - -- **Development Execution without Special Scenarios**: - - ```bash - docker-compose -f build/docker-compose_dev.yaml up - ``` +To run the project using the provided Docker Compose files simply navigate to the files in the VS Code Explorer and select `Compose Up` after right-clicking the file. - **CI/CD Execution**: - The `docker-compose_cicd.yaml` is intended to be used within CI/CD pipelines and may be invoked as part of automated scripts. + The `docker-compose.cicd.yaml` is intended to be used within CI/CD pipelines and may be invoked as part of automated scripts. ## Notes - Ensure that you have NVIDIA GPU support configured if running models that require GPU acceleration. - The `agent_service.yaml` and other base service files are crucial for defining the common configurations and should not be modified unless necessary. - When running in distributed mode, update the `CARLA_SIM_HOST` environment variable in the appropriate service configurations to point to the simulator's IP address. -- The linter services defined in `linter_services.yaml` can be used to maintain code quality and should be run regularly during development. +- The linter services defined in `docker-compose.linter.yaml` can be used to maintain code quality and should be run regularly during development. ## Conclusion From daa6369c299429b68eb529a991717ae56a7d0216 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Wed, 9 Oct 2024 10:18:30 +0200 Subject: [PATCH 4/4] Fixed names of compose files --- .github/workflows/build.yml | 2 +- doc/02_development/14_distributed_simulation.md | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7e00e1b..fa07e68f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: if: github.event_name == 'pull_request' env: AGENT_VERSION: ${{ needs.build-and-push-image.outputs.version }} - COMPOSE_FILE: ./build/docker-compose_cicd.yaml + COMPOSE_FILE: ./build/docker-compose.cicd.yaml steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/doc/02_development/14_distributed_simulation.md b/doc/02_development/14_distributed_simulation.md index 80dc7692..2d7cbe58 100644 --- a/doc/02_development/14_distributed_simulation.md +++ b/doc/02_development/14_distributed_simulation.md @@ -45,15 +45,12 @@ Typically, the ip address is the first one in the list. Replace the ip-address in the following files: -- `docker-compose.distributed.yml` -- `docker-compose.dev.distributed.yml` +- `build/docker-compose.devroute-distributed.yaml` +- `build/docker-compose.leaderboard-distributed.yaml` ### Start the agent on your local machine -```bash -docker compose -f build/docker-compose_distributed.yaml up -docker compose -f build/docker-compose_dev_distributed.yaml up -``` +Navigate to the files mentioned above in the VS Code Explorer and select `Compose Up` after right-clicking one of the files. ## How do you know that you do not have enough compute resources?