From 92e2f9f3c362f1805c700501bce76bf3e23bdef3 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Thu, 24 Oct 2024 16:10:36 +0200 Subject: [PATCH 1/5] Added offscreen cicd & local cache --- .github/workflows/build.yml | 11 ++++++----- .github/workflows/drive.yml | 14 ++++++++++++-- .vscode/settings.json | 2 +- build/docker-compose.carla-simulator.yaml | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8221589..50d1743b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,8 +50,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local + cache-to: type=local,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -67,8 +67,8 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local + cache-to: type=local,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -85,4 +85,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: pr_id - path: pr/ \ No newline at end of file + path: pr/ + retention-days: 1 \ No newline at end of file diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 5aa0733c..a2320227 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -32,6 +32,10 @@ jobs: let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { return artifact.name == "pr_id" })[0]; + if (!matchArtifact) { + core.setFailed('No pr_id artifact found from the build workflow'); + return; + } let download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, @@ -42,12 +46,18 @@ jobs: fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_id.zip`, Buffer.from(download.data)); - name: 'Unzip artifact (PR ID)' - run: unzip pr_id.zip + run: | + unzip pr_id.zip + let issue_number = fs.readFileSync('./pr_id'); + if (!issue_number || isNaN(Number(issue_number))) { + core.setFailed(`Invalid PR ID: ${prIdContent}`); + return; + } - name: Run docker-compose run: | xhost +local: - USERNAME=$(whoami) USER_UID=$(id -u) USER_GID=$(id -g) docker compose up --quiet-pull --exit-code-from agent + USERNAME=$(whoami) USER_UID=$(id -u) USER_GID=$(id -g) RENDER_OFFSCREEN=-RenderOffScreen docker compose up --quiet-pull --exit-code-from agent - name: Copy results run: docker compose cp agent:/tmp/simulation_results.json . - name: Stop docker-compose diff --git a/.vscode/settings.json b/.vscode/settings.json index 14b08a01..b83bfc4e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -22,7 +22,7 @@ "docker.commands.composeUp": [ { "label": "Compose Up", - "template": "xhost +local: && USERNAME=$(whoami) USER_UID=$(id -u) USER_GID=$(id -g) ${composeCommand} ${configurationFile} up" + "template": "xhost +local: && USERNAME=$(whoami) USER_UID=$(id -u) USER_GID=$(id -g) RENDER_OFFSCREEN= ${composeCommand} ${configurationFile} up" } ], "workbench.iconTheme": "vscode-icons", diff --git a/build/docker-compose.carla-simulator.yaml b/build/docker-compose.carla-simulator.yaml index 9106806f..2fe43d31 100644 --- a/build/docker-compose.carla-simulator.yaml +++ b/build/docker-compose.carla-simulator.yaml @@ -1,7 +1,7 @@ services: # based on https://github.com/ll7/paf21-1/blob/master/scenarios/docker-carla-sim-compose.yml carla-simulator: - command: /bin/bash CarlaUE4.sh -quality-level=Epic -world-port=2000 -resx=800 -resy=600 -nosound -carla-settings="/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini" + command: /bin/bash CarlaUE4.sh -quality-level=Epic -world-port=2000 -resx=800 -resy=600 -nosound -carla-settings="/home/carla/CarlaUE4/Config/CustomCarlaSettings.ini" ${RENDER_OFFSCREEN} image: ghcr.io/una-auxme/paf23:leaderboard-2.0 init: true deploy: From f37d3b06df57f1ef0d60dd957dd6a552e196d7eb Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Thu, 24 Oct 2024 16:33:47 +0200 Subject: [PATCH 2/5] Fixed cache dir & PR ID check --- .github/workflows/build.yml | 11 +++++++---- .github/workflows/drive.yml | 15 ++++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50d1743b..30a6ff3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Create cache directory + run: mkdir /cache + - name: Test build if: github.event_name == 'pull_request' uses: docker/build-push-action@v3 @@ -50,8 +53,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - cache-from: type=local - cache-to: type=local,mode=max + cache-from: type=local,dest=/cache/ + cache-to: type=local,src=/cache/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -67,8 +70,8 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=local - cache-to: type=local,mode=max + cache-from: type=local,dest=/cache/ + cache-to: type=local,src=/cache/,mode=max build-args: | USERNAME=paf USER_UID=1000 diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index a2320227..c8a23b83 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -48,11 +48,16 @@ jobs: - name: 'Unzip artifact (PR ID)' run: | unzip pr_id.zip - let issue_number = fs.readFileSync('./pr_id'); - if (!issue_number || isNaN(Number(issue_number))) { - core.setFailed(`Invalid PR ID: ${prIdContent}`); - return; - } + + - name: Check PR ID + uses: actions/github-script@v6 + with: + script: | + let issue_number = fs.readFileSync('./pr_id'); + if (!issue_number || isNaN(Number(issue_number))) { + core.setFailed(`Invalid PR ID: ${prIdContent}`); + return; + } - name: Run docker-compose run: | From 8f91d1a1c9fda9f29260478b2be4dcd53d1f4fd8 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Fri, 25 Oct 2024 10:05:35 +0200 Subject: [PATCH 3/5] Fixed cache filling & io --- .github/workflows/build.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30a6ff3c..088e1c7b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Create cache directory - run: mkdir /cache + run: | + mkdir -p ./.buildx-cache/cache + mkdir ./.buildx-cache/cache-new + mkdir ./.buildx-cache/cache/test + mkdir ./.buildx-cache/cache/latest + mkdir ./.buildx-cache/cache-new/test + mkdir ./.buildx-cache/cache-new/latest - name: Test build if: github.event_name == 'pull_request' @@ -53,8 +59,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - cache-from: type=local,dest=/cache/ - cache-to: type=local,src=/cache/,mode=max + cache-from: type=local,src=./.buildx-cache/cache/test/ + cache-to: type=local,dst=./.buildx-cache/cache-new/test/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -70,8 +76,8 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=local,dest=/cache/ - cache-to: type=local,src=/cache/,mode=max + cache-from: type=local,src=./.buildx-cache/cache/latest/ + cache-to: type=local,dst=./.buildx-cache/cache-new/latest/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -89,4 +95,9 @@ jobs: with: name: pr_id path: pr/ - retention-days: 1 \ No newline at end of file + retention-days: 1 + + - name: Clean up cache + run: | + rm -rf ./.buildx-cache + mv ./.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file From f26a56f61a6aa774b6a41d8fbb05cf24558278fc Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Fri, 25 Oct 2024 10:09:48 +0200 Subject: [PATCH 4/5] Fixed typo --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 088e1c7b..c9c1c141 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,7 +60,7 @@ jobs: load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test cache-from: type=local,src=./.buildx-cache/cache/test/ - cache-to: type=local,dst=./.buildx-cache/cache-new/test/,mode=max + cache-to: type=local,dest=./.buildx-cache/cache-new/test/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -77,7 +77,7 @@ jobs: # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest cache-from: type=local,src=./.buildx-cache/cache/latest/ - cache-to: type=local,dst=./.buildx-cache/cache-new/latest/,mode=max + cache-to: type=local,dest=./.buildx-cache/cache-new/latest/,mode=max build-args: | USERNAME=paf USER_UID=1000 From bcbb6e2ec0d2eed4c9da34041d9336d6d22c2e43 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Fri, 25 Oct 2024 10:13:19 +0200 Subject: [PATCH 5/5] Fixed dirs in clean up --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9c1c141..5def7a0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,5 +99,6 @@ jobs: - name: Clean up cache run: | - rm -rf ./.buildx-cache - mv ./.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file + rm -rf ./.buildx-cache/cache/test + rm -rf ./.buildx-cache/cache/latest + mv ./.buildx-cache/cache-new ./.buildx-cache/cache \ No newline at end of file