From cc897dfc7ccd87d4fb97bd26db6d0fcd23da592e Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Wed, 30 Oct 2024 15:45:11 +0100 Subject: [PATCH 01/18] Changed printing of simulation results --- .github/workflows/build.yml | 25 +++++++++++++------ .github/workflows/drive.yml | 49 +++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 26a0e894..a5b73cee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,20 +83,31 @@ jobs: USER_UID=1000 USER_GID=1000 - - name: Save PR ID + - name: Save merge artifact + if: github.event_name != 'pull_request' + run: | + mkdir -p ./artifact + printf '{ + "is_pr": false, + "pr_id": -1 + }' >> ./artifact/artifact.json + + - name: Save pull request artifact if: github.event_name == 'pull_request' env: PR_ID: ${{ github.event.number }} run: | - mkdir -p ./pr - echo $PR_ID > ./pr/pr_id + mkdir -p ./artifact + printf '{ + "is_pr": true, + "pr_id": $PR_ID + }' >> ./artifact/artifact.json - - name: Upload PR ID - if: github.event_name == 'pull_request' + - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: pr_id - path: pr/ + name: artifact.json + path: ./artifact/ retention-days: 1 - name: Clean up cache diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 933fb9a9..2b60928a 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -20,8 +20,7 @@ jobs: run: | echo "AGENT_VERSION=${AGENT_VERSION}" echo "COMPOSE_FILE=${COMPOSE_FILE}" - - name: 'Download artifact (PR ID)' - if: github.event_name == 'pull_request' + - name: Download artifact uses: actions/github-script@v6 with: script: | @@ -31,10 +30,10 @@ jobs: run_id: context.payload.workflow_run.id, }); let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_id" + return artifact.name == "artifact.json" })[0]; if (!matchArtifact) { - core.setFailed('No pr_id artifact found from the build workflow'); + core.setFailed('No artifact found from the build workflow'); return; } let download = await github.rest.actions.downloadArtifact({ @@ -44,23 +43,20 @@ jobs: archive_format: 'zip', }); let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_id.zip`, Buffer.from(download.data)); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/artifact.zip`, Buffer.from(download.data)); - - name: 'Unzip artifact (PR ID)' - if: github.event_name == 'pull_request' + - name: Unzip artifact run: | - unzip pr_id.zip + unzip artifact.zip - - name: Check PR ID - if: github.event_name == 'pull_request' + - name: Return artifact JSON + id: return-artifact-json uses: actions/github-script@v6 with: script: | - let issue_number = fs.readFileSync('./pr_id'); - if (!issue_number || isNaN(Number(issue_number))) { - core.setFailed(`Invalid PR ID: ${prIdContent}`); - return; - } + let fs = require('fs'); + let data = JSON.parse(fs.readFileSync('./artifacts/artifacts.json')); + return data; - name: Run docker-compose run: | @@ -73,12 +69,10 @@ jobs: if: always() run: docker compose down -v # add rendered JSON as comment to the pull request - - name: Add simulation results as comment - if: github.event_name == 'pull_request' + - name: Create simulation results table + id: simulation-results uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - # this script reads the simulation_results.json and creates a comment on the pull request with the results. script: | const fs = require('fs'); // read the simulation results @@ -94,13 +88,26 @@ jobs: let resultsTableDivider = `| --- | --- |`; // add everything to the resultsTable resultsTable = resultsTableHeader + '\n' + resultsTableDivider + '\n' + resultsTable.join('\n'); + return resultsTable; + result-encoding: string + - name: Print simulation results + run: | + echo "Simulation results\n" + echo "${{ steps.simulation-results.outputs.result }}" + - name: Add simulation results as comment + if: ${{ steps.return-artifact-json.outputs.result.is_pr }} + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + # this script reads the simulation_results.json and creates a comment on the pull request with the results. + script: | + let issue_number = Number(${{ steps.return-artifact-json.outputs.pr_id }}); // add the results as a comment to the pull request - let issue_number = Number(fs.readFileSync('./pr_id')); github.rest.issues.createComment({ issue_number: issue_number, owner: context.repo.owner, repo: context.repo.repo, - body: "## Simulation results\n" + resultsTable + body: "## Simulation results\n" + ${{ steps.simulation-results.outputs.result }} }); - name: Prune all images older than 30 days from self-hosted runner run: docker image prune -a --force --filter "until=720h" \ No newline at end of file From 749860f2214f9733d02baed8644cda64628225fa Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 1 Nov 2024 11:07:26 +0100 Subject: [PATCH 02/18] Update README.md structure and fix broken link --- doc/README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/README.md b/doc/README.md index 7546ccb8..197a48bd 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,16 +2,15 @@ This document provides an overview of the structure of the documentation. -- [PAF Documentation](#paf-documentation) - - [`general`](#general) - - [`development`](#development) - - [`research`](#research) - - [`examples`](#examples) - - [`perception`](#perception) - - [`planning`](#planning) - - [`acting`](#acting) - - [`assets`](#assets) - - [`dev_talks`](#dev_talks) +- [`general`](#general) +- [`development`](#development) +- [`research`](#research) +- [`examples`](#examples) +- [`perception`](#perception) +- [`planning`](#planning) +- [`acting`](#acting) +- [`assets`](#assets) +- [`dev_talks`](#dev_talks) ## `general` @@ -47,4 +46,4 @@ The [`assets`](./assets/) folder contains mainly images that are used inside the ## `dev_talks` -The [`dev_talks`](./dev_talks/) folder contains the protocols of each sprint review and roles that the students fill during the project. +The [`dev_talks`](./dev_talks/README.md) folder contains the protocols of each sprint review and roles that the students fill during the project. From c7baf35ce32969fbbd4a0b6964681fb65b922fce Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 1 Nov 2024 11:07:41 +0100 Subject: [PATCH 03/18] Add Dev Talks README.md --- doc/dev_talks/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/dev_talks/README.md diff --git a/doc/dev_talks/README.md b/doc/dev_talks/README.md new file mode 100644 index 00000000..f902de55 --- /dev/null +++ b/doc/dev_talks/README.md @@ -0,0 +1,5 @@ +# Dev Talks + +- [PAF24](./paf24/README.md) +- [PAF23](./paf23/) + From 4d70e6cb41daeab393493830773edb7842f536dd Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 1 Nov 2024 11:08:28 +0100 Subject: [PATCH 04/18] Add PAF24 README.md --- doc/dev_talks/paf24/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/dev_talks/paf24/README.md diff --git a/doc/dev_talks/paf24/README.md b/doc/dev_talks/paf24/README.md new file mode 100644 index 00000000..04311f31 --- /dev/null +++ b/doc/dev_talks/paf24/README.md @@ -0,0 +1,16 @@ +# PAF24 + +**Summary**: The PAF24 folder contains documents that are relevant for the PAF24 group. + +- [General Information](#general-information) +- [Notes for Sprints](#notes-for-sprints) + +## General Information + +- [Mermaid Gantt Chart](./mermaid_paf24.md) +- [Student Roles](./student_roles.md) +- [Joker Rules] + +## Notes for Sprints + +- [T03](./T03_2024-10-28.md) From 5dc5aa35209585c16da870fb21426bdc3c4513fa Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 1 Nov 2024 11:08:56 +0100 Subject: [PATCH 05/18] Add PAF24 Joker Rules documentation Fixes Add Joker Rules for PAF24 #391 --- doc/dev_talks/paf24/joker_rules_paf24.md | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 doc/dev_talks/paf24/joker_rules_paf24.md diff --git a/doc/dev_talks/paf24/joker_rules_paf24.md b/doc/dev_talks/paf24/joker_rules_paf24.md new file mode 100644 index 00000000..50001e2f --- /dev/null +++ b/doc/dev_talks/paf24/joker_rules_paf24.md @@ -0,0 +1,60 @@ +# Joker Rules PAF24 + +**Purpose**: This document outlines the rules for using the Joker card in the PAF24 project. + +- [1. Overview](#1-overview) +- [2. Using the Joker Card](#2-using-the-joker-card) +- [3. Special Circumstances](#3-special-circumstances) + - [3.1. Unable to Present Work Due to Scheduling Conflicts](#31-unable-to-present-work-due-to-scheduling-conflicts) + - [3.2. Illness or Emergency](#32-illness-or-emergency) + +## 1. Overview + +- **Project Duration**: October 2024 – March 2025 +- The Joker card allows you to be excused from duties during specific periods, especially during the exam phase in February 2025. +- **Total Joker Allocation**: + - **Joker Card** Up to 4 weeks throughout the project. + - **Presentation Only Joker Card** Maximum of 2 uses. +- **When to Use**: The Joker card should be played if you are unable to present your work during a sprint review or if the amount of work completed is insufficient. +- The intention of the Joker card is to ensure you have the appropriate time to prepare your exam phase. + +## 2. Using the Joker Card + +To play the Joker card: + +1. **Notify the Team and Chair**: Inform both the team and chair staff by posting in the `#joker` channel on our Discord server. +2. **Deadline**: Notify the team at least **7 days** before the sprint review. +3. **Example Post**: + - “I would like to play the Joker card for the sprint review on 2024-11-25.” +4. Your name and intent will be confirmed by the team and chair. + +> [!NOTE] 2 week sprints require two Joker cards to cover the entire sprint. + +## 3. Special Circumstances + +### 3.1. Unable to Present Work Due to Scheduling Conflicts + +- If you completed the sprint’s work but cannot present it, you have two options: + 1. Play the standard Joker card. + 2. Use a `Presentation Only Joker Card` by arranging for a teammate to present on your behalf. + +- **Presentation Only Joker Card Details**: + - **Maximum Use**: Limited to two times throughout the project. + - **Requirements**: + - Notify the team and chair at least **7 days** before the sprint review. + - Submit a written summary of your work in the `#joker` Discord channel at least **1 day** before the review. + - Find a teammate who presents your work. + +- **Written Summary Contents**: + - Overview of completed work. + - Key achievements. + - Lessons learned. + - Goals for the next sprint. + +> [!CAUTION] Presentation Only Joker Card +> We encourage all team members to only present your teammates work if absolutely necessary. The work completed by your colleagues is their responsibility, and they should be the ones to present it. Furthermore, you should only present their work if you are familiar with it and can answer questions about it. Additionally, you should be confident that your team member has completed the work to a satisfactory standard. + +### 3.2. Illness or Emergency + +- **Illness**: If you cannot attend due to illness, provide a medical certificate to be excused from duties without using a Joker card. +- **Emergency**: Emergencies are evaluated individually. From 373041a6f18d307bc179f97b666485ab0459dd7c Mon Sep 17 00:00:00 2001 From: ll7 Date: Fri, 1 Nov 2024 11:11:26 +0100 Subject: [PATCH 06/18] Fix linting --- doc/dev_talks/README.md | 1 - doc/dev_talks/paf24/joker_rules_paf24.md | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/dev_talks/README.md b/doc/dev_talks/README.md index f902de55..db9a4f8a 100644 --- a/doc/dev_talks/README.md +++ b/doc/dev_talks/README.md @@ -2,4 +2,3 @@ - [PAF24](./paf24/README.md) - [PAF23](./paf23/) - diff --git a/doc/dev_talks/paf24/joker_rules_paf24.md b/doc/dev_talks/paf24/joker_rules_paf24.md index 50001e2f..be5a3392 100644 --- a/doc/dev_talks/paf24/joker_rules_paf24.md +++ b/doc/dev_talks/paf24/joker_rules_paf24.md @@ -24,7 +24,7 @@ To play the Joker card: 1. **Notify the Team and Chair**: Inform both the team and chair staff by posting in the `#joker` channel on our Discord server. 2. **Deadline**: Notify the team at least **7 days** before the sprint review. -3. **Example Post**: +3. **Example Post**: - “I would like to play the Joker card for the sprint review on 2024-11-25.” 4. Your name and intent will be confirmed by the team and chair. @@ -52,7 +52,10 @@ To play the Joker card: - Goals for the next sprint. > [!CAUTION] Presentation Only Joker Card -> We encourage all team members to only present your teammates work if absolutely necessary. The work completed by your colleagues is their responsibility, and they should be the ones to present it. Furthermore, you should only present their work if you are familiar with it and can answer questions about it. Additionally, you should be confident that your team member has completed the work to a satisfactory standard. +> We encourage all team members to only present your teammates work if absolutely necessary. +> The work completed by your colleagues is their responsibility, and they should be the ones to present it. +> Furthermore, you should only present their work if you are familiar with it and can answer questions about it. +> Additionally, you should be confident that your team member has completed the work to a satisfactory standard. ### 3.2. Illness or Emergency From 0a044cefbe5a768afbb9f66eba3db016be721764 Mon Sep 17 00:00:00 2001 From: ll7 <32880741+ll7@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:20:45 +0100 Subject: [PATCH 07/18] Update doc/dev_talks/paf24/joker_rules_paf24.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- doc/dev_talks/paf24/joker_rules_paf24.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/dev_talks/paf24/joker_rules_paf24.md b/doc/dev_talks/paf24/joker_rules_paf24.md index be5a3392..b325a71a 100644 --- a/doc/dev_talks/paf24/joker_rules_paf24.md +++ b/doc/dev_talks/paf24/joker_rules_paf24.md @@ -28,7 +28,7 @@ To play the Joker card: - “I would like to play the Joker card for the sprint review on 2024-11-25.” 4. Your name and intent will be confirmed by the team and chair. -> [!NOTE] 2 week sprints require two Joker cards to cover the entire sprint. +> [!NOTE] 2-week sprints require two Joker cards to cover the entire sprint. ## 3. Special Circumstances From d4812730d19e7938b328a25dbd5a7182f0f377da Mon Sep 17 00:00:00 2001 From: fedor98 Date: Mon, 4 Nov 2024 02:19:21 +0100 Subject: [PATCH 08/18] Add doc/research/paf24/planning/decision_making_rationale.md --- .../planning/decision_making_rationale.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 doc/research/paf24/planning/decision_making_rationale.md diff --git a/doc/research/paf24/planning/decision_making_rationale.md b/doc/research/paf24/planning/decision_making_rationale.md new file mode 100644 index 00000000..7557b1f3 --- /dev/null +++ b/doc/research/paf24/planning/decision_making_rationale.md @@ -0,0 +1,70 @@ +# Decision Makaing Approaches + +Responsible for making decisions based on information provided by other system components. It should account for all possible traffic scenarios to ensure comprehensive coverage and safe driving behavior + +## Decision Tree - current implementations + +- a tool to organizes decisions and their possible outcomes in a tree-like structure +- a branch represents a choice/condition which may lead to further options. + - starting at the root: checking surrounding conditions (traffic lights, obstacles, speed limits etc) branching out based on yes/no questions at each node +- in our case each node is deterministic (as no ML is used) + +### Pros and Cons + +
    +
  • + are easy to understand and interpret
    • +
  • + decision paths are tracable → suitable for debugging
    • +
  • + deterministic outputs beneficial when predictability is essential
    • +
  • + compared to NN low computational cost
    • +
  • - Adding too many branches for every possible situation can make the tree unwieldy and hard to manage
    • +
      +
    • can easily overfit with too many conditions
      • +
    +
  • - no reasoning and no probabilistic capabilities, trees tend to make not optimal decisions in new unclear situations
    • +
      +
    • hard to account for every situation
      • +
    +
+ +### Changes in Previous Semesters + +- removing redundant behaviours + + - reducing to `Intersection`, `Lane Switch` and `Cruise` + +![decision_tree](doc/assets/behaviour_tree.png) + +## Finite State Machine - previous implementations + +- model with an discrete number of possible states with rule-based control + - can only inherit one state at a time +- each state represents a specific driving mode, such as "Lane Switching" "Stopping," or "Overtaking" + - triggered by inputs like traffic signals, obstacles, or speed limits + +### Pros and Cons + +
    +
  • + each state and transition is explicitly defined → debugging and testing
    • +
  • + easier implementations than a decision tree
    • +
  • - scalability issues with increasing amount of states
    • +
  • - less flexible than decision trees
    • +
+ +## Combining Planning and Reinforcement Learning + +[C.-J. Hoel, K. Driggs-Campbell, K. Wolff, L. Laine, and M. J. Kochenderfer, "Combining Planning and Deep Reinforcement Learning in Tactical Decision Making for Autonomous Driving," arXiv preprint arXiv:1905.02680, 2019.](https://arxiv.org/abs/1905.02680) + +- Monte Carlo Tree Search (MCTS): Used for planning by creating a search tree that simulates possible actions, selecting those that maximize expected rewards (maintaining desired speed, reaching end points, avoinding collision etc.) + - considers all possible actions → Each action creates a new branch that represents what the new situation might look like if the agent took that action +- deep neural network guides the MCTS by predicting action probabilities (which actions are most likely to lead to good outcomes: changing lanes, speeding up, breaking etc.) and state values (scores that tell the agent whether its current situation is favorable or not based on expected future rewards) + +### Pros and Cons + +
    +
  • + generalizes better to complex scenarios
    • +
      +
    • takes advantage of continous and high-dimensional state spaces
      • +
    +
  • - computational complexity and real-time feasibility
    • +
  • - extensive training data and tuning to handle a broad range of driving situations
    • +
From a32d6c62c642ec29b4ab7280b28ba4247330732d Mon Sep 17 00:00:00 2001 From: fedor98 Date: Mon, 4 Nov 2024 10:02:57 +0100 Subject: [PATCH 09/18] Update: resolve linting errors due to use of HTML --- .../planning/decision_making_rationale.md | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/doc/research/paf24/planning/decision_making_rationale.md b/doc/research/paf24/planning/decision_making_rationale.md index 7557b1f3..b051b1d0 100644 --- a/doc/research/paf24/planning/decision_making_rationale.md +++ b/doc/research/paf24/planning/decision_making_rationale.md @@ -11,20 +11,14 @@ Responsible for making decisions based on information provided by other system c ### Pros and Cons -
    -
  • + are easy to understand and interpret
    • -
  • + decision paths are tracable → suitable for debugging
    • -
  • + deterministic outputs beneficial when predictability is essential
    • -
  • + compared to NN low computational cost
    • -
  • - Adding too many branches for every possible situation can make the tree unwieldy and hard to manage
    • -
      -
    • can easily overfit with too many conditions
      • -
    -
  • - no reasoning and no probabilistic capabilities, trees tend to make not optimal decisions in new unclear situations
    • -
      -
    • hard to account for every situation
      • -
    -
+- `+` are easy to understand and interpret +- `+` decision paths are traceable → suitable for debugging +- `+` deterministic outputs beneficial when predictability is essential +- `+` compared to NN, low computational cost +- `-` Adding too many branches for every possible situation can make the tree unwieldy and hard to manage + - can easily overfit with too many conditions +- `-` no reasoning and no probabilistic capabilities; trees tend to make suboptimal decisions in new, unclear situations + - hard to account for every situation ### Changes in Previous Semesters @@ -32,7 +26,7 @@ Responsible for making decisions based on information provided by other system c - reducing to `Intersection`, `Lane Switch` and `Cruise` -![decision_tree](doc/assets/behaviour_tree.png) +![decision_tree](/doc/assets/behaviour_tree.png) ## Finite State Machine - previous implementations @@ -43,12 +37,10 @@ Responsible for making decisions based on information provided by other system c ### Pros and Cons -
    -
  • + each state and transition is explicitly defined → debugging and testing
    • -
  • + easier implementations than a decision tree
    • -
  • - scalability issues with increasing amount of states
    • -
  • - less flexible than decision trees
    • -
+- `+` each state and transition is explicitly defined → suitable for debugging and testing +- `+` easier implementations than a decision tree +- `-` scalability issues with increasing number of states +- `-` less flexible than decision trees ## Combining Planning and Reinforcement Learning @@ -60,11 +52,7 @@ Responsible for making decisions based on information provided by other system c ### Pros and Cons -
    -
  • + generalizes better to complex scenarios
    • -
      -
    • takes advantage of continous and high-dimensional state spaces
      • -
    -
  • - computational complexity and real-time feasibility
    • -
  • - extensive training data and tuning to handle a broad range of driving situations
    • -
+- `+` generalizes better to complex scenarios + - takes advantage of continuous and high-dimensional state spaces +- `-` computational complexity and real-time feasibility +- `-` extensive training data and tuning required to handle a broad range of driving situations From 4a1ba66b6535ac9377aa726b7f968593634d9645 Mon Sep 17 00:00:00 2001 From: fedor98 Date: Mon, 4 Nov 2024 10:30:17 +0100 Subject: [PATCH 10/18] Update: correcting typos --- doc/research/paf24/planning/decision_making_rationale.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/research/paf24/planning/decision_making_rationale.md b/doc/research/paf24/planning/decision_making_rationale.md index b051b1d0..a5f3976c 100644 --- a/doc/research/paf24/planning/decision_making_rationale.md +++ b/doc/research/paf24/planning/decision_making_rationale.md @@ -1,4 +1,4 @@ -# Decision Makaing Approaches +# Decision Making Approaches Responsible for making decisions based on information provided by other system components. It should account for all possible traffic scenarios to ensure comprehensive coverage and safe driving behavior From 6169fef9cab86289a4da32725d6e536c544483b3 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Mon, 4 Nov 2024 13:11:29 +0100 Subject: [PATCH 11/18] Fixed caching on self-hosted runner --- .github/workflows/build.yml | 64 ++++++++++++++++++++++--------------- .github/workflows/drive.yml | 4 +-- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5b73cee..c290776d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,6 +35,18 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: | + /tmp/.buildx-cache/cache/test + /tmp/.buildx-cache/cache/latest + /tmp/.buildx-cache/cache-new/test + /tmp/.buildx-cache/cache-new/latest + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Log in to the Container registry uses: docker/login-action@v2 with: @@ -42,15 +54,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Create cache directory - run: | - mkdir -p ./.buildx-cache/cache - mkdir ./.buildx-cache/cache-new - mkdir ./.buildx-cache/cache/test - mkdir ./.buildx-cache/cache/latest - mkdir ./.buildx-cache/cache-new/test - mkdir ./.buildx-cache/cache-new/latest - - name: Test build if: github.event_name == 'pull_request' uses: docker/build-push-action@v3 @@ -59,8 +62,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - cache-from: type=local,src=./.buildx-cache/cache/test/ - cache-to: type=local,dest=./.buildx-cache/cache-new/test/,mode=max + cache-from: type=local,src=/tmp/.buildx-cache/cache/test/ + cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/test/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -76,22 +79,13 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=local,src=./.buildx-cache/cache/latest/ - cache-to: type=local,dest=./.buildx-cache/cache-new/latest/,mode=max + cache-from: type=local,src=/tmp/.buildx-cache/cache/latest/ + cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/latest/,mode=max build-args: | USERNAME=paf USER_UID=1000 USER_GID=1000 - - name: Save merge artifact - if: github.event_name != 'pull_request' - run: | - mkdir -p ./artifact - printf '{ - "is_pr": false, - "pr_id": -1 - }' >> ./artifact/artifact.json - - name: Save pull request artifact if: github.event_name == 'pull_request' env: @@ -103,6 +97,15 @@ jobs: "pr_id": $PR_ID }' >> ./artifact/artifact.json + - name: Save merge artifact + if: github.event_name != 'pull_request' + run: | + mkdir -p ./artifact + printf '{ + "is_pr": false, + "pr_id": -1 + }' >> ./artifact/artifact.json + - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -110,8 +113,17 @@ jobs: path: ./artifact/ retention-days: 1 - - name: Clean up cache + - name: Clean up PR cache + if: github.event_name == 'pull_request' + run: | + rm -rf /tmp/.buildx-cache/cache/test + mv /tmp/.buildx-cache/cache-new/test /tmp/.buildx-cache/cache/test + + - name: Clean up merge cache + if: github.event_name != 'pull_request' run: | - rm -rf ./.buildx-cache/cache/test - rm -rf ./.buildx-cache/cache/latest - mv ./.buildx-cache/cache-new ./.buildx-cache/cache \ No newline at end of file + rm -rf /tmp/.buildx-cache/cache/latest + mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest + + - name: Prune all images older than 1 days from self-hosted runner + run: docker image prune -a --filter "until=24h" \ No newline at end of file diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 2b60928a..fafdec2b 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -109,5 +109,5 @@ jobs: repo: context.repo.repo, body: "## Simulation results\n" + ${{ steps.simulation-results.outputs.result }} }); - - name: Prune all images older than 30 days from self-hosted runner - run: docker image prune -a --force --filter "until=720h" \ No newline at end of file + - name: Prune all images older than 1 days from self-hosted runner + run: docker image prune -a --filter "until=24h" \ No newline at end of file From 50e75515bba68015ccc4ce800d22c067a05b12da Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 09:42:44 +0100 Subject: [PATCH 12/18] Added restrictions for runner selection --- .github/workflows/build.yml | 4 ++-- .github/workflows/drive.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c290776d..e8efcb1e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ env: jobs: build-and-push-image: name: Build and push image - runs-on: self-hosted + runs-on: [self-hosted, build] outputs: version: ${{ steps.build.version.outputs.version }} permissions: @@ -126,4 +126,4 @@ jobs: mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest - name: Prune all images older than 1 days from self-hosted runner - run: docker image prune -a --filter "until=24h" \ No newline at end of file + run: docker image prune -a -f --filter "until=24h" \ No newline at end of file diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index fafdec2b..8b53cd59 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -8,7 +8,7 @@ on: jobs: drive: - runs-on: self-hosted + runs-on: [self-hosted, drive] env: AGENT_VERSION: latest COMPOSE_FILE: ./build/docker-compose.cicd.yaml From a7c130d410eb4b2919fed3a7ba26aca56e90642c Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 12:36:53 +0100 Subject: [PATCH 13/18] Fixed path for artifact --- .github/workflows/drive.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 8b53cd59..f17806fb 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -20,6 +20,7 @@ jobs: run: | echo "AGENT_VERSION=${AGENT_VERSION}" echo "COMPOSE_FILE=${COMPOSE_FILE}" + echo "GITHUB_WORKSPACE=${process.env.GITHUB_WORKSPACE}" - name: Download artifact uses: actions/github-script@v6 with: @@ -43,11 +44,11 @@ jobs: archive_format: 'zip', }); let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/artifact.zip`, Buffer.from(download.data)); + fs.writeFileSync('${process.env.GITHUB_WORKSPACE}/artifact.zip', Buffer.from(download.data)); - name: Unzip artifact run: | - unzip artifact.zip + unzip ${process.env.GITHUB_WORKSPACE}/artifact.zip - name: Return artifact JSON id: return-artifact-json @@ -55,7 +56,7 @@ jobs: with: script: | let fs = require('fs'); - let data = JSON.parse(fs.readFileSync('./artifacts/artifacts.json')); + let data = JSON.parse(fs.readFileSync('${process.env.GITHUB_WORKSPACE}/artifacts.json')); return data; - name: Run docker-compose From 19853de41b8640d5c094813fb6fa196b12ea1a9b Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 12:45:12 +0100 Subject: [PATCH 14/18] Removed GITHUB_WORKSPACE from run tasks --- .github/workflows/drive.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index f17806fb..4610e7ee 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -20,7 +20,6 @@ jobs: run: | echo "AGENT_VERSION=${AGENT_VERSION}" echo "COMPOSE_FILE=${COMPOSE_FILE}" - echo "GITHUB_WORKSPACE=${process.env.GITHUB_WORKSPACE}" - name: Download artifact uses: actions/github-script@v6 with: @@ -48,7 +47,7 @@ jobs: - name: Unzip artifact run: | - unzip ${process.env.GITHUB_WORKSPACE}/artifact.zip + unzip artifact.zip - name: Return artifact JSON id: return-artifact-json From e6d172c50928a09d43c50237e9f4968a2bf794b4 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 12:55:46 +0100 Subject: [PATCH 15/18] Removed GITHUB_WORKSPACE from drive action --- .github/workflows/drive.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 4610e7ee..788716b3 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -43,7 +43,7 @@ jobs: archive_format: 'zip', }); let fs = require('fs'); - fs.writeFileSync('${process.env.GITHUB_WORKSPACE}/artifact.zip', Buffer.from(download.data)); + fs.writeFileSync('artifact.zip', Buffer.from(download.data)); - name: Unzip artifact run: | @@ -55,7 +55,7 @@ jobs: with: script: | let fs = require('fs'); - let data = JSON.parse(fs.readFileSync('${process.env.GITHUB_WORKSPACE}/artifacts.json')); + let data = JSON.parse(fs.readFileSync('artifacts.json')); return data; - name: Run docker-compose From e2fb5905e5657150248132089665bb4677c97c48 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 13:13:54 +0100 Subject: [PATCH 16/18] Removed caching & reworked artifact --- .github/workflows/build.yml | 54 ++++++++++++++++++------------------- .github/workflows/drive.yml | 31 +++------------------ 2 files changed, 30 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e8efcb1e..9d5c6d20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,17 +35,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: | - /tmp/.buildx-cache/cache/test - /tmp/.buildx-cache/cache/latest - /tmp/.buildx-cache/cache-new/test - /tmp/.buildx-cache/cache-new/latest - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + # - name: Cache Docker layers + # uses: actions/cache@v4 + # with: + # path: | + # /tmp/.buildx-cache/cache/test + # /tmp/.buildx-cache/cache/latest + # /tmp/.buildx-cache/cache-new/test + # /tmp/.buildx-cache/cache-new/latest + # key: ${{ runner.os }}-buildx-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-buildx- - name: Log in to the Container registry uses: docker/login-action@v2 @@ -62,8 +62,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - cache-from: type=local,src=/tmp/.buildx-cache/cache/test/ - cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/test/,mode=max + # cache-from: type=local,src=/tmp/.buildx-cache/cache/test/ + # cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/test/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -79,8 +79,8 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=local,src=/tmp/.buildx-cache/cache/latest/ - cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/latest/,mode=max + # cache-from: type=local,src=/tmp/.buildx-cache/cache/latest/ + # cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/latest/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -109,21 +109,21 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: artifact.json - path: ./artifact/ + name: artifact + path: ./artifact/.artifact.json retention-days: 1 - - name: Clean up PR cache - if: github.event_name == 'pull_request' - run: | - rm -rf /tmp/.buildx-cache/cache/test - mv /tmp/.buildx-cache/cache-new/test /tmp/.buildx-cache/cache/test + # - name: Clean up PR cache + # if: github.event_name == 'pull_request' + # run: | + # rm -rf /tmp/.buildx-cache/cache/test + # mv /tmp/.buildx-cache/cache-new/test /tmp/.buildx-cache/cache/test - - name: Clean up merge cache - if: github.event_name != 'pull_request' - run: | - rm -rf /tmp/.buildx-cache/cache/latest - mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest + # - name: Clean up merge cache + # if: github.event_name != 'pull_request' + # run: | + # rm -rf /tmp/.buildx-cache/cache/latest + # mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest - name: Prune all images older than 1 days from self-hosted runner run: docker image prune -a -f --filter "until=24h" \ No newline at end of file diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml index 788716b3..3f2e0761 100644 --- a/.github/workflows/drive.yml +++ b/.github/workflows/drive.yml @@ -21,41 +21,16 @@ jobs: echo "AGENT_VERSION=${AGENT_VERSION}" echo "COMPOSE_FILE=${COMPOSE_FILE}" - name: Download artifact - uses: actions/github-script@v6 + uses: actions/download-artifact@v4 with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "artifact.json" - })[0]; - if (!matchArtifact) { - core.setFailed('No artifact found from the build workflow'); - return; - } - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync('artifact.zip', Buffer.from(download.data)); - - - name: Unzip artifact - run: | - unzip artifact.zip - + name: artifact - name: Return artifact JSON id: return-artifact-json uses: actions/github-script@v6 with: script: | let fs = require('fs'); - let data = JSON.parse(fs.readFileSync('artifacts.json')); + let data = JSON.parse(fs.readFileSync('${process.env.GITHUB_WORKSPACE}/artifacts.json')); return data; - name: Run docker-compose From fbbfc17a8fe29e51ae7dbdfad704e391c348bcb3 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 13:26:40 +0100 Subject: [PATCH 17/18] Re-added caching --- .github/workflows/build.yml | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d5c6d20..13bd0caa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,17 +35,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - # - name: Cache Docker layers - # uses: actions/cache@v4 - # with: - # path: | - # /tmp/.buildx-cache/cache/test - # /tmp/.buildx-cache/cache/latest - # /tmp/.buildx-cache/cache-new/test - # /tmp/.buildx-cache/cache-new/latest - # key: ${{ runner.os }}-buildx-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-buildx- + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: | + /tmp/.buildx-cache/cache/test + /tmp/.buildx-cache/cache/latest + /tmp/.buildx-cache/cache-new/test + /tmp/.buildx-cache/cache-new/latest + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Log in to the Container registry uses: docker/login-action@v2 @@ -62,8 +62,8 @@ jobs: file: ./build/docker/agent/Dockerfile load: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}:test - # cache-from: type=local,src=/tmp/.buildx-cache/cache/test/ - # cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/test/,mode=max + cache-from: type=local,src=/tmp/.buildx-cache/cache/test/ + cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/test/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -79,8 +79,8 @@ jobs: push: true # tag 'latest' and version on push to main, otherwise use the commit hash tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - # cache-from: type=local,src=/tmp/.buildx-cache/cache/latest/ - # cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/latest/,mode=max + cache-from: type=local,src=/tmp/.buildx-cache/cache/latest/ + cache-to: type=local,dest=/tmp/.buildx-cache/cache-new/latest/,mode=max build-args: | USERNAME=paf USER_UID=1000 @@ -113,17 +113,17 @@ jobs: path: ./artifact/.artifact.json retention-days: 1 - # - name: Clean up PR cache - # if: github.event_name == 'pull_request' - # run: | - # rm -rf /tmp/.buildx-cache/cache/test - # mv /tmp/.buildx-cache/cache-new/test /tmp/.buildx-cache/cache/test + - name: Clean up PR cache + if: github.event_name == 'pull_request' + run: | + rm -rf /tmp/.buildx-cache/cache/test + mv /tmp/.buildx-cache/cache-new/test /tmp/.buildx-cache/cache/test - # - name: Clean up merge cache - # if: github.event_name != 'pull_request' - # run: | - # rm -rf /tmp/.buildx-cache/cache/latest - # mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest + - name: Clean up merge cache + if: github.event_name != 'pull_request' + run: | + rm -rf /tmp/.buildx-cache/cache/latest + mv /tmp/.buildx-cache/cache-new/latest /tmp/.buildx-cache/cache/latest - name: Prune all images older than 1 days from self-hosted runner run: docker image prune -a -f --filter "until=24h" \ No newline at end of file From a4de11c07ea7ffeb7f10bfc4ebbdf8919f8f6d32 Mon Sep 17 00:00:00 2001 From: JulianTrommer Date: Tue, 5 Nov 2024 13:36:10 +0100 Subject: [PATCH 18/18] Fixed typo in build action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 13bd0caa..86c59e1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: artifact - path: ./artifact/.artifact.json + path: ./artifact/artifact.json retention-days: 1 - name: Clean up PR cache