From 7a99efb78abfc26426f0fdd4c73e6eca021d6324 Mon Sep 17 00:00:00 2001 From: Tomas Baca Date: Sat, 16 Sep 2023 15:54:28 +0200 Subject: [PATCH] updated scripts, added thirdparty --- .ci/build.sh | 6 +- .ci/get_build_matrix.sh | 6 +- .github/workflows/stable_mrs_amd64.yml | 115 ++++++++++++++ .github/workflows/stable_mrs_arm64.yml | 140 ++++++++++++++++++ ...table_amd64.yml => unstable_mrs_amd64.yml} | 11 +- ...table_arm64.yml => unstable_mrs_arm64.yml} | 11 +- .../workflows/unstable_thirdparty_amd64.yml | 112 ++++++++++++++ packages.yaml => mrs.yaml | 0 thirdparty.yaml | 35 +++++ 9 files changed, 424 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/stable_mrs_amd64.yml create mode 100644 .github/workflows/stable_mrs_arm64.yml rename .github/workflows/{unstable_amd64.yml => unstable_mrs_amd64.yml} (86%) rename .github/workflows/{unstable_arm64.yml => unstable_mrs_arm64.yml} (88%) create mode 100644 .github/workflows/unstable_thirdparty_amd64.yml rename packages.yaml => mrs.yaml (100%) create mode 100644 thirdparty.yaml diff --git a/.ci/build.sh b/.ci/build.sh index e54f023f77..19a418c4fa 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -5,10 +5,10 @@ set -e trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG trap 'echo "$0: \"${last_command}\" command failed with exit code $?"' ERR -VARIANT=$1 -PACKAGE_NAME=$2 +YAML_FILE=$1 +VARIANT=$2 +PACKAGE_NAME=$3 WORKSPACE=/tmp/workspace -YAML_FILE=packages.yaml ARTIFACTS_FOLDER=/tmp/artifacts IDX_FILE=$ARTIFACTS_FOLDER/idx.txt diff --git a/.ci/get_build_matrix.sh b/.ci/get_build_matrix.sh index c8f83e7ac7..11cce71646 100755 --- a/.ci/get_build_matrix.sh +++ b/.ci/get_build_matrix.sh @@ -7,10 +7,10 @@ trap 'echo "$0: \"${last_command}\" command failed with exit code $?, log:" && c DEBUG=false -VARIANT=$1 -ARCH=$2 +YAML_FILE=$1 +VARIANT=$2 +ARCH=$3 WORKSPACE=/tmp/workspace -YAML_FILE=packages.yaml ARTIFACTS_FOLDER=/tmp/artifacts ./.ci_scripts/package_build/add_ros_ppa.sh >> /tmp/log.txt 2>&1 diff --git a/.github/workflows/stable_mrs_amd64.yml b/.github/workflows/stable_mrs_amd64.yml new file mode 100644 index 0000000000..8a34e22ff5 --- /dev/null +++ b/.github/workflows/stable_mrs_amd64.yml @@ -0,0 +1,115 @@ +name: stable-amd64 + +on: + workflow_dispatch: + + schedule: + - cron: '0 20 * * *' # every day at 10pm UTC+2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + init-artifacts: + runs-on: ubuntu-20.04 + steps: + - id: execute + run: | + mkdir -p /tmp/artifacts + mkdir -p /tmp/artifacts/metarepositories + touch /tmp/artifacts/generated_amd64.yaml + touch /tmp/artifacts/compiled.txt + echo "0" >> /tmp/artifacts/idx.txt + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + generate-jobs: + needs: init-artifacts + runs-on: ubuntu-20.04 + outputs: + packages: ${{ steps.generate.outputs.packages }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: generate + run: | + JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh mrs.yaml stable amd64) + cat /tmp/log.txt + echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" + + build-job: + needs: generate-jobs + runs-on: ubuntu-20.04 + timeout-minutes: 360 # 6 hour timeout + strategy: + matrix: + job: ${{ fromJson(needs.generate-jobs.outputs.packages) }} + max-parallel: 1 # Run jobs serially + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: build + run: | + .ci/build.sh mrs.yaml stable "${{ matrix.job }}" + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + collect-artifacts: + runs-on: ubuntu-20.04 + needs: build-job + env: + PUSH_TOKEN: ${{ secrets.PUSH_TOKEN }} + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - id: execute + run: | + [ -e /tmp/artifacts/compiled.txt ] && rm /tmp/artifacts/compiled.txt + [ -e /tmp/artifacts/idx.txt ] && rm /tmp/artifacts/idx.txt + [ -e /tmp/artifacts/compile_further.txt ] && rm /tmp/artifacts/compile_further.txt + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - name: Deploy + run: .ci_scripts/package_build/push_to_ppa.sh stable /tmp/artifacts diff --git a/.github/workflows/stable_mrs_arm64.yml b/.github/workflows/stable_mrs_arm64.yml new file mode 100644 index 0000000000..2467e1d04f --- /dev/null +++ b/.github/workflows/stable_mrs_arm64.yml @@ -0,0 +1,140 @@ +name: stable-arm64 + +on: + workflow_dispatch: + + schedule: + - cron: '0 20 * * *' # every day at 10pm UTC+2 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + init-artifacts: + runs-on: ubuntu-20.04 + steps: + - id: execute + run: | + mkdir -p /tmp/artifacts + mkdir -p /tmp/artifacts/metarepositories + touch /tmp/artifacts/generated_arm64.yaml + touch /tmp/artifacts/compiled.txt + echo "0" >> /tmp/artifacts/idx.txt + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + generate-jobs: + needs: init-artifacts + runs-on: ubuntu-20.04 + outputs: + packages: ${{ steps.generate.outputs.packages }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: generate + run: | + JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh mrs.yaml stable arm64) + cat /tmp/log.txt + echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" + + build-job: + needs: generate-jobs + runs-on: ubuntu-20.04 + timeout-minutes: 360 # 6 hour timeout + strategy: + matrix: + job: ${{ fromJson(needs.generate-jobs.outputs.packages) }} + max-parallel: 1 # Run jobs serially + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - uses: ctu-mrs/run-on-arch-action@master + name: build + + id: build + + with: + arch: aarch64 + distro: noetic + + githubToken: ${{ github.token }} + + # Create an artifacts directory + setup: | + mkdir -p "/tmp/artifacts" + mkdir -p "/tmp/repository" + + # Mount the artifacts directory as /artifacts in the container + dockerRunArgs: | + --volume "$PWD:/tmp/repository" + --volume "/tmp/artifacts:/tmp/artifacts" + + # The shell to run commands with in the container + shell: /bin/sh + + # Produce a binary artifact and place it in the mounted volume + run: | + /tmp/repository/.ci/build.sh mrs.yaml stable "${{ matrix.job }}" + + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + collect-artifacts: + runs-on: ubuntu-20.04 + needs: build-job + env: + PUSH_TOKEN: ${{ secrets.PUSH_TOKEN }} + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - id: execute + run: | + [ -e /tmp/artifacts/compiled.txt ] && rm /tmp/artifacts/compiled.txt + [ -e /tmp/artifacts/idx.txt ] && rm /tmp/artifacts/idx.txt + [ -e /tmp/artifacts/compile_further.txt ] && rm /tmp/artifacts/compile_further.txt + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - name: Deploy + run: .ci_scripts/package_build/push_to_ppa.sh stable /tmp/artifacts diff --git a/.github/workflows/unstable_amd64.yml b/.github/workflows/unstable_mrs_amd64.yml similarity index 86% rename from .github/workflows/unstable_amd64.yml rename to .github/workflows/unstable_mrs_amd64.yml index e54a42aedd..e0b23ef9f8 100644 --- a/.github/workflows/unstable_amd64.yml +++ b/.github/workflows/unstable_mrs_amd64.yml @@ -1,4 +1,4 @@ -name: unstable-amd64 +name: unstable-mrs-amd64 on: workflow_dispatch: @@ -46,7 +46,7 @@ jobs: token: ${{ secrets.PUSH_TOKEN }} - id: generate run: | - JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh unstable amd64) + JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh mrs.yaml unstable amd64) cat /tmp/log.txt echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" @@ -77,7 +77,7 @@ jobs: token: ${{ secrets.PUSH_TOKEN }} - id: build run: | - .ci/build.sh unstable "${{ matrix.job }}" + .ci/build.sh mrs.yaml unstable "${{ matrix.job }}" - name: Save artifacts uses: actions/upload-artifact@v3 with: @@ -95,6 +95,11 @@ jobs: with: name: artifacts path: /tmp/artifacts/ + - id: execute + run: | + [ -e /tmp/artifacts/compiled.txt ] && rm /tmp/artifacts/compiled.txt + [ -e /tmp/artifacts/idx.txt ] && rm /tmp/artifacts/idx.txt + [ -e /tmp/artifacts/compile_further.txt ] && rm /tmp/artifacts/compile_further.txt - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/unstable_arm64.yml b/.github/workflows/unstable_mrs_arm64.yml similarity index 88% rename from .github/workflows/unstable_arm64.yml rename to .github/workflows/unstable_mrs_arm64.yml index ea0b63c98f..12c1af3876 100644 --- a/.github/workflows/unstable_arm64.yml +++ b/.github/workflows/unstable_mrs_arm64.yml @@ -1,4 +1,4 @@ -name: unstable-arm64 +name: unstable-mrs-arm64 on: workflow_dispatch: @@ -46,7 +46,7 @@ jobs: token: ${{ secrets.PUSH_TOKEN }} - id: generate run: | - JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh unstable arm64) + JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh mrs.yaml unstable arm64) cat /tmp/log.txt echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" @@ -101,7 +101,7 @@ jobs: # Produce a binary artifact and place it in the mounted volume run: | - /tmp/repository/.ci/build.sh unstable "${{ matrix.job }}" + /tmp/repository/.ci/build.sh mrs.yaml unstable "${{ matrix.job }}" - name: Save artifacts uses: actions/upload-artifact@v3 @@ -120,6 +120,11 @@ jobs: with: name: artifacts path: /tmp/artifacts/ + - id: execute + run: | + [ -e /tmp/artifacts/compiled.txt ] && rm /tmp/artifacts/compiled.txt + [ -e /tmp/artifacts/idx.txt ] && rm /tmp/artifacts/idx.txt + [ -e /tmp/artifacts/compile_further.txt ] && rm /tmp/artifacts/compile_further.txt - uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/unstable_thirdparty_amd64.yml b/.github/workflows/unstable_thirdparty_amd64.yml new file mode 100644 index 0000000000..8aa135ab83 --- /dev/null +++ b/.github/workflows/unstable_thirdparty_amd64.yml @@ -0,0 +1,112 @@ +name: unstable-thirdparty-amd64 + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + init-artifacts: + runs-on: ubuntu-20.04 + steps: + - id: execute + run: | + mkdir -p /tmp/artifacts + mkdir -p /tmp/artifacts/metarepositories + touch /tmp/artifacts/generated_amd64.yaml + touch /tmp/artifacts/compiled.txt + echo "0" >> /tmp/artifacts/idx.txt + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + generate-jobs: + needs: init-artifacts + runs-on: ubuntu-20.04 + outputs: + packages: ${{ steps.generate.outputs.packages }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: generate + run: | + JOB_STRATEGY_MATRIX=$(./.ci/get_build_matrix.sh thirdparty.yaml unstable amd64) + cat /tmp/log.txt + echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT" + + build-job: + needs: generate-jobs + runs-on: ubuntu-20.04 + timeout-minutes: 360 # 6 hour timeout + strategy: + matrix: + job: ${{ fromJson(needs.generate-jobs.outputs.packages) }} + max-parallel: 1 # Run jobs serially + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - id: build + run: | + .ci/build.sh thirdparty.yaml unstable "${{ matrix.job }}" + - name: Save artifacts + uses: actions/upload-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + + collect-artifacts: + runs-on: ubuntu-20.04 + needs: build-job + env: + PUSH_TOKEN: ${{ secrets.PUSH_TOKEN }} + steps: + - name: Load artifacts + uses: actions/download-artifact@v3 + with: + name: artifacts + path: /tmp/artifacts/ + - id: execute + run: | + [ -e /tmp/artifacts/compiled.txt ] && rm /tmp/artifacts/compiled.txt + [ -e /tmp/artifacts/idx.txt ] && rm /tmp/artifacts/idx.txt + [ -e /tmp/artifacts/compile_further.txt ] && rm /tmp/artifacts/compile_further.txt + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + - name: Checkout CI scripts + uses: actions/checkout@v3 + with: + repository: ctu-mrs/ci_scripts + ref: master + path: .ci_scripts + token: ${{ secrets.PUSH_TOKEN }} + - name: Deploy + run: .ci_scripts/package_build/push_to_ppa.sh unstable /tmp/artifacts diff --git a/packages.yaml b/mrs.yaml similarity index 100% rename from packages.yaml rename to mrs.yaml diff --git a/thirdparty.yaml b/thirdparty.yaml new file mode 100644 index 0000000000..1272cf4460 --- /dev/null +++ b/thirdparty.yaml @@ -0,0 +1,35 @@ +nlopt: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/nlopt_ros + unstable_ref: hw_api_ci + stable_ref: hw_api_ci + +ouster_ros: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/ouster-ros + unstable_ref: mrs + stable_ref: mrs + +mavros: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/mavros + unstable_ref: 1.13.0-deb + stable_ref: 1.13.0-deb + +px4_sitl_gazebo: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/px4_sitl_gazebo + unstable_ref: firmware_1.13.2 + stable_ref: firmware_1.13.2 + +mav_comm: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/mav_comm + unstable_ref: master + stable_ref: master + +geometry2: + architecture: [amd64, arm64] + source: https://github.com/ctu-mrs/geometry2 + unstable_ref: warning-fix + stable_ref: warning-fix