From 157b107887eceda14fa4e62efe45f724e4604caa Mon Sep 17 00:00:00 2001 From: Loki077 Date: Tue, 10 Dec 2024 12:32:10 +1100 Subject: [PATCH] Enhance CI workflow by adding automated testing steps and improving error handling for build failures --- .github/workflows/carbonix_build.yml | 324 ++++++++------------------- 1 file changed, 88 insertions(+), 236 deletions(-) diff --git a/.github/workflows/carbonix_build.yml b/.github/workflows/carbonix_build.yml index 81cc5ff59f..fa13ef9cef 100644 --- a/.github/workflows/carbonix_build.yml +++ b/.github/workflows/carbonix_build.yml @@ -331,244 +331,95 @@ jobs: echo "Uploading Artifacts to: $PATH_TO_S3" aws s3 cp final-output/ $PATH_TO_S3 --recursive + build-sitl: + runs-on: 'windows-latest' + if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' + needs: setup-s3-path + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: bash + run: | + NOW=$(date -u +"%F-%T") + echo "timestamp=${NOW}" >> $GITHUB_OUTPUT + WORKFLOWNAME="${{github.workflow}}" + NAME_DASHED=${WORKFLOWNAME//+( )/_} + echo "cache-key=${NAME_DASHED}" >> $GITHUB_OUTPUT + - uses: cygwin/cygwin-install-action@master + with: + packages: cygwin64 gcc-g++=10.2.0-1 ccache python37 python37-future python37-lxml python37-pip python37-setuptools python37-wheel git procps gettext + add-to-path: false + # Put ccache into github cache for faster build + - name: setup ccache + env: + PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 + shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' + run: >- + mkdir -p /cygdrive/d/a/ardupilot/ardupilot/ccache && + mkdir -p /usr/local/etc && + echo "export CCACHE_SLOPPINESS=file_stat_matches" >> ~/ccache.conf && + echo "export CCACHE_DIR=/cygdrive/d/a/ardupilot/ardupilot/ccache" >> ~/ccache.conf && + echo "export CCACHE_BASEDIR=/cygdrive/d/a/ardupilot/ardupilot" >> ~/ccache.conf && + echo "export CCACHE_COMPRESS=1" >> ~/ccache.conf && + echo "export CCACHE_COMPRESSLEVEL=6" >> ~/ccache.conf && + echo "export CCACHE_MAXSIZE=400M" >> ~/ccache.conf && + source ~/ccache.conf && + ccache -s - # build-sitl: - # runs-on: 'windows-latest' - # if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' - # needs: setup-s3-path - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: 'recursive' - # - name: Prepare ccache timestamp - # id: ccache_cache_timestamp - # shell: bash - # run: | - # NOW=$(date -u +"%F-%T") - # echo "timestamp=${NOW}" >> $GITHUB_OUTPUT - # WORKFLOWNAME="${{github.workflow}}" - # NAME_DASHED=${WORKFLOWNAME//+( )/_} - # echo "cache-key=${NAME_DASHED}" >> $GITHUB_OUTPUT - - # - uses: cygwin/cygwin-install-action@master - # with: - # packages: cygwin64 gcc-g++=10.2.0-1 ccache python37 python37-future python37-lxml python37-pip python37-setuptools python37-wheel git procps gettext - # add-to-path: false - # # Put ccache into github cache for faster build - # - name: setup ccache - # env: - # PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 - # shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' - # run: >- - # mkdir -p /cygdrive/d/a/ardupilot/ardupilot/ccache && - # mkdir -p /usr/local/etc && - # echo "export CCACHE_SLOPPINESS=file_stat_matches" >> ~/ccache.conf && - # echo "export CCACHE_DIR=/cygdrive/d/a/ardupilot/ardupilot/ccache" >> ~/ccache.conf && - # echo "export CCACHE_BASEDIR=/cygdrive/d/a/ardupilot/ardupilot" >> ~/ccache.conf && - # echo "export CCACHE_COMPRESS=1" >> ~/ccache.conf && - # echo "export CCACHE_COMPRESSLEVEL=6" >> ~/ccache.conf && - # echo "export CCACHE_MAXSIZE=400M" >> ~/ccache.conf && - # source ~/ccache.conf && - # ccache -s - - # - name: ccache cache files - # uses: actions/cache@v3 - # with: - # path: D:/a/ardupilot/ardupilot/ccache - # key: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} - # restore-keys: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache- # restore ccache from either previous build on this branch or on base branch - - # - name: Prepare Python environment - # env: - # PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 - # shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' - # run: >- - # ln -sf /usr/bin/python3.7 /usr/bin/python && ln -sf /usr/bin/pip3.7 /usr/bin/pip && - # python -m pip install --progress-bar off empy==3.3.4 pexpect && - # python -m pip install --progress-bar off dronecan --upgrade && - # cp /usr/bin/ccache /usr/local/bin/ && - # cd /usr/local/bin && ln -s ccache /usr/local/bin/gcc && - # ln -s ccache /usr/local/bin/g++ && - # ln -s ccache /usr/local/bin/x86_64-pc-cygwin-gcc && - # ln -s ccache /usr/local/bin/x86_64-pc-cygwin-g++ - - # - name: Build SITL - # env: - # PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 - # shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' - # run: >- - # git config --global --add safe.directory /cygdrive/d/a/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/} && - # export PATH=/usr/local/bin:/usr/bin:$(cygpath ${SYSTEMROOT})/system32 && - # source ~/ccache.conf && - # Tools/scripts/cygwin_build.sh && - # ccache -s - - # - name: Check build files - # id: check_files - # uses: andstor/file-existence-action@v2 - # with: - # files: "artifacts/*.exe" - # fail: true - - # - name: Archive build - # uses: actions/upload-artifact@v4 - # with: - # name: sitl - # path: artifacts - # retention-days: 90 - - # - name: Download Artifacts - # uses: actions/download-artifact@v4 - # with: - # path: temp - - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - # aws-region: us-east-1 - - # - name: Upload artifacts to S3 - # shell: pwsh - # run: | - # $env:PATH_TO_S3 = '${{ needs.setup-s3-path.outputs.s3_path }}' - # echo "Uploading to: $env:PATH_TO_S3" - # aws s3 cp temp/ $env:PATH_TO_S3 --recursive - - # build-apj: - # runs-on: ubuntu-22.04 - # needs: setup-s3-path - # container: ardupilot/ardupilot-dev-${{ matrix.toolchain }}:v0.1.3 - # strategy: - # fail-fast: false - # matrix: - # config: [ - # CubeOrange-Volanti, - # CubeOrange-Ottano, - # ] - - # toolchain: [ chibios ] - # gcc: [10] - # exclude: - # - gcc: 10 - # toolchain: chibios-clang - - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: 'recursive' - - # - name: Prepare ccache timestamp - # id: ccache_cache_timestamp - # run: | - # NOW=$(date -u +"%F-%T") - # echo "timestamp=${NOW}" >> $GITHUB_OUTPUT - - # - name: ccache cache files - # uses: actions/cache@v3 - # with: - # path: ~/.ccache - # key: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }}-${{steps.ccache_cache_timestamp.outputs.timestamp}} - # restore-keys: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }} + - name: ccache cache files + uses: actions/cache@v3 + with: + path: D:/a/ardupilot/ardupilot/ccache + key: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} + restore-keys: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache- # restore ccache from either previous build on this branch or on base branch - # - name: setup ccache - # run: | - # . .github/workflows/ccache.env - - # - name: Install bash tools - # run: | - # sudo apt-get update - # sudo apt-get -y install xxd - - # - name: build ${{matrix.config}} - # shell: bash - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # if [[ ${{ matrix.toolchain }} == "chibios-clang" ]]; then - # export CC=clang - # export CXX=clang++ - # fi - # PATH="/usr/lib/ccache:/opt/gcc-arm-none-eabi-${{matrix.gcc}}/bin:$PATH" - # PATH="/github/home/.local/bin:$PATH" - # Tools/Carbonix_scripts/carbonix_waf_build.sh ${{ matrix.config }} - # ccache -s - # ccache -z - - # - name: Organize Build Output - # shell: bash - # run: | - # # Determine aircraft folder - # AIRCRAFT_FOLDER="Other" - # for aircraft in ${AIRCRAFTS//[\[\]\",]/}; do - # if [[ "${{ matrix.config }}" == *"$aircraft"* ]]; then - # AIRCRAFT_FOLDER="$aircraft" - # break - # fi - # done - - # # Move files - # mkdir -p $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/${{ matrix.config }} - # cp -r build/${{ matrix.config }}/bin/* $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/${{ matrix.config }} + - name: Prepare Python environment + env: + PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 + shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' + run: >- + ln -sf /usr/bin/python3.7 /usr/bin/python && ln -sf /usr/bin/pip3.7 /usr/bin/pip && + python -m pip install --progress-bar off empy==3.3.4 pexpect && + python -m pip install --progress-bar off dronecan --upgrade && + cp /usr/bin/ccache /usr/local/bin/ && + cd /usr/local/bin && ln -s ccache /usr/local/bin/gcc && + ln -s ccache /usr/local/bin/g++ && + ln -s ccache /usr/local/bin/x86_64-pc-cygwin-gcc && + ln -s ccache /usr/local/bin/x86_64-pc-cygwin-g++ + + - name: Build SITL + env: + PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 + shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' + run: >- + git config --global --add safe.directory /cygdrive/d/a/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/} && + export PATH=/usr/local/bin:/usr/bin:$(cygpath ${SYSTEMROOT})/system32 && + source ~/ccache.conf && + Tools/scripts/cygwin_build.sh && + ccache -s - # - name: Organize Build Peripherals Output - # shell: bash - # run: | - # # Move files from output folder - # if [ -d "output" ]; then - # for build in output/*; do - # if [ -d "$build" ]; then - # build_name=$(basename "$build") - # # Determine aircraft folder - # AIRCRAFT_FOLDER="Other" - # for aircraft in ${AIRCRAFTS//[\[\]\",]/}; do - # if [[ "$build_name" == *"$aircraft"* ]]; then - # AIRCRAFT_FOLDER="$aircraft" - # break - # fi - # done - # mkdir -p $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/$build_name - # cp -r $build/* $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/$build_name - # fi - # done - # fi + - name: Check build files + id: check_files + uses: andstor/file-existence-action@v2 + with: + files: "artifacts/*.exe" + fail: true - # - name: Install AWS CLI - # run: | - # apt-get update -y - # DEBIAN_FRONTEND=noninteractive apt-get install -y curl unzip - # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - # unzip -q awscliv2.zip - # ./aws/install --update - - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - # aws-region: us-east-1 - - # - name: Upload to S3 - # run: | - # PATH_TO_S3=${{ needs.setup-s3-path.outputs.s3_path }} - # echo "Uploading Artifacts to: $PATH_TO_S3" - # aws s3 cp $GITHUB_WORKSPACE/Aircrafts/ $PATH_TO_S3 --recursive + - name: Archive build + uses: actions/upload-artifact@v4 + with: + name: sitl + path: artifacts + retention-days: 90 - upload_other_files: - runs-on: ubuntu-22.04 - needs: setup-s3-path - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Prepare Upload Directory - run: mkdir -p temp - - - name: Upload Release Notes and Payloads - run: | - cp -v ArduPlane/ReleaseNotes.txt temp/ || echo "No ReleaseNotes found" - cp -vr libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/payloads temp/ || echo "No Payloads found" + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: temp - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -577,8 +428,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} aws-region: us-east-1 - - name: Upload to S3 + - name: Upload artifacts to S3 + shell: pwsh run: | - PATH_TO_S3=${{ needs.setup-s3-path.outputs.s3_path }} - echo "Uploading Release Notes and Payloads to: $PATH_TO_S3" - aws s3 cp temp/ $PATH_TO_S3 --recursive \ No newline at end of file + $env:PATH_TO_S3 = '${{ needs.setup-s3-path.outputs.s3_path }}' + echo "Uploading to: $env:PATH_TO_S3" + aws s3 cp temp/ $env:PATH_TO_S3 --recursive