Tools: speed up Carbonix build script #660
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Carbonix Build | |
| on: | |
| push: | |
| branches: | |
| - CxPilot | |
| - CxPilot-* | |
| - master | |
| paths-ignore: | |
| # remove non chibios HAL | |
| - 'libraries/AP_HAL_Linux/**' | |
| - 'libraries/AP_HAL_ESP32/**' | |
| - 'libraries/AP_HAL_SITL/**' | |
| # remove non stm directories | |
| - 'Tools/CHDK-Script/**' | |
| - 'Tools/CodeStyle/**' | |
| - 'Tools/completion/**' | |
| - 'Tools/debug/**' | |
| - 'Tools/environment_install/**' | |
| - 'Tools/FilterTestTool/**' | |
| - 'Tools/Frame_params/**' | |
| - 'Tools/geotag/**' | |
| - 'Tools/GIT_Test/**' | |
| - 'Tools/gittools/**' | |
| - 'Tools/Hello/**' | |
| - 'Tools/LogAnalyzer/**' | |
| - 'Tools/mavproxy_modules/**' | |
| - 'Tools/Pozyx/**' | |
| - 'Tools/PrintVersion.py' | |
| - 'Tools/Replay/**' | |
| - 'Tools/simulink/**' | |
| - 'Tools/UDP_Proxy/**' | |
| - 'Tools/vagrant/**' | |
| - 'Tools/Vicon/**' | |
| # Remove vehicles autotest we need support of test_build_option.py in the Tools/autotest directory | |
| - 'Tools/autotest/antennatracker.py' | |
| - 'Tools/autotest/arduplane.py' | |
| - 'Tools/autotest/ardusub.py' | |
| - 'Tools/autotest/balancebot.py' | |
| - 'Tools/autotest/location.txt' | |
| - 'Tools/autotest/quadplane.py' | |
| - 'Tools/autotest/rover.py' | |
| - 'Tools/autotest/sailboat.py' | |
| - 'Tools/autotest/swarminit.txt' | |
| # Remove markdown files as irrelevant | |
| - '**.md' | |
| # Remove dotfile at root directory | |
| - './.dir-locals.el' | |
| - './.dockerignore' | |
| - './.editorconfig' | |
| - './.flake8' | |
| - './.gitattributes' | |
| - './.github' | |
| - './.gitignore' | |
| - './.pre-commit-config.yaml' | |
| - './.pydevproject' | |
| - './.valgrind-suppressions' | |
| - './.valgrindrc' | |
| - 'Dockerfile' | |
| - 'Vagrantfile' | |
| - 'Makefile' | |
| # Remove some directories check | |
| - '.vscode/**' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| pull_request: | |
| paths-ignore: | |
| # remove non chibios HAL | |
| - 'libraries/AP_HAL_Linux/**' | |
| - 'libraries/AP_HAL_ESP32/**' | |
| - 'libraries/AP_HAL_SITL/**' | |
| # remove non stm directories | |
| - 'Tools/CHDK-Script/**' | |
| - 'Tools/CodeStyle/**' | |
| - 'Tools/completion/**' | |
| - 'Tools/debug/**' | |
| - 'Tools/environment_install/**' | |
| - 'Tools/FilterTestTool/**' | |
| - 'Tools/Frame_params/**' | |
| - 'Tools/geotag/**' | |
| - 'Tools/GIT_Test/**' | |
| - 'Tools/gittools/**' | |
| - 'Tools/Hello/**' | |
| - 'Tools/LogAnalyzer/**' | |
| - 'Tools/mavproxy_modules/**' | |
| - 'Tools/Pozyx/**' | |
| - 'Tools/PrintVersion.py' | |
| - 'Tools/Replay/**' | |
| - 'Tools/simulink/**' | |
| - 'Tools/UDP_Proxy/**' | |
| - 'Tools/vagrant/**' | |
| - 'Tools/Vicon/**' | |
| # Remove vehicles autotest we need support of test_build_option.py in the Tools/autotest directory | |
| - 'Tools/autotest/antennatracker.py' | |
| - 'Tools/autotest/arduplane.py' | |
| - 'Tools/autotest/ardusub.py' | |
| - 'Tools/autotest/autotest.py' | |
| - 'Tools/autotest/balancebot.py' | |
| - 'Tools/autotest/common.py' | |
| - 'Tools/autotest/examples.py' | |
| - 'Tools/autotest/quadplane.py' | |
| - 'Tools/autotest/rover.py' | |
| - 'Tools/autotest/sailboat.py' | |
| - 'Tools/autotest/**.txt' | |
| - 'Tools/autotest/logger_metadata/**' | |
| - 'Tools/autotest/param_metadata/**' | |
| # Remove markdown files as irrelevant | |
| - '**.md' | |
| # Remove dotfile at root directory | |
| - './.dir-locals.el' | |
| - './.dockerignore' | |
| - './.editorconfig' | |
| - './.flake8' | |
| - './.gitattributes' | |
| - './.github' | |
| - './.gitignore' | |
| - './.pre-commit-config.yaml' | |
| - './.pydevproject' | |
| - './.valgrind-suppressions' | |
| - './.valgrindrc' | |
| - 'Dockerfile' | |
| - 'Vagrantfile' | |
| - 'Makefile' | |
| # Remove some directories check | |
| - '.vscode/**' | |
| - '.github/ISSUE_TEMPLATE/**' | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-${{github.workflow}}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup-s3-path: | |
| runs-on: ubuntu-22.04 | |
| if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' | |
| outputs: | |
| s3_path: ${{ steps.set-s3-path.outputs.s3_path }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Extract firmware version, commit id, and branch name | |
| id: extract_info | |
| run: | | |
| FIRMWARE_VERSION=$(grep -oP 'define AP_CUSTOM_FIRMWARE_STRING "\K(.*)(?=")' libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/version.inc) | |
| COMMIT_ID=$(git rev-parse --short HEAD) | |
| BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/}) | |
| echo "firmware_version=$FIRMWARE_VERSION" >> $GITHUB_ENV | |
| echo "commit_id=$COMMIT_ID" >> $GITHUB_ENV | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV | |
| shell: sh -e {0} | |
| - name: Set S3 Path | |
| id: set-s3-path | |
| run: | | |
| DATE_HR=$(date +%Y%m%d_%H%M) | |
| if [ "${{ github.event_name }}" == "release" ]; then | |
| PATH_TO_S3=s3://carbonix-firmware-release-files/Carbopilot_V2/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/ | |
| echo "Release to: $PATH_TO_S3" | |
| elif [ "${{ github.event_name }}" == "push" ] && [[ "${{ env.branch_name }}" == CxPilot* ]]; then | |
| PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/${{ env.branch_name }}/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/ | |
| echo "PUSH : $PATH_TO_S3" | |
| elif [ "${{ github.event_name }}" == "pull_request" ]; then | |
| PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/PR/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}_${{ github.event.pull_request.number }}/ | |
| echo "PR : $PATH_TO_S3" | |
| else | |
| PATH_TO_S3="s3://carbonix-firmware-dev-files/Carbopilot_V2/Manual/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/" | |
| echo "Manual trigger or other: $PATH_TO_S3" | |
| fi | |
| echo "::set-output name=s3_path::$PATH_TO_S3" | |
| shell: bash | |
| build-periph: | |
| runs-on: ubuntu-22.04 | |
| if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' | |
| needs: setup-s3-path | |
| container: ardupilot/ardupilot-dev-${{ matrix.toolchain }}:v0.1.3 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: [ | |
| CarbonixF405, | |
| CarbonixF405-no-crystal | |
| ] | |
| toolchain: [ chibios ] | |
| gcc: [10] | |
| exclude: | |
| - gcc: 10 | |
| toolchain: chibios-clang | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| run: | | |
| NOW=$(date -u +"%F-%T") | |
| echo "timestamp=${NOW}" >> $GITHUB_OUTPUT | |
| - name: ccache cache files | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.ccache | |
| key: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }}-${{steps.ccache_cache_timestamp.outputs.timestamp}} | |
| restore-keys: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }} | |
| - name: setup ccache | |
| run: | | |
| . .github/workflows/ccache.env | |
| - name: Install bash tools | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install xxd | |
| - name: build ${{matrix.config}} | |
| shell: bash | |
| run: | | |
| git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| if [[ ${{ matrix.toolchain }} == "chibios-clang" ]]; then | |
| export CC=clang | |
| export CXX=clang++ | |
| fi | |
| PATH="/usr/lib/ccache:/opt/gcc-arm-none-eabi-${{matrix.gcc}}/bin:$PATH" | |
| PATH="/github/home/.local/bin:$PATH" | |
| Tools/Carbonix_scripts/carbonix_waf_build.sh ${{ matrix.config }} | |
| ccache -s | |
| ccache -z | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-periph-${{ matrix.config }} | |
| path: output/ | |
| collect-aircraft-config-files: | |
| if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' | |
| runs-on: ubuntu-22.04 | |
| needs: build-periph | |
| outputs: | |
| aircraft-config-files: ${{ steps.collect.outputs.aircraft-config-files }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install jq | |
| run: sudo apt-get update && sudo apt-get install -y jq | |
| - name: Collect XML files | |
| id: collect | |
| run: | | |
| xml_files=$(find libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/aircraft_configuration -name "*.xml" -print0 | xargs -0 echo | tr -d '\n' | jq -R -s -c 'split(" ")') | |
| echo "aircraft-config-files=$xml_files" >> $GITHUB_OUTPUT | |
| process-aircraft-config: | |
| if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' | |
| runs-on: ubuntu-22.04 | |
| needs: [collect-aircraft-config-files, setup-s3-path] | |
| container: ardupilot/ardupilot-dev-${{ matrix.toolchain }}:v0.1.3 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| xml_file: ${{ fromJson(needs.collect-aircraft-config-files.outputs.aircraft-config-files) }} | |
| toolchain: [ chibios ] | |
| gcc: [10] | |
| exclude: | |
| - gcc: 10 | |
| toolchain: chibios-clang | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Install bash tools | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install xxd | |
| - name: Download all build artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: periph-build/ | |
| - name: List files | |
| run: | | |
| ls -la periph-build/*/ | |
| - name: Configure Git Safe Directory | |
| run: | | |
| git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
| - name: Get Commit ID | |
| id: get_commit_id | |
| run: | | |
| COMMIT_ID=$(git rev-parse --short HEAD) | |
| echo "commit_id=$COMMIT_ID" >> $GITHUB_ENV | |
| shell: sh -e {0} | |
| - name: Run aircraft_config.py | |
| run: | | |
| python Tools/Carbonix_scripts/aircraft_config.py ${{ matrix.xml_file }} ${{ env.commit_id }} | |
| ls -la final-output/*/ || echo "No files found" | |
| - name: Install AWS CLI | |
| run: | | |
| apt-get update -y | |
| DEBIAN_FRONTEND=noninteractive apt-get install -y curl unzip | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| ./aws/install --update | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Upload to S3 | |
| run: | | |
| PATH_TO_S3=${{ needs.setup-s3-path.outputs.s3_path }} | |
| echo "Uploading Artifacts to: $PATH_TO_S3" | |
| aws s3 cp final-output/ $PATH_TO_S3 --recursive | |
| build-sitl: | |
| runs-on: 'windows-latest' | |
| if: github.event.pull_request.labels.*.name != 'SKIP_BUILD' | |
| needs: setup-s3-path | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| shell: bash | |
| run: | | |
| NOW=$(date -u +"%F-%T") | |
| echo "timestamp=${NOW}" >> $GITHUB_OUTPUT | |
| WORKFLOWNAME="${{github.workflow}}" | |
| NAME_DASHED=${WORKFLOWNAME//+( )/_} | |
| echo "cache-key=${NAME_DASHED}" >> $GITHUB_OUTPUT | |
| - uses: cygwin/cygwin-install-action@master | |
| with: | |
| packages: cygwin64 gcc-g++=10.2.0-1 ccache python37 python37-future python37-lxml python37-pip python37-setuptools python37-wheel git procps gettext | |
| add-to-path: false | |
| # Put ccache into github cache for faster build | |
| - name: setup ccache | |
| env: | |
| PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 | |
| shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' | |
| run: >- | |
| mkdir -p /cygdrive/d/a/ardupilot/ardupilot/ccache && | |
| mkdir -p /usr/local/etc && | |
| echo "export CCACHE_SLOPPINESS=file_stat_matches" >> ~/ccache.conf && | |
| echo "export CCACHE_DIR=/cygdrive/d/a/ardupilot/ardupilot/ccache" >> ~/ccache.conf && | |
| echo "export CCACHE_BASEDIR=/cygdrive/d/a/ardupilot/ardupilot" >> ~/ccache.conf && | |
| echo "export CCACHE_COMPRESS=1" >> ~/ccache.conf && | |
| echo "export CCACHE_COMPRESSLEVEL=6" >> ~/ccache.conf && | |
| echo "export CCACHE_MAXSIZE=400M" >> ~/ccache.conf && | |
| source ~/ccache.conf && | |
| ccache -s | |
| - name: ccache cache files | |
| uses: actions/cache@v3 | |
| with: | |
| path: D:/a/ardupilot/ardupilot/ccache | |
| key: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} | |
| restore-keys: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache- # restore ccache from either previous build on this branch or on base branch | |
| - name: Prepare Python environment | |
| env: | |
| PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 | |
| shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' | |
| run: >- | |
| ln -sf /usr/bin/python3.7 /usr/bin/python && ln -sf /usr/bin/pip3.7 /usr/bin/pip && | |
| python -m pip install --progress-bar off empy==3.3.4 pexpect && | |
| python -m pip install --progress-bar off dronecan --upgrade && | |
| cp /usr/bin/ccache /usr/local/bin/ && | |
| cd /usr/local/bin && ln -s ccache /usr/local/bin/gcc && | |
| ln -s ccache /usr/local/bin/g++ && | |
| ln -s ccache /usr/local/bin/x86_64-pc-cygwin-gcc && | |
| ln -s ccache /usr/local/bin/x86_64-pc-cygwin-g++ | |
| - name: Build SITL | |
| env: | |
| PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32 | |
| shell: C:\cygwin\bin\bash.exe -eo pipefail '{0}' | |
| run: >- | |
| git config --global --add safe.directory /cygdrive/d/a/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}/${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/} && | |
| export PATH=/usr/local/bin:/usr/bin:$(cygpath ${SYSTEMROOT})/system32 && | |
| source ~/ccache.conf && | |
| Tools/scripts/cygwin_build.sh && | |
| ccache -s | |
| - name: Check build files | |
| id: check_files | |
| uses: andstor/file-existence-action@v2 | |
| with: | |
| files: "artifacts/*.exe" | |
| fail: true | |
| - name: Archive build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sitl | |
| path: artifacts | |
| retention-days: 90 | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: temp | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Upload artifacts to S3 | |
| shell: pwsh | |
| run: | | |
| $env:PATH_TO_S3 = '${{ needs.setup-s3-path.outputs.s3_path }}' | |
| echo "Uploading to: $env:PATH_TO_S3" | |
| aws s3 cp temp/ $env:PATH_TO_S3 --recursive |