Skip to content

Commit

Permalink
CI: Cx Build enhance to build/upload in parallel.
Browse files Browse the repository at this point in the history
  • Loading branch information
@loki077
loki077 committed Dec 9, 2024
1 parent 698cb38 commit 748abd3
Show file tree
Hide file tree
Showing 2 changed files with 159 additions and 90 deletions.
232 changes: 147 additions & 85 deletions .github/workflows/carbonix_build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,52 @@ concurrency:
group: ci-${{github.workflow}}-${{ github.ref }}
cancel-in-progress: true

env:
AIRCRAFTS: '["Volanti", "Ottano", "Other"]'

jobs:
setup-s3-path:
runs-on: ubuntu-22.04
outputs:
s3_path: ${{ steps.set-s3-path.outputs.s3_path }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Extract firmware version, commit id, and branch name
id: extract_info
run: |
FIRMWARE_VERSION=$(grep -oP 'define AP_CUSTOM_FIRMWARE_STRING "\K(.*)(?=")' libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/version.inc)
COMMIT_ID=$(git rev-parse --short HEAD)
BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})
echo "firmware_version=$FIRMWARE_VERSION" >> $GITHUB_ENV
echo "commit_id=$COMMIT_ID" >> $GITHUB_ENV
echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV
shell: sh -e {0}

- name: Set S3 Path
id: set-s3-path
run: |
DATE_HR=$(date +%Y%m%d_%H%M)
if [ "${{ github.event_name }}" == "release" ]; then
PATH_TO_S3=s3://carbonix-firmware-release-files/Carbopilot_V2/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/
echo "Release to: $PATH_TO_S3"
elif [ "${{ github.event_name }}" == "push" ] && [[ "${{ env.branch_name }}" == CxPilot* ]]; then
PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/${{ env.branch_name }}/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/
echo "PUSH : $PATH_TO_S3"
elif [ "${{ github.event_name }}" == "pull_request" ]; then
PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/PR/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}_${{ github.event.pull_request.number }}/
echo "PR : $PATH_TO_S3"
else
PATH_TO_S3="s3://carbonix-firmware-dev-files/Carbopilot_V2/Manual/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/"
echo "Manual trigger or other: $PATH_TO_S3"
fi
echo "::set-output name=s3_path::$PATH_TO_S3"
shell: bash

build-sitl:
runs-on: 'windows-latest'
needs: setup-s3-path
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -171,12 +214,14 @@ jobs:
echo "export CCACHE_MAXSIZE=400M" >> ~/ccache.conf &&
source ~/ccache.conf &&
ccache -s
- name: ccache cache files
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: D:/a/ardupilot/ardupilot/ccache
key: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
restore-keys: ${{ steps.ccache_cache_timestamp.outputs.cache-key }}-ccache- # restore ccache from either previous build on this branch or on base branch

- name: Prepare Python environment
env:
PATH: /usr/bin:$(cygpath ${SYSTEMROOT})/system32
Expand All @@ -201,8 +246,6 @@ jobs:
source ~/ccache.conf &&
Tools/scripts/cygwin_build.sh &&
ccache -s
mkdir -p output/sitl
cp -r artifacts output/sitl
- name: Check build files
id: check_files
Expand All @@ -215,51 +258,65 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: sitl
path: output
path: artifacts
retention-days: 90

- name: Download Artifacts
uses: actions/download-artifact@v4
with:
path: temp

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
aws-region: us-east-1

- name: Upload artifacts to S3
shell: pwsh
run: |
$env:PATH_TO_S3 = '${{ needs.setup-s3-path.outputs.s3_path }}'
echo "Uploading to: $env:PATH_TO_S3"
aws s3 cp temp/ $env:PATH_TO_S3 --recursive
build-apj:
runs-on: ubuntu-22.04
needs: setup-s3-path
container: ardupilot/ardupilot-dev-${{ matrix.toolchain }}:v0.1.3
strategy:
fail-fast: false # don't cancel if a job from the matrix fails
fail-fast: false
matrix:
config: [
CubeOrange,
CubeOrange-Volanti,
CubeOrange-Ottano,
CubeOrangePlus,
CubeOrangePlus-Volanti,
CubeOrangePlus-Ottano,
CarbonixF405,
CarbonixF405-no-crystal,
]
toolchain: [
chibios,
#chibios-clang,
]

toolchain: [ chibios ]
gcc: [10]
exclude:
- gcc: 10
toolchain: chibios-clang

steps:
# git checkout the PR
- uses: actions/checkout@v4
with:
submodules: 'recursive'
# Put ccache into github cache for faster build
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
run: |
NOW=$(date -u +"%F-%T")
echo "timestamp=${NOW}" >> $GITHUB_OUTPUT
- name: ccache cache files
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: ~/.ccache
key: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }}-${{steps.ccache_cache_timestamp.outputs.timestamp}}
restore-keys: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }} # restore ccache from either previous build on this branch or on master
restore-keys: ${{github.workflow}}-ccache-${{matrix.config}}-${{ matrix.toolchain }}-${{ matrix.gcc }}

- name: setup ccache
run: |
. .github/workflows/ccache.env
Expand All @@ -269,98 +326,103 @@ jobs:
sudo apt-get update
sudo apt-get -y install xxd
- name: build ${{matrix.config}} ${{ matrix.toolchain }} gcc-${{matrix.gcc}}
- name: build ${{matrix.config}}
shell: bash
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
if [[ ${{ matrix.toolchain }} == "chibios-clang" ]]; then
export CC=clang
export CXX=clang++
export CC=clang
export CXX=clang++
fi
PATH="/usr/lib/ccache:/opt/gcc-arm-none-eabi-${{matrix.gcc}}/bin:$PATH"
PATH="/github/home/.local/bin:$PATH"
Tools/Carbonix_scripts/carbonix_waf_build.sh ${{ matrix.config }}
ccache -s
ccache -z
- name: Check build files
id: check_files
uses: andstor/file-existence-action@v2
with:
files: "build/${{ matrix.config }}/*"
fail: true
- name: Gather build output
- name: Organize Build Output
shell: bash
run: |
mkdir -p temp/boards
mkdir -p temp/boards/${{ matrix.config }}/bin
cp -vr build/${{ matrix.config }}/bin/* temp/boards/$dir/bin/
# Determine aircraft folder
AIRCRAFT_FOLDER="Other"
for aircraft in ${AIRCRAFTS//[\[\]\",]/}; do
if [[ "${{ matrix.config }}" == *"$aircraft"* ]]; then
AIRCRAFT_FOLDER="$aircraft"
break
fi
done
cp -vr output/* temp/
mv temp/boards/CubeOrange-Volanti temp/Volanti
mv temp/boards/CubeOrangePlus-Volanti temp/Volanti
mv temp/boards/CubeOrange-Ottano temp/Ottano
mv temp/boards/CubeOrangePlus-Ottano temp/Ottano
if [ -d "ArduPlane/ReleaseNotes.txt" ]; then
cp -v ArduPlane/ReleaseNotes.txt temp/
else
echo "ReleaseNotes.txt File does not exist"
fi
if [ -d "libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/payloads" ]; then
cp -vr libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/payloads temp/
else
echo "payloads Folder does not exist"
# Move files
mkdir -p $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/${{ matrix.config }}
cp -r build/${{ matrix.config }}/bin/* $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/${{ matrix.config }}

- name: Organize Build Peripherals Output
shell: bash
run: |
# Move files from output folder
if [ -d "output" ]; then
for build in output/*; do
if [ -d "$build" ]; then
build_name=$(basename "$build")
# Determine aircraft folder
AIRCRAFT_FOLDER="Other"
for aircraft in ${AIRCRAFTS//[\[\]\",]/}; do
if [[ "$build_name" == *"$aircraft"* ]]; then
AIRCRAFT_FOLDER="$aircraft"
break
fi
done
mkdir -p $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/$build_name
cp -r $build/* $GITHUB_WORKSPACE/Aircrafts/$AIRCRAFT_FOLDER/$build_name
fi
done
fi
shell: sh -e {0}
- name: Archive production artifacts
uses: actions/upload-artifact@v4
- name: Install AWS CLI
run: |
apt-get update -y
DEBIAN_FRONTEND=noninteractive apt-get install -y curl unzip
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip -q awscliv2.zip
./aws/install --update
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
name: apj
path: temp
retention-days: 90
aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
aws-region: us-east-1

- name: Upload to S3
run: |
PATH_TO_S3=${{ needs.setup-s3-path.outputs.s3_path }}
echo "Uploading Artifacts to: $PATH_TO_S3"
aws s3 cp $GITHUB_WORKSPACE/Aircrafts/ $PATH_TO_S3 --recursive
upload:
upload_other_files:
runs-on: ubuntu-22.04
needs: [build-apj, build-sitl]
needs: setup-s3-path
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Extract firmware version, commit id, and branch name
id: extract_info
uses: actions/checkout@v3

- name: Prepare Upload Directory
run: mkdir -p temp

- name: Upload Release Notes and Payloads
run: |
FIRMWARE_VERSION=$(grep -oP 'define AP_CUSTOM_FIRMWARE_STRING "\K(.*)(?=")' libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/version.inc)
COMMIT_ID=$(git rev-parse --short HEAD)
BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})
echo "firmware_version=$FIRMWARE_VERSION" >> $GITHUB_ENV
echo "commit_id=$COMMIT_ID" >> $GITHUB_ENV
echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV
shell: sh -e {0}

- name: Download Artifacts
uses: actions/download-artifact@v4
with:
path: temp
cp -v ArduPlane/ReleaseNotes.txt temp/ || echo "No ReleaseNotes found"
cp -vr libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/payloads temp/ || echo "No Payloads found"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Upload artifacts to S3

- name: Upload to S3
run: |
DATE_HR=$(date +%Y%m%d_%H%M)
if ${{ github.event_name == 'release' }}; then
PATH_TO_S3=s3://carbonix-firmware-release-files/Carbopilot_V2/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/
echo "Uploading to: $PATH_TO_S3"
aws s3 cp temp/ $PATH_TO_S3 --recursive
elif ${{ github.event_name == 'push' && startsWith(env.branch_name, 'CxPilot') }}; then
PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/${{ env.branch_name }}/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}/
echo "Uploading to: $PATH_TO_S3"
aws s3 cp temp/ $PATH_TO_S3 --recursive
elif ${{ github.event_name == 'pull_request' }}; then
PATH_TO_S3=s3://carbonix-firmware-dev-files/Carbopilot_V2/PR/${DATE_HR}_${{ env.firmware_version }}_${{ env.commit_id }}_${{ github.event.pull_request.number }}/
echo "Uploading to: $PATH_TO_S3"
aws s3 cp temp/ $PATH_TO_S3 --recursive
fi
PATH_TO_S3=${{ needs.setup-s3-path.outputs.s3_path }}
echo "Uploading Release Notes and Payloads to: $PATH_TO_S3"
aws s3 cp temp/ $PATH_TO_S3 --recursive
17 changes: 12 additions & 5 deletions Tools/Carbonix_scripts/carbonix_waf_build.sh
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
#!/bin/bash

# Exit if any command fails
set -e

echo "Running distclean..."
./waf distclean
rm -Rf output
Expand All @@ -14,8 +11,17 @@ fi

# Set the board name
board=$1
echo "********************************"
echo "Board name: $board"
echo "********************************"

# is_periph_board=$(./waf configure --board "$board" -g | tee >(grep -c "env set AP_PERIPH=1"))
is_periph_board=$(./waf configure --board "$board" -g | grep -c "env set AP_PERIPH=1")

# Exit if any command fails
set -e

is_periph_board= ./waf configure --board "$board" -g | tee >(grep -c "env set AP_PERIPH=1")
echo "is_periph_board: $is_periph_board"

if [ $is_periph_board -eq 0 ]; then
echo "Compiling ArduPlane for $board..."
Expand Down Expand Up @@ -61,7 +67,7 @@ for file in $(find libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/cpn_params/ -na
foldername=$(basename -- "$(dirname -- "$file")")

# Create output folder
output_folder=output/$foldername/${filename}_$board
output_folder=output/${filename}_$board
mkdir -p $output_folder
# Copy param file in the output folder
cp $file $output_folder
Expand Down Expand Up @@ -90,6 +96,7 @@ for file in $(find libraries/AP_HAL_ChibiOS/hwdef/CarbonixCommon/cpn_params/ -na
cp $binary $binary.tmp
xxd -p $binary | tr -d '\n' | sed "s/$board_magic_string_hex/$board_name_hex/g" | xxd -r -p > $binary.tmp
mv $binary.tmp $output_folder/$(basename $binary)
echo "Output binary Saved at : $output_folder/$(basename $binary)"
done
echo ""
done
Expand Down

0 comments on commit 748abd3

Please sign in to comment.