Skip to content
Build, test and deploy

aarch64 CI test #110

Sign in to view logs

aarch64 CI test

aarch64 CI test #110

Workflow file for this run

.github/workflows/build-test-deploy.yml at 4111253
# build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact
# test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir
# deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets
# build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact
# test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script
# deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub
name: Build, test and deploy
on:
pull_request:
push:
tags:
- '**'
jobs:
build-executable-x64:
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: Get and verify tag value
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
./scripts/cmp_tags.sh
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]"
- name: Build gProfiler executable
# Using BuildKit although it has another cache mechanism which is not supported by satackey/[email protected]
# We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching,
# decided not to use buildkit here to get nice caches.
run: |
mkdir -p output
./scripts/build_x86_64_executable.sh
mv build/x86_64/gprofiler output/gprofiler_x86_64
cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility
- name: Upload the executables as job artifacts
uses: actions/upload-artifact@v2
with:
name: gprofiler
path: output/
retention-days: 1
test-executable-x64:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running
# on ubuntu-22.04/ubuntu-latest:
# stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError)
needs: build-executable-x64
strategy:
fail-fast: false
matrix:
containers:
- alpine
- ubuntu:14.04
- ubuntu:16.04
- ubuntu:18.04
- ubuntu:20.04
- ubuntu:20.10
- ubuntu:22.04
- centos:6
- centos:7
- centos:8
- debian:8
- debian:9
- debian:10
- debian:11
steps:
- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: "3.8"
- name: Install Java
uses: actions/setup-java@v1
with:
java-version: '8.0.275'
java-package: jdk
architecture: x64
- name: Install Node.JS
uses: actions/setup-node@v2
with:
# same version as used in tests/containers/nodejs/Dockerfile
node-version: 10.x
- name: Checkout Code
uses: actions/checkout@v3
with:
submodules: true
- name: Download the executable from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler
path: dist/
- name: Run gProfiler tests
run: |
mv dist/gprofiler_x86_64 dist/gprofiler
chmod +x dist/gprofiler
NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable
build-executable-aarch64:
runs-on:
- self-hosted
- public
- ARM64
steps:
- name: Checkout Code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: Set up requirements
run: scripts/setup_runner_requirements.sh
- name: Get and verify tag value
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
./scripts/cmp_tags.sh
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build gProfiler executable
run: |
mkdir -p output
./scripts/build_aarch64_executable.sh
mv build/aarch64/gprofiler output/gprofiler_aarch64
- name: Upload the executables as job artifacts
uses: actions/upload-artifact@v2
with:
name: gprofiler_aarch64
path: output/
retention-days: 1
test-executable-aarch64:
needs: build-container-aarch64
runs-on:
- self-hosted
- public
- ARM64
steps:
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Checkout Code
uses: actions/checkout@v3
with:
submodules: true
- name: Download the executable from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler-aarch64
path: dist/
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]"
- name: Run gProfiler tests
run: |
sudo python3 -m pip install virtualenv
sudo virtualenv --python python3.8 aarch64-venv
sudo chown -R ${USER} aarch64-venv
sudo chmod -R a+rX aarch64-venv
source aarch64-venv/bin/activate
./tests/test.sh --executable dist/gprofiler_aarch64 -k test_executable
deploy-executable:
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-20.04
needs:
- build-executable-x64
- build-executable-aarch64
steps:
- name: Download executables from the previous job
uses: actions/download-artifact@v2
with:
name: gprofiler
path: output/
- name: Get tag value
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Release
uses: softprops/action-gh-release@v1
with:
files: |
output/gprofiler_x86_64
output/gprofiler_aarch64
output/gprofiler
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-container-x64:
runs-on: ubuntu-latest
needs:
- build-executable-x64
steps:
- name: Checkout Code
uses: actions/checkout@v3
with:
submodules: true
fetch-depth: 0
- name: Download executables from the previous job
uses: actions/download-artifact@v2
with:
name: gprofiler
path: output/
# the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs
# more nicely. the tests will then be able to use the built image.
- name: Build gProfiler image
run: ./scripts/build_x86_64_container.sh --skip-exe-build --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64
- name: Export gProfiler image
run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img
- name: Upload the image artifact
uses: actions/upload-artifact@v2
with:
name: gprofiler_x86_64.img
path: output/
retention-days: 1
test-container-x64:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-container-x64
runs-on: ubuntu-20.04 # for 3.10.0 python tag
strategy:
fail-fast: false # helps detecting flakiness / errors specific to one Python version
matrix:
python-version:
- "3.8"
- "3.9"
- "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests.
steps:
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install Java
uses: actions/setup-java@v1
with:
java-version: '8.0.275'
java-package: jdk
architecture: x64
- name: Install Node.JS
uses: actions/setup-node@v2
with:
# same version as used in tests/containers/nodejs/Dockerfile
node-version: 10.x
- name: Checkout Code
uses: actions/checkout@v3
with:
submodules: true
- name: Download the executable from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler
path: output/
- name: Download the image from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler_x86_64.img
path: output/
- name: Add +x to gprofiler
run: chmod +x ./output/gprofiler
- name: Extract resources from gProfiler executable
run: sudo ./output/gprofiler extract-resources --resources-dest=./gprofiler/resources
# used in the tests
- name: Import gProfiler image
run: docker image load < output/gprofiler_x86_64.img
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]"
- name: Run gProfiler tests
run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py
build-container-aarch64:
runs-on:
- self-hosted
- public
- ARM64
steps:
- name: Checkout Code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: setup runner requirements
run: ./scripts/setup_runner_requirements.sh
- name: Get and verify tag value
run: ./scripts/verify_tag.sh
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
# the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs
# more nicely. the tests will then be able to use the built image.
- name: Build gProfiler image
run: ./scripts/build_aarch64_container.sh -t gprofiler_aarch64 --output type=docker
- name: Export gProfiler image
run: mkdir -p output && docker image save gprofiler_aarch64 > output/gprofiler_aarch64.img
- name: Upload the image artifact
uses: actions/upload-artifact@v2
with:
name: gprofiler_aarch64.img
path: output/
retention-days: 1
test-container-aarch64:
needs: build-container-aarch64
runs-on:
- self-hosted
- public
- ARM64
steps:
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Checkout Code
uses: actions/checkout@v3
with:
submodules: true
- name: Download the executable from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler_aarch64.img
path: output/
- name: Import gProfiler image
run: docker image load < output/gprofiler_aarch64.img
- name: Extract resources from gProfiler executable
run: sudo docker run -v $PWD/gprofiler/resources:/app/gprofiler/resources gprofiler_aarch64 extract-resources --resources-dest=/app/gprofiler/resources
# TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]"
- name: Run gProfiler tests
run: |
sudo python3 -m pip install virtualenv
sudo virtualenv --python python3.8 aarch64-venv
sudo chown -R ${USER} aarch64-venv
sudo chmod -R a+rX aarch64-venv
source aarch64-venv/bin/activate
./tests/test.sh --ignore=tests/test_executable.py
deploy-container:
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
needs:
- build-container-x64
- build-container-aarch64
steps:
# needed to set up env.GH_REPO etc
- name: Checkout Code
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
# build-container-x64 has uploaded the image as an artifact, we donwload it here.
# build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating
# the manifest.
- name: Download the x86_64 image from previous job
uses: actions/download-artifact@v2
with:
name: gprofiler_x86_64.img
path: output/
- name: Get and verify tag value
run: ./scripts/verify_tag.sh
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Push manifest
run: |
set -x
BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}"
LATEST_IMAGE="${{ env.GH_REPO }}:latest"
AARCH64_IMAGE="$BASE_IMAGE-aarch64"
X86_64_IMAGE="$BASE_IMAGE-x86_64"
docker pull $AARCH64_IMAGE
docker image load < output/gprofiler_x86_64.img
docker tag gprofiler_x86_64 $X86_64_IMAGE
docker push $X86_64_IMAGE
# create manifests for the tag + for 'latest'
docker manifest create "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE"
docker manifest push "$BASE_IMAGE"
docker manifest create "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE"
docker manifest push "$LATEST_IMAGE"