build: run linter on all files on main (#504) #2958
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: LAPIS-SILO | |
on: | |
pull_request: | |
push: | |
branches: | |
- main | |
concurrency: | |
group: ci-${{ github.ref }} | |
cancel-in-progress: true | |
env: | |
DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
jobs: | |
dockerImage: | |
name: Build docker images | |
strategy: | |
matrix: | |
arch: | |
['amd64', 'arm64'] | |
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} | |
permissions: | |
packages: write | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- name: Generate dependency files hash | |
run: | | |
DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', './Dockerfile_dependencies') }}) | |
echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
- name: Check if dependency image exists | |
run: | | |
EXISTS=$(docker manifest inspect \ | |
${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} \ | |
> /dev/null 2>&1 && echo "true" || echo "false") | |
echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
- name: Build and push dependency image if input files changed | |
if: env.CACHE_HIT == 'false' | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
file: Dockerfile_dependencies | |
push: true | |
tags: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
cache-from: type=gha,ref=dependency-image-cache-${{ matrix.arch }}-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
cache-to: type=gha,mode=min,ref=dependency-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
- name: Tag dependency image with commit hash | |
run: | | |
docker buildx imagetools create \ | |
--tag ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} \ | |
${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:filehash-${{ env.DIR_HASH }}-${{ matrix.arch }} | |
- name: Build unit test image | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
target: builder | |
tags: builder | |
load: true | |
cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
build-args: | | |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
- name: Run unit tests | |
run: | | |
docker run \ | |
--entrypoint "./silo_test" \ | |
builder | |
- name: Build and push production image | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
push: true | |
cache-from: type=gha,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
cache-to: type=gha,mode=min,ref=image-cache-${{ github.ref_name }}-${{ matrix.arch }} | |
tags: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
build-args: | | |
DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-${{ matrix.arch }} | |
multiPlatformImages: | |
name: Create multi-platform images | |
needs: dockerImage | |
runs-on: ubuntu-latest | |
steps: | |
- name: Login to GitHub Container Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- name: Docker metadata | |
id: dockerMetadataImage | |
uses: docker/metadata-action@v5 | |
with: | |
images: ${{ env.DOCKER_IMAGE_NAME }} | |
tags: | | |
type=ref,event=branch | |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
type=raw,value=commit-${{ env.HEAD_SHA }} | |
- name: Tag images | |
run: | | |
TAGS=(${{ steps.dockerMetadataImage.outputs.tags }}) | |
for TAG in "${TAGS[@]}"; do | |
docker buildx imagetools create --tag $TAG \ | |
"${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-arm64" \ | |
"${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64" | |
done | |
endToEndTests: | |
name: Run End To End Tests | |
needs: dockerImage | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Use Node.js ${{ matrix.node-version }} | |
uses: actions/setup-node@v4 | |
with: | |
node-version: 18.x | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- uses: actions/cache@v4 | |
with: | |
path: ~/.npm | |
key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" | |
- name: npm install | |
run: cd endToEndTests && npm ci | |
- name: Check Format | |
run: cd endToEndTests && npm run check-format | |
- name: Start Docker Container and preprocess data | |
run: docker compose -f docker-compose-for-tests-preprocessing-from-ndjson.yml up | |
env: | |
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
- name: Start Docker Container and run api | |
run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
env: | |
SILO_IMAGE: ${{ env.DOCKER_IMAGE_NAME }}:commit-${{ env.HEAD_SHA }}-amd64 | |
- name: Run Tests | |
run: cd endToEndTests && SILO_URL=localhost:8080 npm run test | |
linterChanges: | |
name: Build/Run linter on changed files | |
needs: dockerImage | |
if: github.event_name == 'pull_request' | |
runs-on: ubuntu-latest | |
env: | |
PR_NUMBER: ${{ github.event.number }} | |
container: | |
image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.event.pull_request.head.sha }}-amd64 | |
steps: | |
- uses: actions/checkout@v4 | |
- shell: bash | |
name: Configure and run clang-tidy on changed files | |
run: | | |
mv /src/build . | |
cmake -DBUILD_WITH_CLANG_TIDY=on -D CMAKE_BUILD_TYPE=Debug -B build/Debug | |
echo "Successfully configured cmake" | |
files="" | |
PAGE=1 | |
while true; do | |
page_files=$(curl -s \ | |
"https://api.github.com/repos/${{ github.repository }}/pulls/${PR_NUMBER}/files?per_page=100&page=$PAGE" \ | |
| jq -r '.[] | select(.status != "removed") | .filename') | |
# If there are no more files, break the loop | |
if [[ -z "$page_files" ]]; then | |
break | |
fi | |
files+="$page_files"$'\n' | |
PAGE=$((PAGE + 1)) | |
done | |
echo "Changed files of this PR:" | |
echo "$files" | |
IFS=$'\n' | |
for file in $files; do | |
echo "Check ending for file: $file" | |
if [[ $file == *.cpp ]]; then | |
echo "Now linting the file: $file" | |
echo "cmake --build build/Debug --target ${file%.cpp}.o" | |
cmake --build build/Debug --target ${file%.cpp}.o | |
fi | |
done | |
linterAll: | |
name: Build/Run linter on all files | |
needs: dockerImage | |
if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'trigger-linter') | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Determine HEAD_SHA | |
run: | | |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
else | |
echo "HEAD_SHA=${{ github.sha }}" >> $GITHUB_ENV | |
fi | |
- shell: bash | |
name: Build all files with clang-tidy | |
run: | | |
docker build -f ./Dockerfile_linter \ | |
--build-arg DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ env.HEAD_SHA }}-amd64 . |