wip #2741
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LAPIS-SILO | |
| on: | |
| push: | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies | |
| DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo | |
| jobs: | |
| formatting-check: | |
| name: Formatting Check | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| path: | |
| - check: 'src' | |
| - check: 'include' | |
| exclude: '(PerfEvent.hpp)' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run clang-format style check | |
| uses: jidicula/[email protected] | |
| with: | |
| clang-format-version: '17' | |
| check-path: ${{ matrix.path['check'] }} | |
| exclude-regex: ${{ matrix.path['exclude'] }} | |
| dependencyImage: | |
| name: Build Docker Image Dependencies | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate dependency files hash | |
| id: files-hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Docker metadata | |
| id: dockerMetadata | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} | |
| type=raw,value=${{ env.DIR_HASH }} | |
| type=sha,format=long,prefix=commit- | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check if image exists | |
| id: check-image | |
| run: | | |
| EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false") | |
| echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV | |
| - name: Set up Docker Buildx | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push image if input files changed | |
| if: env.CACHE_HIT == 'false' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile_dependencies | |
| push: true | |
| tags: ${{ steps.dockerMetadata.outputs.tags }} | |
| cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }} | |
| platforms: linux/amd64,linux/arm64 | |
| - name: Retag and push existing image if cache hit | |
| if: env.CACHE_HIT == 'true' | |
| run: | | |
| TAGS=(${{ steps.dockerMetadata.outputs.tags }}) | |
| for TAG in "${TAGS[@]}"; do | |
| docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} | |
| done | |
| dockerImageUnitTests: | |
| name: Build Docker Image and Run Unit Tests | |
| runs-on: ubuntu-latest | |
| needs: dependencyImage | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate dependency files hash | |
| id: files-hash | |
| run: | | |
| DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }}) | |
| echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV | |
| - name: Docker metadata | |
| id: dockerMetadata | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=sha,format=long,prefix=commit- | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build unit test image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| target: builder | |
| tags: builder | |
| load: true | |
| cache-from: type=gha,ref=${{ github.ref_name }}-image-cache | |
| cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache | |
| build-args: | | |
| DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }} | |
| - name: Run tests | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: builder | |
| run: ./silo_test | |
| - name: Build and push production image | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| platforms: ${{ github.ref == 'refs/heads/main' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} | |
| tags: ${{ steps.dockerMetadata.outputs.tags }} | |
| build-args: | | |
| DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }} | |
| endToEndTests: | |
| name: Run End To End Tests | |
| needs: dockerImageUnitTests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| preprocessing-docker-compose: [ | |
| docker-compose-for-tests-preprocessing-from-tsv.yml, | |
| docker-compose-for-tests-preprocessing-from-ndjson.yml | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.x | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} | |
| - name: npm install | |
| run: cd endToEndTests && npm ci | |
| - name: Check Format | |
| run: cd endToEndTests && npm run check-format | |
| - name: Docker Metadata | |
| id: dockerMetadata | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE_NAME }} | |
| tags: type=ref,event=branch | |
| - name: Start Docker Container and preprocess data | |
| run: docker compose -f ${{ matrix.preprocessing-docker-compose }} up | |
| env: | |
| SILO_IMAGE: ${{ steps.dockerMetadata.outputs.tags }} | |
| - name: Start Docker Container and run api | |
| run: docker compose -f docker-compose-for-tests-api.yml up -d --wait | |
| env: | |
| SILO_IMAGE: ${{ steps.dockerMetadata.outputs.tags }} | |
| - name: Run Tests | |
| run: cd endToEndTests && SILO_URL=localhost:8080 npm run test |