build_and_test.yml

name: Build and Test

on:
  push:
    branches:
      - "main"
      - "mw-*"
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '**/*.md'
      - '**/*.txt'

jobs:
  lint:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@v4

      - name: Lint
        run: make lint

  build:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    needs:
      - lint

    strategy:
      fail-fast: false
      matrix:
        component:
          [
            "wikibase",
            "elasticsearch",
            "wdqs",
            "wdqs-frontend",
            "wdqs-proxy",
            "quickstatements",
          ]

    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4

      - name: Prepare environment and caches
        uses: ./.github/actions/prep-env-and-cache

      - name: Build
        run: ./build.sh ${{ matrix.component }}
        shell: bash

      - name: Upload results
        uses: ./.github/actions/upload-results
        with:
          component: ${{ matrix.component }}

      # TODO: remove extra step with https://phabricator.wikimedia.org/T347000
      - name: Upload additional bundle results for wikibase build
        if: matrix.component == 'wikibase'
        uses: ./.github/actions/upload-results
        with:
          component: ${{ matrix.component }}-bundle

      - name: Scan Image
        uses: ./.github/actions/scan-image
        continue-on-error: true
        with:
          image_name: ${{ matrix.component }}

      # TODO: remove extra step with https://phabricator.wikimedia.org/T347000
      - name: Scan additional bundle Image for wikibase builds
        if: matrix.component == 'wikibase'
        uses: ./.github/actions/scan-image
        continue-on-error: true
        with:
          image_name: "${{ matrix.component }}-bundle"

  test:
    strategy:
      fail-fast: false
      matrix:
        suite:
          [
            repo,
            fedprops,
            repo_client,
            quickstatements,
            pingback,
            confirm_edit,
            elasticsearch,

            base__repo,
            base__repo_client,
            base__pingback,
            base__fedprops,
          ]


        databaseImageName: [
            # 'mysql:5.6', disabled https://phabricator.wikimedia.org/T296066
            "mariadb:10.9",
          ]

    needs:
      - build

    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:
      - uses: actions/checkout@v4

      - name: Get built Docker artifacts
        uses: actions/download-artifact@v3
        with:
          name: DockerImages
          path: artifacts/

      - name: Test ${{ matrix.suite }}
        env:
          DATABASE_IMAGE_NAME: ${{ matrix.databaseImageName }}
        run: make test SUITE=${{ matrix.suite }}

      - name: Report Selenium
        if: always()
        run: |
          cd .github/reporter
          npm install
          SUITE=${{ matrix.suite }} node report.js

      - name: Show Logs
        if: always()
        continue-on-error: true
        run: |
          ls -lahr test/suites/${{ matrix.suite }}/results test/suites/${{ matrix.suite }}/results/*
          tail -n +1 test/suites/${{ matrix.suite }}/results/*.log

      - name: Archive Docker test artifacts
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults
          path: test/suites/**/results

  test_upgrade_base:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    strategy:
      fail-fast: false
      matrix:
        # TODO: can we get this from the environment to not have it hardcoded in the pipeline code?
        version: [
          # Latest 1.37
          'wmde.9',
          # Latest 1.38
          'wmde.12',
          # Latest 1.39
          'wmde.11',
        ]
    needs:
      - build

    steps:
      - uses: actions/checkout@v4

      - name: Get built Docker artifacts
        uses: actions/download-artifact@v3
        with:
          name: DockerImages
          path: artifacts/

      - name: Test upgrade from ${{ matrix.version }}
        # This step should only take ~5 mins to complete, but sometimes seems to lock up and use the whole job timeout
        # Set a specific lower timeout to allow us to retry sooner
        timeout-minutes: 10
        run: make test-upgrade VERSION=${{ matrix.version }}

      - name: Show Docker Logs
        if: always()
        run: test/scripts/docker_logs.sh

      # TODO: DRY
      - name: Archive Docker test artifacts
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults
          path: test/suites/**/results

  test_upgrade_bundle:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    needs:
      - build

    strategy:
      fail-fast: false
      matrix:
        # TODO: can we get this from the environment to not have it hardcoded in the pipeline code?
        version: [
          # Latest 1.37
          'wmde.9-bundle',
          # Latest 1.38
          'wmde.12-bundle',
          # Latest 1.39
          'wmde.11-bundle',
        ]

    env:
      TARGET_WIKIBASE_UPGRADE_IMAGE_NAME: wikibase-bundle

    steps:
      - uses: actions/checkout@v4

      - name: Get built Docker artifacts
        uses: actions/download-artifact@v3
        with:
          name: DockerImages
          path: artifacts/

      - name: Test upgrade from ${{ matrix.version }}
        # This step should only take ~5 mins to complete, but sometimes seems to lock up and use the whole job timeout
        # Set a specific lower timeout to allow us to retry sooner
        timeout-minutes: 10
        run: make test-upgrade VERSION=${{ matrix.version }}

      - name: Show Docker Logs
        if: always()
        run: test/scripts/docker_logs.sh

      # TODO: DRY
      - name: Archive Docker test artifacts
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestResults
          path: test/suites/**/results

  # TODO: https://phabricator.wikimedia.org/T347084
  metadata:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    needs:
      - test

    strategy:
      fail-fast: false

    steps:
      - uses: actions/checkout@v4

      - uses: c-py/action-dotenv-to-setenv@v4
        with:
          env-file: "variables.env"

      - name: Build version requirements
        shell: bash
        run: |
          ./build.sh requirements

      - name: Archive metadata artifacts
        uses: actions/upload-artifact@v3
        with:
          name: Metadata
          path: artifacts/built_versions.log
          if-no-files-found: error

  upload_ghcr:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    # if: github.ref == 'refs/heads/main'

    needs:
      - test
      - test_upgrade_base
      - test_upgrade_bundle

    strategy:
      fail-fast: false
      matrix:
        docker_image:
          [
            "wikibase",
            "wikibase-bundle",
            "elasticsearch",
            "wdqs",
            "wdqs-frontend",
            "wdqs-proxy",
            "quickstatements",
          ]

    steps:
      - uses: actions/checkout@v4

      - name: Get built Docker images
        uses: actions/download-artifact@v3
        with:
          name: DockerImages
          path: artifacts/

      - name: Load docker images
        shell: bash
        run: |
          for file in artifacts/*.docker.tar.gz; do     
            docker load -i "$file"
          done

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Store built docker image on GHCR
        uses: wmde/tag-push-ghcr-action@v3
        with:
          image_name: "wikibase/${{ matrix.docker_image }}"
          tag: ${{ github.run_id }}