build-and-deploy.yml

name: build-and-deploy

on:
  push:
    branches:
      - master
      - staging
      - release
  pull_request:
  repository_dispatch:
    types: build-and-deploy
  workflow_dispatch:
  workflow_call:

concurrency:
  group: cli-${{ github.workflow }}-${{ github.ref_type }}-${{ github.event.pull_request.number || github.ref || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash -euxo pipefail {0}

env:
  VERBOSE: 1

jobs:
  build-and-deploy-datasets:
    # Run only if it's not a fork
    if: ${{ !((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository))) }}

    runs-on: ubuntu-20.04

    environment:
      name: ${{ github.ref }}

    env:
      VERBOSE: 0
      PYTHONUNBUFFERED: 1
      GH_TOKEN: ${{ github.token }}

    steps:
      - name: config
        id: config
        run: |
          echo 'is_fork=${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository)) }}' >> $GITHUB_OUTPUT

          echo 'should_deploy=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT

          echo 'is_release=${{ github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT

      - name: "DEBUG"
        run: |
          set +x

          echo is_fork=${{ steps.config.outputs.is_fork }}
          echo should_deploy=${{ steps.config.outputs.should_deploy }}
          echo is_release=${{ steps.config.outputs.is_release }}

          echo '\n'

          echo 'github.ref =' ${{ github.ref }}
          echo 'github.event_name =' ${{ github.event_name }}
          echo 'github.repository =' ${{ github.repository }}

          echo '\n'

          echo 'github.event.pull_request.head.repo.fork =' ${{ github.event.pull_request.head.repo.fork }}
          echo 'github.event.pull_request.head.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name }}

          echo '\n'

          echo 'github.event.pull_request.base.repo.fork =' ${{ github.event.pull_request.base.repo.fork }}
          echo 'github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.base.repo.full_name }}

          echo '\n'

          echo 'github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name }}
          echo '(github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository =' ${{ (github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository }}
          echo 'github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) =' ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) }}

      - name: "Checkout code (fork)"
        if: ${{ steps.config.outputs.is_fork == 'true' }}
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags.
          fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0.
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: "Checkout code"
        if: ${{ steps.config.outputs.is_fork != 'true' }}
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags.
          fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0.
          ref: ${{ github.head_ref || github.ref }}

      - name: "Install system dependencies"
        run: |
          sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null

      - name: "Install Python dependencies"
        run: |
          pip3 install -r requirements.txt

      - name: "Rebuild datasets (fork)"
        if: ${{ steps.config.outputs.is_fork == 'true' }}
        run: |
          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --no-pull

      - name: "Rebuild, commit and push datasets (non-release branch)"
        if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release != 'true' }}
        run: |
          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"

          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --push --repo="${GITHUB_REPOSITORY}"

      - name: "Rebuild, commit, push and make a release (release branch)"
        if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release == 'true' }}
        run: |
          git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}"
          git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}"

          ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}"

      - name: "Install awscli"
        if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }}
        run: |
          pushd /tmp >/dev/null
            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip -oqq awscliv2.zip
            sudo ./aws/install --update
          popd >/dev/null
          aws --version

      - name: "Deploy dataset server"
        if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }}
        env:
          DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }}
          DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-2
        run: |
          ./scripts/upload 'data_output/'

      - name: "Upload build artifacts: dataset server"
        uses: actions/upload-artifact@v4
        with:
          name: server
          path: ./data_output/*

      - name: "Upload build artifacts: zip archives"
        uses: actions/upload-artifact@v4
        with:
          name: zips
          path: ./data_temp/*