Update SC2 datasets #440
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-and-deploy | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - staging | |
| - release | |
| pull_request: | |
| repository_dispatch: | |
| types: build-and-deploy | |
| workflow_dispatch: | |
| workflow_call: | |
| concurrency: | |
| group: cli-${{ github.workflow }}-${{ github.ref_type }}-${{ github.event.pull_request.number || github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash -euxo pipefail {0} | |
| env: | |
| VERBOSE: 1 | |
| jobs: | |
| build-and-deploy-datasets: | |
| # Run only if it's not a fork | |
| if: ${{ !((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository))) }} | |
| runs-on: ubuntu-20.04 | |
| environment: | |
| name: ${{ github.ref }} | |
| env: | |
| VERBOSE: 0 | |
| PYTHONUNBUFFERED: 1 | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: config | |
| id: config | |
| run: | | |
| echo 'is_fork=${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) != github.repository)) }}' >> $GITHUB_OUTPUT | |
| echo 'should_deploy=${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT | |
| echo 'is_release=${{ github.ref == 'refs/heads/release' }}' >> $GITHUB_OUTPUT | |
| - name: "DEBUG" | |
| run: | | |
| set +x | |
| echo is_fork=${{ steps.config.outputs.is_fork }} | |
| echo should_deploy=${{ steps.config.outputs.should_deploy }} | |
| echo is_release=${{ steps.config.outputs.is_release }} | |
| echo '\n' | |
| echo 'github.ref =' ${{ github.ref }} | |
| echo 'github.event_name =' ${{ github.event_name }} | |
| echo 'github.repository =' ${{ github.repository }} | |
| echo '\n' | |
| echo 'github.event.pull_request.head.repo.fork =' ${{ github.event.pull_request.head.repo.fork }} | |
| echo 'github.event.pull_request.head.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name }} | |
| echo '\n' | |
| echo 'github.event.pull_request.base.repo.fork =' ${{ github.event.pull_request.base.repo.fork }} | |
| echo 'github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.base.repo.full_name }} | |
| echo '\n' | |
| echo 'github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name =' ${{ github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name }} | |
| echo '(github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository =' ${{ (github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository }} | |
| echo 'github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) =' ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork || ((github.event.pull_request.head.repo.full_name || github.event.pull_request.base.repo.full_name) == github.repository)) }} | |
| - name: "Checkout code (fork)" | |
| if: ${{ steps.config.outputs.is_fork == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags. | |
| fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0. | |
| ref: ${{github.event.pull_request.head.ref}} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - name: "Checkout code" | |
| if: ${{ steps.config.outputs.is_fork != 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Number of commits to fetch. 0 indicates all history for all branches and tags. | |
| fetch-tags: true # Whether to fetch tags, even if fetch-depth > 0. | |
| ref: ${{ github.head_ref || github.ref }} | |
| - name: "Install system dependencies" | |
| run: | | |
| sudo apt-get install brotli pigz parallel python3 rename --yes -qq >/dev/null | |
| - name: "Install Python dependencies" | |
| run: | | |
| pip3 install -r requirements.txt | |
| - name: "Rebuild datasets (fork)" | |
| if: ${{ steps.config.outputs.is_fork == 'true' }} | |
| run: | | |
| ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --no-pull | |
| - name: "Rebuild, commit and push datasets (non-release branch)" | |
| if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release != 'true' }} | |
| run: | | |
| git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}" | |
| git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}" | |
| ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --push --repo="${GITHUB_REPOSITORY}" | |
| - name: "Rebuild, commit, push and make a release (release branch)" | |
| if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.is_release == 'true' }} | |
| run: | | |
| git config --global user.email "${{ secrets.BOT_GIT_USER_EMAIL }}" | |
| git config --global user.name "${{ secrets.BOT_GIT_USER_NAME }}" | |
| ./scripts/rebuild --input-dir 'data/' --output-dir 'data_output/' --release --repo="${GITHUB_REPOSITORY}" | |
| - name: "Install awscli" | |
| if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }} | |
| run: | | |
| pushd /tmp >/dev/null | |
| curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -oqq awscliv2.zip | |
| sudo ./aws/install --update | |
| popd >/dev/null | |
| aws --version | |
| - name: "Deploy dataset server" | |
| if: ${{ steps.config.outputs.is_fork != 'true' && steps.config.outputs.should_deploy == 'true' }} | |
| env: | |
| DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }} | |
| DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-2 | |
| run: | | |
| ./scripts/upload 'data_output/' | |
| - name: "Upload build artifacts: dataset server" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: server | |
| path: ./data_output/* | |
| - name: "Upload build artifacts: zip archives" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: zips | |
| path: ./data_temp/* |