data-curation.yml

on:
  push:
    branches:
      - master
      - staging
      - release
  workflow_dispatch:

jobs:
  Deploy-Datasets:
    runs-on: ubuntu-20.04

    environment:
      name: ${{ github.ref }}

    env:
      VERBOSE: 0
      PYTHONUNBUFFERED: 1
      DATA_AWS_S3_BUCKET: ${{ secrets.DATA_AWS_S3_BUCKET }}
      DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.DATA_AWS_CLOUDFRONT_DISTRIBUTION_ID }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: us-east-2

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1

      - name: install dependencies
        run: |
          sudo apt-get install brotli pigz parallel rename --yes -qq >/dev/null

      - name: install awscli
        run: |
          set -x
          pushd /tmp >/dev/null
            curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip -oqq awscliv2.zip
            sudo ./aws/install --update
          popd >/dev/null
          aws --version

      - name: rebuild
        run: |
          set -x
          ./scripts/rebuild

      - name: download
        run: |
          set -x
          ./scripts/download

      - name: compare
        run: |
          set -x
          ./scripts/compare

      - name: upload
        run: |
          set -x
          ./scripts/upload

      - name: create git tag
        if: github.ref == 'refs/heads/release'
        run: |
          set -x
          git tag $(TZ=UTC date "+%Y-%m-%d--%H-%M-%S--%Z")
          git push --tags