Force numpy version #125
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deployment | |
| on: | |
| push: | |
| branches: | |
| - staging | |
| - main | |
| jobs: | |
| upload-prefect-deployments: | |
| name: Upload Prefect deployments | |
| environment: ${{ github.ref_name }} | |
| runs-on: ubuntu-latest | |
| env: | |
| PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} | |
| PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }} | |
| steps: | |
| - name: Set staging environment | |
| if: ${{ github.ref_name == 'staging' }} | |
| run: | | |
| echo "FLOW_ENV=dev" >> $GITHUB_ENV | |
| - name: Set production environment | |
| if: ${{ github.ref_name == 'main' }} | |
| run: | | |
| echo "FLOW_ENV=prod" >> $GITHUB_ENV | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| - name: Setup python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: '3.11' | |
| - name: Install python packages | |
| uses: BSFishy/pip-action@v1 | |
| with: | |
| requirements: requirements.txt | |
| - run: scripts/flow_prefect_deployment.sh clean_up_old_artifacts --interval 86400 --anchor-date 2023-05-18T00:00:00.000000+02:00 | |
| - run: scripts/flow_prefect_deployment.sh build_all --interval 86400 --anchor-date 2023-05-18T15:15:00.000000+02:00 | |
| - run: scripts/flow_prefect_deployment.sh build_frequent --interval 3600 --anchor-date 2023-05-18T11:45:00.000000+02:00 | |
| build-and-upload-agent-image: | |
| name: Build Docker image | |
| environment: ${{ github.ref_name }} | |
| runs-on: ubuntu-latest | |
| needs: upload-prefect-deployments | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} | |
| - name: Log in to DO Container Registry | |
| run: doctl registry login --expiry-seconds 600 | |
| - name: Build and push | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: ./agent/image/Dockerfile | |
| push: true | |
| tags: registry.digitalocean.com/${{ secrets.DIGITALOCEAN_CONTAINER_REPO }}/data-pipelines-agent:${{ github.sha }} | |
| cache-from: type=gha | |
| cache-to: type=gha | |
| target: prod | |
| deploy-agent-k8s: | |
| name: Deploy to Kubernetes | |
| environment: ${{ github.ref_name }} | |
| runs-on: ubuntu-latest | |
| needs: build-and-upload-agent-image | |
| env: | |
| PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }} | |
| PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }} | |
| GRAPH_API_KEY: ${{ secrets.GRAPH_API_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.DIGITALOCEAN_SPACES_API_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SPACES_API_SECRET }} | |
| WEB3_INFURA_PROJECT_ID: ${{ secrets.WEB3_INFURA_PROJECT_ID }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| DOCKER_IMAGE: registry.digitalocean.com/${{ secrets.DIGITALOCEAN_CONTAINER_REPO }}/data-pipelines-agent:${{ github.sha }} | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v2 | |
| - name: Set staging environment | |
| if: ${{ github.ref_name == 'staging' }} | |
| run: | | |
| echo "PREFECT_POOL=dev-agent-pool" >> $GITHUB_ENV | |
| mv agent/k8s/kustomization_dev.yaml agent/k8s/kustomization.yaml | |
| - name: Set production environment | |
| if: ${{ github.ref_name == 'main' }} | |
| run: | | |
| echo "PREFECT_POOL=prod-agent-pool" >> $GITHUB_ENV | |
| mv agent/k8s/kustomization_prod.yaml agent/k8s/kustomization.yaml | |
| # TODO update doctl to use 2fcbadcb46efd8dca0d93fbef2a3a8394f5981ba with the error code fix | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} | |
| # k8s Deployment | |
| - name: Setup Kustomize | |
| uses: imranismail/setup-kustomize@v1 | |
| with: | |
| kustomize-version: "4.5.4" | |
| - name: Update deployment file with image | |
| run: sed -i 's|<IMAGE>|'${DOCKER_IMAGE}'|' $GITHUB_WORKSPACE/agent/k8s/base/deployment.yaml | |
| - name: Write secrets file | |
| run: $GITHUB_WORKSPACE/scripts/properties_from_env.sh PREFECT_API_KEY PREFECT_API_URL AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID PREFECT_POOL WEB3_INFURA_PROJECT_ID GRAPH_API_KEY > $GITHUB_WORKSPACE/agent/k8s/secret.properties | |
| - name: Save DigitalOcean kubeconfig with short-lived credentials | |
| run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.DIGITALOCEAN_CLUSTER_NAME }} | |
| - name: Deploy to DigitalOcean Kubernetes | |
| run: kubectl apply -k agent/k8s | |
| - name: Verify deployments | |
| run: deploy=$(kubectl get deploy -n data-pipelines-agent -o name); for i in $deploy; do kubectl -n data-pipelines-agent rollout status $i; done |