Skip to content
Run ESA Challenge

Run ESA Challenge #118

Sign in to view logs

Run ESA Challenge

Run ESA Challenge #118

Workflow file for this run

.github/workflows/workflow-esa-challenge.yaml at 5f65daf
name: Run ESA Challenge
on:
schedule:
# every 16th day of the month at 11am
- cron: '00 11 16 * *'
# every last week of the month at 1pm
- cron: '00 13 25-31 * *'
pull_request:
workflow_dispatch:
jobs:
run-data-retrieval:
runs-on: ubuntu-latest
steps:
- name: Checkout out repository
uses: actions/checkout@v3
with:
token: ${{ secrets.TOKEN_CI }}
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip' # caching pip dependencies
- name: Install python Dependencies
run: pip install -r requirements.txt
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Run Pipeline
run: |
targets::tar_make()
shell: Rscript {0}
env:
TAR_PROJECT: data
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Save pipeline data in s3
run: |
targets::tar_source(files = "R")
put_dir_s3(local_dir = "store_data/",
s3_dir = "2024/targets-data/store_data/",
bucket = "projet-esa-nowcasting")
shell: Rscript {0}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
run-gas-challenge:
needs: [run-data-retrieval]
runs-on: ubuntu-latest
steps:
- name: Checkout out repository
uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.TOKEN_CI }}
- name: Configure Git
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
- name: Merge recent changes from main branch
run: |
git fetch origin
git checkout -b gh-submissions origin/gh-submissions
git merge main
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip' # caching pip dependencies
- name: Install python Dependencies
run: pip install -r requirements.txt
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Run Pipeline
run: |
targets::tar_make()
shell: Rscript {0}
env:
TAR_PROJECT: gas
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Commit and push submissions
run: |
git add Submissions/
git commit -m "New submission GAS with Github Actions"
git fetch origin
git merge origin/gh-submissions
git push origin gh-submissions --set-upstream
- name: Save pipeline data in s3
run: |
targets::tar_source(files = "R")
put_dir_s3(local_dir = "store_gas/",
s3_dir = "2024/targets-data/store_gas/",
bucket = "projet-esa-nowcasting")
shell: Rscript {0}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
run-oil-challenge:
needs: [run-data-retrieval]
runs-on: ubuntu-latest
steps:
- name: Checkout out repository
uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.TOKEN_CI }}
- name: Configure Git
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
- name: Merge recent changes from main branch
run: |
git fetch origin
git checkout -b gh-submissions origin/gh-submissions
git merge main
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip' # caching pip dependencies
- name: Install python Dependencies
run: pip install -r requirements.txt
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Run Pipeline
run: |
targets::tar_make()
shell: Rscript {0}
env:
TAR_PROJECT: oil
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Commit and push submissions
run: |
git add Submissions/
git commit -m "New submission OIL with Github Actions"
git fetch origin
git merge origin/gh-submissions
git push origin gh-submissions --set-upstream
- name: Save pipeline data in s3
run: |
targets::tar_source(files = "R")
put_dir_s3(local_dir = "store_oil/",
s3_dir = "2024/targets-data/store_oil/",
bucket = "projet-esa-nowcasting")
shell: Rscript {0}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
run-electricity-challenge:
needs: [run-data-retrieval]
runs-on: ubuntu-latest
steps:
- name: Checkout out repository
uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.TOKEN_CI }}
- name: Configure Git
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
- name: Merge recent changes from main branch
run: |
git fetch origin
git checkout -b gh-submissions origin/gh-submissions
git merge main
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip' # caching pip dependencies
- name: Install python Dependencies
run: pip install -r requirements.txt
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Run Pipeline
run: |
targets::tar_make()
shell: Rscript {0}
env:
TAR_PROJECT: electricity
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Commit and push submissions
run: |
git add Submissions/
git commit -m "New submission ELECTRICITY with Github Actions"
git fetch origin
git merge origin/gh-submissions
git push origin gh-submissions --set-upstream
- name: Save pipeline data in s3
run: |
targets::tar_source(files = "R")
put_dir_s3(local_dir = "store_electricity/",
s3_dir = "2024/targets-data/store_electricity/",
bucket = "projet-esa-nowcasting")
shell: Rscript {0}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
run-post-mortem:
needs: [run-gas-challenge, run-oil-challenge, run-electricity-challenge]
runs-on: ubuntu-latest
steps:
- name: Checkout out repository
uses: actions/checkout@v3
with:
token: ${{ secrets.TOKEN_CI }}
- name: Merge submissions on main
run: |
git fetch origin
git merge origin/gh-submissions
git push origin main
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.10'
cache: 'pip' # caching pip dependencies
- name: Install python Dependencies
run: pip install -r requirements.txt
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Run Pipeline
run: |
targets::tar_make()
shell: Rscript {0}
env:
TAR_PROJECT: post_mortem
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Save pipeline data in s3
run: |
targets::tar_source(files = "R")
put_dir_s3(local_dir = "store_post_mortem/",
s3_dir = "2024/targets-data/store_post_mortem/",
bucket = "projet-esa-nowcasting")
shell: Rscript {0}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
publish-website:
needs: [run-post-mortem]
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Set up Quarto
uses: quarto-dev/quarto-actions/setup@v2
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
r-version: '4.2.2'
- name: Reconfigure Java support
run: sudo R CMD javareconf
- name: Install libcurl4
run: sudo apt install libcurl4-openssl-dev
- name: Install R Dependencies
uses: r-lib/actions/setup-renv@v2
with:
cache-version: 1
- name: Install Minio Client
run: |
wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc
chmod +x /usr/local/bin/mc
export
- name: Retrieve stores from s3
run: |
export MC_HOST_s3=https://$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY@$AWS_S3_ENDPOINT
mc cp -r s3/projet-esa-nowcasting/2024/targets-data/ .
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
- name: Render and Publish
uses: quarto-dev/quarto-actions/publish@v2
with:
target: gh-pages
path: website/
env:
GITHUB_TOKEN: ${{ secrets.TOKEN_CI }}