Transform R notebooks to Rmd; Add checks for R notebooks; Small mod to checks in Python notebooks #39
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check R Notebooks | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: '0 12 * * 0' # Runs every Sunday at 12 PM UTC | |
| concurrency: | |
| group: test-R-notebooks-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-R-notebooks: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| directory: ['PM1', 'PM2', 'PM3', 'PM4', 'PM5', 'CM1', 'CM2', 'CM3', 'AC1', 'AC2', 'T'] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} | |
| - name: Find changed notebooks in PR | |
| if: github.event_name == 'pull_request' | |
| id: find_notebooks_pr | |
| run: | | |
| # git fetch origin ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }} | |
| # git diff --name-only origin/${{ github.event.pull_request.base.ref }}...origin/${{ github.event.pull_request.head.ref }} > changed_files.txt | |
| git diff --name-only -r HEAD^1 HEAD > changed_files.txt | |
| if grep -q -E '^${{ matrix.directory }}/.*\.Rmd$' changed_files.txt; then | |
| echo "Changing directly the .Rmd files is prohibited. You should only be changing the .irnb files" | |
| echo "The .Rmd files will be automatically generated and updated when the PR is merged in the main branch" | |
| echo "It seems that you changed directly the following files:" | |
| grep -E '^${{ matrix.directory }}/.*\.Rmd$' changed_files.txt | |
| exit 1 | |
| fi | |
| grep -E '^${{ matrix.directory }}/.*\.irnb$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt | |
| - name: Find changed notebooks in Push | |
| if: github.event_name == 'push' | |
| id: find_notebooks_push | |
| run: | | |
| git diff --name-only ${{ github.event.before }} ${{ github.event.after }} > changed_files.txt | |
| grep -E '^${{ matrix.directory }}/.*\.irnb$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt | |
| - name: Check if any notebooks changed in PR or Push | |
| if: (github.event_name == 'push') || (github.event_name == 'pull_request') | |
| id: check_notebooks | |
| run: | | |
| cat changed_notebooks.txt | |
| if grep -q -E '^${{ matrix.directory }}/.*\.irnb$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_notebooks.txt; then | |
| echo "notebooks_changed=true" >> $GITHUB_ENV | |
| else | |
| echo "notebooks_changed=false" >> $GITHUB_ENV | |
| echo "No R notebooks changed in folder ${{ matrix.directory }} in this PR." | |
| fi | |
| - name: Set notebooks changed to true for schedule | |
| if: "! ((github.event_name == 'push') || (github.event_name == 'pull_request'))" | |
| id: set_check_notebooks_true | |
| run: | | |
| # we run all folders if it is the weekly scheduled run to | |
| # check if something broke due to changes in dependencies | |
| echo "notebooks_changed=true" >> $GITHUB_ENV | |
| - name: Install system dependencies | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libcurl4-openssl-dev | |
| - name: Set up Python | |
| if: env.notebooks_changed == 'true' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' # Specify your Python version here | |
| - name: Install Python dependencies | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install nbstripout | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| shell: bash | |
| - name: Set up R | |
| if: env.notebooks_changed == 'true' | |
| uses: r-lib/actions/setup-r@v2 | |
| - name: Install rmarkdown, knitr, and lintr packages | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| R -e 'install.packages(c("rmarkdown", "knitr", "lintr", "xfun", "remotes"), repos="https://cloud.r-project.org")' | |
| - name: Strip outputs from .irnb files | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| for notebook in ${{ matrix.directory }}/*.irnb; do | |
| ipynb_notebook="${notebook%.irnb}.ipynb" | |
| mv "$notebook" "$ipynb_notebook" | |
| nbstripout "$ipynb_notebook" | |
| mv "$ipynb_notebook" "$notebook" | |
| done | |
| - name: Convert .irnb to .Rmd and .R | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| R -e ' | |
| files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.irnb$", full.names = TRUE, recursive = FALSE) | |
| lapply(files, function(input) { | |
| rmarkdown::convert_ipynb(input) | |
| rmd_file <- xfun::with_ext(input, "Rmd") | |
| knitr::purl(rmd_file, output = xfun::with_ext(input, "R")) | |
| }) | |
| ' | |
| - name: Lint .Rmd files | |
| if: env.notebooks_changed == 'true' | |
| id: lint | |
| run: | | |
| R -e ' | |
| library(lintr) | |
| linters <- with_defaults(line_length_linter = line_length_linter(120), | |
| object_name_linter = object_name_linter(styles = c("snake_case", "CamelCase", "camelCase")), | |
| object_usage_linter = NULL) | |
| rmd_files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.Rmd$", full.names = TRUE) | |
| results <- lapply(rmd_files, function(file) { | |
| lints <- lint(file, linters) | |
| if (length(lints) > 0) { | |
| cat("Warnings found during linting:\n") | |
| print(lints) | |
| stop("Linting failed with warnings") | |
| } | |
| }) | |
| ' | |
| - name: Execute R scripts and log output | |
| if: env.notebooks_changed == 'true' | |
| id: execute | |
| run: | | |
| log_file="${{ matrix.directory }}_r_script_execution.log" | |
| R -e ' | |
| options(show.error.locations = TRUE) | |
| files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.R$", full.names = TRUE, recursive = FALSE) | |
| log_con <- file("'$log_file'", open = "wt") | |
| sink(log_con, type = "output") | |
| sink(log_con, type = "message") | |
| errors <- list() | |
| for (gitrfile in files) { | |
| withCallingHandlers( | |
| withRestarts( | |
| source(gitrfile), | |
| muffleStop = function() NULL | |
| ), | |
| error = function(e) { | |
| traceback_info <- sys.calls() | |
| errors[[length(errors) + 1]] <<- list(gitrfile = gitrfile, location = capture.output(print(e$call)), message = e$message, traceback = traceback_info) | |
| invokeRestart("muffleStop") | |
| } | |
| ) | |
| } | |
| sink(type = "output") | |
| sink(type = "message") | |
| close(log_con) | |
| if (length(errors) > 0) { | |
| for (error in errors) { | |
| cat("Error found in file:", error$gitrfile, "\n") | |
| cat("at line::", error$location, "\n") | |
| cat("Error message:", error$message, "\n") | |
| print("Traceback:\n") | |
| cat(paste(error$traceback, collapse = "\n")) | |
| print("\n") | |
| } | |
| quit(status = 1, save = "no") # Exit with an error status if errors are found | |
| } | |
| ' 2>/dev/null | |
| env: | |
| GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload execution log | |
| if: env.notebooks_changed == 'true' | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: ${{ matrix.directory }}-r-script-execution-log | |
| path: ${{ matrix.directory }}_r_script_execution.log | |
| - name: Zip .R files | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| mkdir r_scripts | |
| mv ${{ matrix.directory }}/*.R r_scripts/ | |
| zip -r ${{ matrix.directory }}_r_scripts.zip r_scripts | |
| - name: Upload artifact | |
| if: env.notebooks_changed == 'true' | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: ${{ matrix.directory }}-r-scripts | |
| path: ${{ matrix.directory }}_r_scripts.zip | |
| - name: Delete .R files and zip | |
| if: env.notebooks_changed == 'true' | |
| run: | | |
| rm -rf r_scripts | |
| rm ${{ matrix.directory }}_r_scripts.zip | |
| # - name: Check out the branch for pull request | |
| # if: "(github.event_name == 'pull_request') && (env.notebooks_changed == 'true')" | |
| # run: | | |
| # git fetch --all | |
| # git checkout ${{ github.event.pull_request.head.ref }} | |
| # - name: Check if there are any changes | |
| # if: env.notebooks_changed == 'true' | |
| # id: verify_diff | |
| # run: | | |
| # git pull | |
| # git diff --quiet ${{ matrix.directory }}/*.irnb ${{ matrix.directory }}/*.Rmd || echo "changed=true" >> $GITHUB_OUTPUT | |
| # - name: Commit and push stripped .irnb and .Rmd files | |
| # if: "(env.notebooks_changed == 'true') && (steps.verify_diff.outputs.changed == 'true')" | |
| # run: | | |
| # git config --global user.name 'github-actions[bot]' | |
| # git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| # git pull | |
| # git add ${{ matrix.directory }}/*.irnb ${{ matrix.directory }}/*.Rmd | |
| # git commit -m 'Strip outputs from .irnb, convert to .Rmd, lint .Rmd files, and execute .R files in ${{ matrix.directory }} [skip ci]' | |
| # git push --force-with-lease | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |