-
Notifications
You must be signed in to change notification settings - Fork 41
220 lines (198 loc) · 8.77 KB
/
check-and-transform-R-notebooks.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
name: Check and Transform R Notebooks
on:
push:
pull_request:
branches:
- main
schedule:
- cron: '0 12 * * 0' # Runs every Sunday at 12 PM UTC
concurrency:
group: test-R-notebooks-${{ github.ref }}
cancel-in-progress: true
jobs:
test-R-notebooks:
runs-on: ubuntu-latest
strategy:
matrix:
directory: ['PM1', 'PM2', 'PM3', 'PM4', 'PM5', 'CM1', 'CM2', 'CM3', 'AC1', 'AC2', 'T']
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
- name: Find changed notebooks in PR
if: github.event_name == 'pull_request'
id: find_notebooks_pr
run: |
git fetch origin ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }}
git diff --name-only origin/${{ github.event.pull_request.base.ref }}...origin/${{ github.event.pull_request.head.ref }} > changed_files.txt
grep -E '\.irnb$|\.Rmd$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
- name: Find changed notebooks in Push
if: github.event_name == 'push'
id: find_notebooks_push
run: |
git diff --name-only ${{ github.event.before }} ${{ github.event.after }} > changed_files.txt
grep -E '\.irnb$|\.Rmd$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
- name: Check if any notebooks changed in PR or Push
if: (github.event_name == 'push') || (github.event_name == 'pull_request')
id: check_notebooks
run: |
cat changed_notebooks.txt
if grep -q -E '^${{ matrix.directory }}/.*\.irnb$|^${{ matrix.directory }}/.*\.Rmd$|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_notebooks.txt; then
echo "notebooks_changed=true" >> $GITHUB_ENV
else
echo "notebooks_changed=false" >> $GITHUB_ENV
echo "No R notebooks changed in folder ${{ matrix.directory }} in this PR."
fi
- name: Set notebooks changed to true for schedule
if: "! ((github.event_name == 'push') || (github.event_name == 'pull_request'))"
id: set_check_notebooks_true
run: |
# we run all folders if it is the weekly scheduled run to
# check if something broke due to changes in dependencies
echo "notebooks_changed=true" >> $GITHUB_ENV
- name: Install system dependencies
if: env.notebooks_changed == 'true'
run: |
sudo apt-get update
sudo apt-get install -y libcurl4-openssl-dev
- name: Set up Python
if: env.notebooks_changed == 'true'
uses: actions/setup-python@v2
with:
python-version: '3.10' # Specify your Python version here
- name: Install Python dependencies
if: env.notebooks_changed == 'true'
run: |
python -m pip install --upgrade pip
pip install nbstripout
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
shell: bash
- name: Set up R
if: env.notebooks_changed == 'true'
uses: r-lib/actions/setup-r@v2
- name: Install rmarkdown, knitr, and lintr packages
if: env.notebooks_changed == 'true'
run: |
R -e 'install.packages(c("rmarkdown", "knitr", "lintr", "xfun", "remotes"), repos="https://cloud.r-project.org")'
- name: Strip outputs from .irnb files
if: env.notebooks_changed == 'true'
run: |
for notebook in ${{ matrix.directory }}/*.irnb; do
ipynb_notebook="${notebook%.irnb}.ipynb"
mv "$notebook" "$ipynb_notebook"
nbstripout "$ipynb_notebook"
mv "$ipynb_notebook" "$notebook"
done
- name: Convert .irnb to .Rmd and .R
if: env.notebooks_changed == 'true'
run: |
R -e '
files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.irnb$", full.names = TRUE, recursive = FALSE)
lapply(files, function(input) {
rmarkdown::convert_ipynb(input)
rmd_file <- xfun::with_ext(input, "Rmd")
knitr::purl(rmd_file, output = xfun::with_ext(input, "R"))
})
'
- name: Lint .Rmd files
if: env.notebooks_changed == 'true'
id: lint
run: |
R -e '
library(lintr)
linters <- with_defaults(line_length_linter = line_length_linter(120),
object_name_linter = object_name_linter(styles = c("snake_case", "CamelCase", "camelCase")),
object_usage_linter = NULL)
rmd_files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.Rmd$", full.names = TRUE)
results <- lapply(rmd_files, function(file) {
lints <- lint(file, linters)
if (length(lints) > 0) {
cat("Warnings found during linting:\n")
print(lints)
stop("Linting failed with warnings")
}
})
'
- name: Execute R scripts and log output
if: env.notebooks_changed == 'true'
id: execute
run: |
log_file="${{ matrix.directory }}_r_script_execution.log"
R -e '
options(show.error.locations = TRUE)
files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.R$", full.names = TRUE, recursive = FALSE)
log_con <- file("'$log_file'", open = "wt")
sink(log_con, type = "output")
sink(log_con, type = "message")
errors <- list()
for (gitrfile in files) {
withCallingHandlers(
withRestarts(
source(gitrfile),
muffleStop = function() NULL
),
error = function(e) {
traceback_info <- sys.calls()
errors[[length(errors) + 1]] <<- list(gitrfile = gitrfile, location = capture.output(print(e$call)), message = e$message, traceback = traceback_info)
invokeRestart("muffleStop")
}
)
}
sink(type = "output")
sink(type = "message")
close(log_con)
if (length(errors) > 0) {
for (error in errors) {
cat("Error found in file:", error$gitrfile, "\n")
cat("at line::", error$location, "\n")
cat("Error message:", error$message, "\n")
print("Traceback:\n")
cat(paste(error$traceback, collapse = "\n"))
print("\n")
}
quit(status = 1, save = "no") # Exit with an error status if errors are found
}
' 2>/dev/null
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
- name: Upload execution log
if: env.notebooks_changed == 'true'
uses: actions/upload-artifact@v2
with:
name: ${{ matrix.directory }}-r-script-execution-log
path: ${{ matrix.directory }}_r_script_execution.log
- name: Zip .R files
if: env.notebooks_changed == 'true'
run: |
mkdir r_scripts
mv ${{ matrix.directory }}/*.R r_scripts/
zip -r ${{ matrix.directory }}_r_scripts.zip r_scripts
- name: Upload artifact
if: env.notebooks_changed == 'true'
uses: actions/upload-artifact@v2
with:
name: ${{ matrix.directory }}-r-scripts
path: ${{ matrix.directory }}_r_scripts.zip
- name: Delete .R files and zip
if: env.notebooks_changed == 'true'
run: |
rm -rf r_scripts
rm ${{ matrix.directory }}_r_scripts.zip
- name: Check if there are any changes
if: env.notebooks_changed == 'true'
id: verify_diff
run: |
git pull
git diff --quiet ${{ matrix.directory }}/*.irnb ${{ matrix.directory }}/*.Rmd || echo "changed=true" >> $GITHUB_OUTPUT
- name: Commit and push stripped .irnb and .Rmd files
if: "(env.notebooks_changed == 'true') && (steps.verify_diff.outputs.changed == 'true')"
run: |
git config --global user.name 'github-actions[bot]'
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
git pull
git add ${{ matrix.directory }}/*.irnb ${{ matrix.directory }}/*.Rmd
git commit -m 'Strip outputs from .irnb, convert to .Rmd, lint .Rmd files, and execute .R files in ${{ matrix.directory }}'
git push
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}