forked from mala-project/mala
-
Notifications
You must be signed in to change notification settings - Fork 1
280 lines (229 loc) · 10.6 KB
/
cpu-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
name: CPU tests
on:
workflow_dispatch:
pull_request:
# Trigger on pull requests to master or develop that are
# marked as "ready for review" (non-draft PRs)
types:
- opened
- synchronize
- reopened
- ready_for_review
branches:
- master
- develop
push:
# Trigger on pushes to master or develop and for git tag pushes
branches:
- master
- develop
tags:
- v*
env:
IMAGE_NAME: mala_conda_cpu
IMAGE_REGISTRY: ghcr.io
DOCKER_CACHE_PATH: cache/docker
jobs:
build-docker-image-cpu:
# do not trigger on draft PRs
if: ${{ ! github.event.pull_request.draft }}
# Build and push temporary Docker image to GitHub's container registry
runs-on: ubuntu-22.04
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
fetch-depth: '1'
- name: Set environment variables
run: |
# Change all uppercase letters to lowercase
IMAGE_REPO=$IMAGE_REGISTRY/$(echo $GITHUB_REPOSITORY_OWNER | tr '[A-Z]' '[a-z]')
# Create environment variable to which any subsequent steps inside this workflow's job have access
echo "IMAGE_REPO=$IMAGE_REPO" >> $GITHUB_ENV
echo "IMAGE_REPO=$IMAGE_REPO"
- name: Restore cache
uses: actions/cache@v4
id: cache-docker
with:
path: ${{ env.DOCKER_CACHE_PATH }}
key: ${{ github.run_id }}
- name: Check existence of Docker tar archive
id: check_file
run: |
if [[ -f "$DOCKER_CACHE_PATH/docker-image.tar.gz" ]]; then
echo "FILE_EXISTS=true" >> $GITHUB_OUTPUT
else
echo "FILE_EXISTS=false" >> $GITHUB_OUTPUT
fi
- name: Pull latest image from container registry
run: docker pull $IMAGE_REPO/$IMAGE_NAME --quiet || true
- name: Build temporary Docker image
run: |
if [[ "${{ steps.check_file.outputs.FILE_EXISTS }}" == 'true' ]]
then
docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz
CACHE=$IMAGE_NAME:$GITHUB_RUN_ID
else
CACHE=$IMAGE_REPO/$IMAGE_NAME:latest
fi
DOCKER_BUILDKIT=0 docker build . --file Dockerfile --tag $IMAGE_NAME:local --cache-from=$CACHE --build-arg DEVICE=cpu
# Show images
docker images --filter=reference=$IMAGE_NAME --filter=reference=$IMAGE_REPO/$IMAGE_NAME
# Get image IDs (hash of the local image JSON configuration) and check if images are equal
IMAGE_ID_OLD=$(docker images --format "{{.ID}}" --no-trunc --filter=reference=$IMAGE_REPO/$IMAGE_NAME:latest)
IMAGE_ID_NEW=$(docker images --format "{{.ID}}" --no-trunc --filter=reference=$IMAGE_NAME:local)
echo "IMAGE_ID_OLD=$IMAGE_ID_OLD" >> $GITHUB_ENV
echo "IMAGE_ID_NEW=$IMAGE_ID_NEW" >> $GITHUB_ENV
if [[ "$IMAGE_ID_OLD" == "$IMAGE_ID_NEW" ]]
then
echo "Image IDs are equal"; DOCKER_TAG=latest
else
echo "Image IDs are different"; DOCKER_TAG=$GITHUB_RUN_ID
fi
# Environment variable DOCKER_TAG references the image in subsequent jobs
echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
- name: Tag and save temporary built Docker image
# If temporary image is different from latest on ghcr.io
if: env.IMAGE_ID_OLD != env.IMAGE_ID_NEW
run: |
mkdir -p $DOCKER_CACHE_PATH
# Use GITHUB_RUN_ID, a unique number for each workflow run within a repository as a temporary Docker tag
docker tag $IMAGE_NAME:local $IMAGE_NAME:$GITHUB_RUN_ID
# Save Docker image locally
docker save $IMAGE_NAME:$GITHUB_RUN_ID -o $DOCKER_CACHE_PATH/docker-image.tar
pigz -f -v --fast $DOCKER_CACHE_PATH/docker-image.tar
outputs:
# Make variables available to all downstream jobs that depend on this job
image-repo: ${{ env.IMAGE_REPO }}
docker-tag: ${{ env.DOCKER_TAG }}
cpu-tests:
needs: build-docker-image-cpu
runs-on: ubuntu-20.04
env:
IMAGE_REPO: ${{ needs.build-docker-image-cpu.outputs.image-repo }}
DOCKER_TAG: ${{ needs.build-docker-image-cpu.outputs.docker-tag }}
steps:
- name: "Prepare environment: Restore cache"
if: env.DOCKER_TAG != 'latest'
uses: actions/cache@v4
id: cache-docker
with:
path: ${{ env.DOCKER_CACHE_PATH }}
key: ${{ github.run_id }}
- name: "Prepare environment: Load Docker image from cache"
if: env.DOCKER_TAG != 'latest'
run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz --quiet
- name: "Prepare environment: Pull latest image from container registry"
if: env.DOCKER_TAG == 'latest'
run: |
docker pull $IMAGE_REPO/$IMAGE_NAME:latest --quiet
docker image tag $IMAGE_REPO/$IMAGE_NAME:latest $IMAGE_NAME:latest
- name: "Prepare environment: Run Docker container"
run: |
# Make the github workspace (i.e. checked out mala repository) available inside Docker container by binding it to /home via -v
docker run -i -d --rm --name mala-cpu -v ${{ github.workspace }}:/home -w /home $IMAGE_NAME:$DOCKER_TAG /bin/bash &
# Wait for Docker container to come online
wait
# Check running docker container
docker ps
# Check if docker container is running
[[ $(docker inspect --format '{{json .State.Running}}' mala-cpu) == 'true' ]]
- name: Check out repository (mala)
uses: actions/checkout@v4
with:
fetch-depth: '1'
- name: Install mala package
# Exec all commands inside the mala-cpu container
shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
run: |
# export Docker image Conda environment for a later comparison
conda env export -n mala-cpu > env_before.yml
# install mala package
pip --no-cache-dir install -e .[opt,test] --no-build-isolation
- name: Check if Conda environment meets the specified requirements
shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
run: |
# export Conda environment _with_ mala package installed in it (and extra dependencies)
conda env export -n mala-cpu > env_after.yml
# if comparison fails, `install/mala_cpu_[base]_environment.yml` needs to be aligned with
# `requirements.txt` and/or extra dependencies are missing in the Docker Conda environment
if diff --brief env_before.yml env_after.yml
then
echo "Files env_before.yml and env_after.yml do not differ."
else
diff --side-by-side --color-always env_before.yml env_after.yml
fi
- name: Download test data repository from RODARE
shell: 'bash -c "docker exec -i mala-cpu python < {0}"'
run: |
import requests, shutil, zipfile
# This DOI represents all versions, and will always resolve to the latest one
DOI = "https://doi.org/10.14278/rodare.2900"
# Resolve DOI and get record ID and the associated API URL
response = requests.get(DOI)
*_, record_id = response.url.split("/")
api_url = f"https://rodare.hzdr.de/api/records/{record_id}"
# Download record from API and get the first file
response = requests.get(api_url)
record = response.json()
size = record["files"][0]["size"]
download_link = record["files"][0]["links"]["self"]
print(size, "bytes", "--", download_link)
# TODO: implement some sort of auto retry for failed HTTP requests
response = requests.get(download_link)
# Saving downloaded content to a file
with open("test-data.zip", mode="wb") as file:
file.write(response.content)
# Get top level directory name
dir_name = zipfile.ZipFile("test-data.zip").namelist()[0]
shutil.unpack_archive("test-data.zip", ".")
print(f"Rename {dir_name} to mala_data")
shutil.move(dir_name, "mala_data")
- name: Test mala
shell: 'bash -c "docker exec -i mala-cpu bash < {0}"'
run: MALA_DATA_REPO=$(pwd)/mala_data pytest --cov=mala --cov-fail-under=60 -m "not examples" --disable-warnings
retag-docker-image-cpu:
needs: [cpu-tests, build-docker-image-cpu]
runs-on: ubuntu-22.04
permissions:
packages: write
env:
IMAGE_REPO: ${{ needs.build-docker-image-cpu.outputs.image-repo }}
DOCKER_TAG: ${{ needs.build-docker-image-cpu.outputs.docker-tag }}
# Trigger on pushes to master or develop (this includes _merged_ PRs) only if Docker image has changed and on git tag pushes
# NOTE: The `env` context is not available for workflow key `jobs.<job_id>.if`.
if: |
((contains(github.ref_name, 'develop') || contains(github.ref_name, 'master')) && needs.build-docker-image-cpu.outputs.docker-tag != 'latest')
|| startsWith(github.ref, 'refs/tags/')
steps:
- name: "Prepare environment: Restore cache"
if: env.DOCKER_TAG != 'latest'
uses: actions/cache@v4
id: cache-docker
with:
path: ${{ env.DOCKER_CACHE_PATH }}
key: ${{ github.run_id }}
- name: "Prepare environment: Load Docker image from cache"
if: env.DOCKER_TAG != 'latest'
run: docker load -i $DOCKER_CACHE_PATH/docker-image.tar.gz --quiet
- name: "Prepare environment: Pull latest image from container registry"
if: env.DOCKER_TAG == 'latest'
run: docker pull $IMAGE_REPO/$IMAGE_NAME:latest --quiet
- name: Tag Docker image
run: |
# Execute on change of Docker image
if [[ "$DOCKER_TAG" != 'latest' ]]; then
docker tag $IMAGE_NAME:$GITHUB_RUN_ID $IMAGE_REPO/$IMAGE_NAME:latest
docker tag $IMAGE_NAME:$GITHUB_RUN_ID $IMAGE_REPO/$IMAGE_NAME:${GITHUB_REF_NAME}-${GITHUB_SHA:0:7}
fi
# Execute on push of git tag
if ${{ startsWith(github.ref, 'refs/tags/') }}; then
GIT_TAG=$(echo "${{ github.ref_name }}" | sed -e 's/^v//')
echo "GIT_TAG=$GIT_TAG"
docker tag $IMAGE_REPO/$IMAGE_NAME:latest $IMAGE_REPO/$IMAGE_NAME:$GIT_TAG
fi
- name: Log into container registry
# Authentication required for pushing images
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push Docker image
run: docker push $IMAGE_REPO/$IMAGE_NAME --all-tags | grep -v -E 'Waiting|Layer already|Preparing|Pushed'