forked from OpenRefine/OpenRefine
-
Notifications
You must be signed in to change notification settings - Fork 0
404 lines (347 loc) · 15.9 KB
/
snapshot_release.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
name: Snapshot release
on:
push:
branches:
- 'master'
- '4.0'
paths-ignore:
- 'docs/**'
release:
types: [created]
jobs:
prepare_e2e_test_matrix:
if: github.repository == 'OpenRefine/OpenRefine'
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
- id: set-matrix
run: npm install --save [email protected] && node main/tests/cypress/build-test-matrix.js >> $GITHUB_OUTPUT
env:
browsers: chrome
e2e_test:
if: github.repository == 'OpenRefine/OpenRefine'
name: e2e_test ${{ matrix.specs.group }}
needs: prepare_e2e_test_matrix
runs-on: ubuntu-latest
strategy:
matrix: ${{fromJSON(needs.prepare_e2e_test_matrix.outputs.matrix)}}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set up secrets
run: |
echo "CYPRESS_RECORD_KEY=$(echo YzE3ZDU4OGItZTBkOC00ZjJmLTg4NjYtNzJmNmFmYmRhNGQxCg== | base64 -d)" >> $GITHUB_ENV
echo "CYPRESS_PROJECT_ID=s5du3k" >> $GITHUB_ENV
- name: Set up Java 11
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 11
cache: 'maven'
- name: Build OpenRefine
run: ./refine build
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
- name: Restore Tests dependency cache
uses: actions/cache@v4
with:
path: |
~/cache
~/.cache
**/node_modules
!~/cache/exclude
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
npm i -g yarn
yarn install
- name: Test with Cypress on ${{ matrix.browser }}
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine e2e_tests
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
CYPRESS_RECORD_KEY: ${{ env.CYPRESS_RECORD_KEY }}
CYPRESS_PROJECT_ID: ${{ env.CYPRESS_PROJECT_ID }}
CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
CYPRESS_SPECS: ${{ matrix.specs.paths }}
CYPRESS_GROUP: ${{ matrix.specs.group }}
build:
if: github.repository == 'OpenRefine/OpenRefine'
services:
postgres:
image: postgres
ports:
- 5432
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: 'postgres'
POSTGRES_DB: test_db
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: mysql:8
ports:
- 3306
env:
MYSQL_ROOT_PASSWORD: root
options: >-
--health-cmd "mysqladmin ping"
--health-interval 5s
--health-timeout 2s
--health-retries 3
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # This is wasteful, but needed for git describe
- name: Set up secrets
run: |
echo "COVERALLS_TOKEN=$(echo eUVUVGRHOFJhQm9GMFJBYTNibjVhcWFEblpac1lmMlE3Cg== | base64 -d)" >> $GITHUB_ENV
- name: Set up Java 11
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 11
cache: 'maven'
server-id: ossrh
server-username: OSSRH_USER
server-password: OSSRH_PASS
- name: Install genisoimage and jq
run: sudo apt-get install genisoimage jq
- name: Configure connections to databases
id: configure_db_connections
run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
- name: Populate databases with test data
id: populate_databases_with_test_data
run: |
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
env:
PGPASSWORD: postgres
- name: Install webapp dependencies
run: cd main/webapp && npm install
- name: Build and test with Maven
run: mvn -B jacoco:prepare-agent test jacoco:report
- name: Coveralls
uses: coverallsapp/github-action@v2
with:
github-token: ${{ env.COVERALLS_TOKEN }}
format: jacoco
flag-name: Java ${{ matrix.java }}
pull-request: ${{ github.event.number }}
fail-on-error: false
- name: Install webapp dependencies
run: cd main/webapp && npm install
- name: Generate dist files
run: mvn -B package -DskipTests=true
mac_test_and_deploy:
if: github.repository == 'OpenRefine/OpenRefine'
runs-on: macos-12
outputs:
version_number_string: ${{ steps.version_string_variable.outputs.output_value }}
steps:
- uses: actions/checkout@v4
- name: Set up Java 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 17
cache: 'maven'
server-id: ossrh
server-username: OSSRH_USER
server-password: OSSRH_PASS
gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
gpg-passphrase: GPG_PASSPHRASE
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3'
- name: Install dmgbuild
run: pip install dmgbuild
- name: Decrypt Apple certificates
run: |
gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.cer --decrypt packaging/apple_certs/apple_cert.cer.enc
gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/apple_cert.p12 --decrypt packaging/apple_certs/apple_cert.p12.enc
.github/workflows/release/add_apple_keys.sh
env:
P12_PASSPHRASE: ${{ secrets.APPLE_P12_PASSPHRASE }}
- name: Apply patch to insert Google API credentials
run: |
gpg --pinentry-mode loopback --passphrase "${{ secrets.GPG_PASSPHRASE }}" --output .github/workflows/release/gdata.patch --decrypt .github/workflows/release/gdata.patch.asc
git am .github/workflows/release/gdata.patch
git reset HEAD^
- name: Install webapp dependencies
run: cd main/webapp && npm install
- name: Get release upload URL
id: get_release_upload_url
if: github.event_name == 'release'
uses: bruceadams/[email protected]
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Set version
if: github.event_name == 'release'
run: mvn versions:set -DnewVersion=$(echo '${{ steps.get_release_upload_url.outputs.tag_name }}' | sed 's/[^a-zA-Z0-9_\.\-]/_/g')
- name: Build and test
run: mvn -B test
env:
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
- name: Set version string output variable
id: version_string_variable
run: echo "output_value=$(mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)" >> "$GITHUB_OUTPUT"
- name: Package and upload to OSSRH
run: |
mvn -B deploy -DskipTests=true -DrepositoryId=ossrh -Dapple.notarization.username=$APPLE_USERNAME -Dapple.notarization.password=$APPLE_PASSWORD -Dapple.notarization.team.id=$APPLE_TEAM_ID -Dgpg.signer=bc
env:
OSSRH_USER: ${{ secrets.OSSRH_USER }}
OSSRH_PASS: ${{ secrets.OSSRH_PASS }}
MAVEN_GPG_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
APPLE_USERNAME: ${{ secrets.APPLE_USERNAME }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
- name: Upload release asset for Windows (with Java)
id: upload-release-asset-win-with-java
if: github.event_name == 'release'
uses: shogo82148/[email protected]
with:
upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }}
asset_path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip
asset_name: openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip
asset_content_type: application/zip
- name: Upload build artifact for Windows (with Java)
id: upload-build-artifact-win-with-java
uses: actions/upload-artifact@v4
with:
path: "./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip"
name: openrefine-win-with-java-archive-${{ steps.version_string_variable.outputs.output_value }}
- name: Unzip build artifact for Windows (with Java) for Installer
id: unzip-build-artifact-win-with-java
run: unzip ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}.zip -d ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}
- name: Upload build artifact for Windows (with Java) for Installer
id: upload-build-artifact-win-with-java-for-installer
uses: actions/upload-artifact@v4
with:
name: installertemp-win-with-java-${{ steps.version_string_variable.outputs.output_value }}
path: ./packaging/target/openrefine-win-with-java-${{ steps.version_string_variable.outputs.output_value }}/openrefine-${{ steps.version_string_variable.outputs.output_value }}
- name: Upload Release Asset Linux
id: upload-release-asset-linux
if: github.event_name == 'release'
uses: shogo82148/[email protected]
with:
upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }}
asset_path: ./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz
asset_name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz
asset_content_type: application/tar+gzip
- name: Upload build artifact for Linux
id: upload-build-artifact-linux
uses: actions/upload-artifact@v4
with:
path: "./packaging/target/openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}.tar.gz"
name: openrefine-linux-${{ steps.version_string_variable.outputs.output_value }}
- name: Upload Release Asset Mac
id: upload-release-asset-mac
if: github.event_name == 'release'
uses: shogo82148/[email protected]
with:
upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }}
asset_path: ./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg
asset_name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg
asset_content_type: application/x-apple-diskimage
- name: Upload build artifact for Mac
id: upload-build-artifact-mac
uses: actions/upload-artifact@v4
with:
path: "./packaging/target/openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}.dmg"
name: openrefine-mac-${{ steps.version_string_variable.outputs.output_value }}
windows_installer:
if: github.repository == 'OpenRefine/OpenRefine'
runs-on: windows-2022
needs: mac_test_and_deploy
steps:
- uses: actions/checkout@v4
- name: Get release upload URL
id: get_release_upload_url
if: github.event_name == 'release'
uses: bruceadams/[email protected]
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Download build artifact for Windows Installer (with Java)
id: download-build-artifact-win-with-java
uses: actions/download-artifact@v4
with:
name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}
path: ./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}
# - name: Sign the executables
# uses: nextgens/[email protected]
# with:
# certificate: '${{ secrets.CERTIFICATES }}'
# credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}'
# key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1"
# timestamp-url: 'http://timestamp.digicert.com'
# description: 'OpenRefine'
# description-url: 'https://openrefine.org'
# folder: "./packaging/target/openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}"
# recursive: true
- name: Build self-containing installer with InnoSetup 6 for Windows
id: build-installer-innosetup-win-with-java
run: |
"%ProgramFiles(x86)%\Inno Setup 6\iscc.exe" -DMyAppVersion="${{ needs.mac_test_and_deploy.outputs.version_number_string }}" -DMyProgramFiles="target\openrefine-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}" "packaging\openrefine.iss"
shell: cmd
- name: Cleanup unused installer build artifact (delete)
id: delete-build-artifact-win-with-java
uses: geekyeggo/delete-artifact@v5
with:
name: installertemp-win-with-java-${{ needs.mac_test_and_deploy.outputs.version_number_string }}
useGlob: false
failOnError: false
# - name: Sign the installer
# uses: nextgens/[email protected]
# with:
# certificate: '${{ secrets.CERTIFICATES }}'
# credentials: '${{ secrets.CERTIFICATE_CREDENTIALS }}'
# key-uri: "projects/myProject/locations/europe-west2/keyRings/code-signing/cryptoKeys/ev/cryptoKeyVersions/1"
# timestamp-url: 'http://timestamp.digicert.com'
# description: 'OpenRefine Installer'
# description-url: 'https://openrefine.org'
# folder: "./packaging/target"
# recursive: false
# - name: Verify the signature
# run: call "%ProgramFiles(x86)%\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe" verify /v /pa "OpenRefine-Installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe"
# shell: cmd
- name: Upload the installer as an artifact
id: upload-build-artifact-win-with-java-installer
uses: actions/upload-artifact@v4
with:
path: "./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe"
name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}
- name: Upload release asset for Windows (with Java)
id: upload-release-asset-win-with-java-installer
if: github.event_name == 'release'
uses: shogo82148/[email protected]
with:
upload_url: ${{ steps.get_release_upload_url.outputs.upload_url }}
asset_path: ./packaging/target/openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe
asset_name: openrefine-win-with-java-installer-${{ needs.mac_test_and_deploy.outputs.version_number_string }}.exe
asset_content_type: application/vnd.microsoft.portable-executable