From f37bb19707ecce3949da6b9b74cabcdd94c949df Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Thu, 14 Mar 2024 13:42:31 +0100 Subject: [PATCH 01/11] Split building and testing in workflow --- .github/workflows/build.yml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d239a54..b0abe971 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,17 +19,18 @@ jobs: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v2.1.1 - gradle: + build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - name: Gradle build, test and check + - name: Gradle build without tests uses: ./.github/workflows/gradle-goal with: - command: "./gradlew check" + # We also compile the test-classes, even though we are skipping the tests + command: "./gradlew build testClasses -x test" - name: Warmup gradle wrapper uses: ./.github/workflows/gradle-goal @@ -42,9 +43,36 @@ jobs: name: test-results path: '**/build/test-results/test/TEST-*.xml' + - name: Cache working directory with build results + uses: actions/upload-artifact@v3 + with: + name: working-dir-build-cache + path: ./ + - name: Agent artifact uses: actions/upload-artifact@v3 with: name: elastic-otel-javaagent path: | ./agent/build/libs/elastic-otel-javaagent-*.jar + + test: + runs-on: ubuntu-latest + needs: + - build + steps: + - name: Download cached build working directory + uses: actions/download-artifact@v3 + with: + name: working-dir-build-cache + path: ./ + - name: Run tests + uses: ./.github/workflows/gradle-goal + with: + command: "./gradlew test" + - name: Store test results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: test-results + path: '**/build/test-results/test/TEST-*.xml' From 8055104b7f95571320a6420340c645a86b254a76 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Thu, 14 Mar 2024 13:45:55 +0100 Subject: [PATCH 02/11] Remove leftover steps --- .github/workflows/build.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0abe971..ec5a0b22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,17 +32,6 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" - - name: Warmup gradle wrapper - uses: ./.github/workflows/gradle-goal - with: - command: "./gradlew check" - - name: Store test results - if: success() || failure() - uses: actions/upload-artifact@v3 - with: - name: test-results - path: '**/build/test-results/test/TEST-*.xml' - - name: Cache working directory with build results uses: actions/upload-artifact@v3 with: From 8c81458583d9f72cdc40583f6043afc8ac412c67 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Thu, 14 Mar 2024 14:06:49 +0100 Subject: [PATCH 03/11] Tar working dir before uploading --- .github/workflows/build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec5a0b22..b359f6f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,11 +32,14 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" + - name: Create working dir archive + run: "tar --exclude='./working-dir-build-cache.tar.gz' -czf ./working-dir-build-cache.tar.gz ." + - name: Cache working directory with build results uses: actions/upload-artifact@v3 with: name: working-dir-build-cache - path: ./ + path: ./working-dir-build-cache.tar.gz - name: Agent artifact uses: actions/upload-artifact@v3 @@ -55,6 +58,8 @@ jobs: with: name: working-dir-build-cache path: ./ + - name: Untar cached build working directory + run: "tar -xvf working-dir-build-cache.tar.gz" - name: Run tests uses: ./.github/workflows/gradle-goal with: From e259247e4951e5e29179c839523383d3e81e7390 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Thu, 14 Mar 2024 14:16:38 +0100 Subject: [PATCH 04/11] Fix self-taring problems --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b359f6f2..7cb12909 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,13 +33,13 @@ jobs: command: "./gradlew build testClasses -x test" - name: Create working dir archive - run: "tar --exclude='./working-dir-build-cache.tar.gz' -czf ./working-dir-build-cache.tar.gz ." + run: "mkdir ./wdarch && tar --exclude='./wdarch' -czf ./wdarch/working-dir-build-cache.tar.gz ." - name: Cache working directory with build results uses: actions/upload-artifact@v3 with: name: working-dir-build-cache - path: ./working-dir-build-cache.tar.gz + path: ./wdarch/working-dir-build-cache.tar.gz - name: Agent artifact uses: actions/upload-artifact@v3 From f94e12c4923d315bc8926d9f92e286bb541bf12e Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 08:48:39 +0100 Subject: [PATCH 05/11] Add comments, do not recompile native libs --- .github/workflows/build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cb12909..f3f9c8ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,9 +32,10 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" + # We first tar the working directory and then upload it + # We do this because uploading directories with many files directly as artifacts is very slow - name: Create working dir archive run: "mkdir ./wdarch && tar --exclude='./wdarch' -czf ./wdarch/working-dir-build-cache.tar.gz ." - - name: Cache working directory with build results uses: actions/upload-artifact@v3 with: @@ -53,6 +54,7 @@ jobs: needs: - build steps: + # We use the cached working directory so that we don't have to recompile everything - name: Download cached build working directory uses: actions/download-artifact@v3 with: @@ -63,7 +65,11 @@ jobs: - name: Run tests uses: ./.github/workflows/gradle-goal with: - command: "./gradlew test" + # We manually skip the compileJni task because we know it is up-to-date in the cached + # working directory. The up-to-date check of this task checks for the presence of docker + # images used for compiling the native library, which do not exist because we are in a new + # environment. + command: "./gradlew test -x compileJni" - name: Store test results if: success() || failure() uses: actions/upload-artifact@v3 From 0d80a0d4b4f5ed19c2ef8837f7098fdd6a7dee18 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:05:27 +0100 Subject: [PATCH 06/11] Test performance of upload-artifact@v4 --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3f9c8ed..4983498b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,11 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" + - name: Test upload v4 performance + uses: actions/upload-artifact@v4 + with: + name: wd-upload-perf-test + path: ./ # We first tar the working directory and then upload it # We do this because uploading directories with many files directly as artifacts is very slow - name: Create working dir archive @@ -41,7 +46,6 @@ jobs: with: name: working-dir-build-cache path: ./wdarch/working-dir-build-cache.tar.gz - - name: Agent artifact uses: actions/upload-artifact@v3 with: From 3089821b0bcf18576667954b7be8d1b3c724f8c1 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:18:26 +0100 Subject: [PATCH 07/11] Switch to @v4 actions for simplicity and performance --- .github/workflows/build.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4983498b..c44a58e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,22 +32,14 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" - - name: Test upload v4 performance - uses: actions/upload-artifact@v4 - with: - name: wd-upload-perf-test - path: ./ - # We first tar the working directory and then upload it - # We do this because uploading directories with many files directly as artifacts is very slow - - name: Create working dir archive - run: "mkdir ./wdarch && tar --exclude='./wdarch' -czf ./wdarch/working-dir-build-cache.tar.gz ." - name: Cache working directory with build results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: working-dir-build-cache - path: ./wdarch/working-dir-build-cache.tar.gz + path: ./ + retention-days: 1 # No need to waste space when this artifact is only used as part of the build process - name: Agent artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: elastic-otel-javaagent path: | @@ -60,12 +52,10 @@ jobs: steps: # We use the cached working directory so that we don't have to recompile everything - name: Download cached build working directory - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: working-dir-build-cache path: ./ - - name: Untar cached build working directory - run: "tar -xvf working-dir-build-cache.tar.gz" - name: Run tests uses: ./.github/workflows/gradle-goal with: From 7bc46338e442a473a5db9d539ade90b218c46182 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:26:57 +0100 Subject: [PATCH 08/11] Revert "Switch to @v4 actions for simplicity and performance" This reverts commit 3089821b0bcf18576667954b7be8d1b3c724f8c1. --- .github/workflows/build.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c44a58e6..4983498b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,14 +32,22 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" - - name: Cache working directory with build results + - name: Test upload v4 performance uses: actions/upload-artifact@v4 with: - name: working-dir-build-cache + name: wd-upload-perf-test path: ./ - retention-days: 1 # No need to waste space when this artifact is only used as part of the build process + # We first tar the working directory and then upload it + # We do this because uploading directories with many files directly as artifacts is very slow + - name: Create working dir archive + run: "mkdir ./wdarch && tar --exclude='./wdarch' -czf ./wdarch/working-dir-build-cache.tar.gz ." + - name: Cache working directory with build results + uses: actions/upload-artifact@v3 + with: + name: working-dir-build-cache + path: ./wdarch/working-dir-build-cache.tar.gz - name: Agent artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: elastic-otel-javaagent path: | @@ -52,10 +60,12 @@ jobs: steps: # We use the cached working directory so that we don't have to recompile everything - name: Download cached build working directory - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: working-dir-build-cache path: ./ + - name: Untar cached build working directory + run: "tar -xvf working-dir-build-cache.tar.gz" - name: Run tests uses: ./.github/workflows/gradle-goal with: From 695a6752d6257d776934a9d245dfd214f33a3190 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:26:57 +0100 Subject: [PATCH 09/11] Revert "Test performance of upload-artifact@v4" This reverts commit 0d80a0d4b4f5ed19c2ef8837f7098fdd6a7dee18. --- .github/workflows/build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4983498b..f3f9c8ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,11 +32,6 @@ jobs: # We also compile the test-classes, even though we are skipping the tests command: "./gradlew build testClasses -x test" - - name: Test upload v4 performance - uses: actions/upload-artifact@v4 - with: - name: wd-upload-perf-test - path: ./ # We first tar the working directory and then upload it # We do this because uploading directories with many files directly as artifacts is very slow - name: Create working dir archive @@ -46,6 +41,7 @@ jobs: with: name: working-dir-build-cache path: ./wdarch/working-dir-build-cache.tar.gz + - name: Agent artifact uses: actions/upload-artifact@v3 with: From 030b87f88c87828acf826a5bf546cf929ee71966 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:28:43 +0100 Subject: [PATCH 10/11] use upload-artifact@v4, but keep tar-ing to preserve permissions --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3f9c8ed..b5d2e629 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,17 +33,17 @@ jobs: command: "./gradlew build testClasses -x test" # We first tar the working directory and then upload it - # We do this because uploading directories with many files directly as artifacts is very slow + # We do this because the upload-artifact action doesn't preserve file permissions - name: Create working dir archive run: "mkdir ./wdarch && tar --exclude='./wdarch' -czf ./wdarch/working-dir-build-cache.tar.gz ." - name: Cache working directory with build results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: working-dir-build-cache path: ./wdarch/working-dir-build-cache.tar.gz - name: Agent artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: elastic-otel-javaagent path: | @@ -56,7 +56,7 @@ jobs: steps: # We use the cached working directory so that we don't have to recompile everything - name: Download cached build working directory - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: working-dir-build-cache path: ./ From 6907919e4b9eb061a35faf8ede8eba2dd50d8a72 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 25 Mar 2024 10:29:26 +0100 Subject: [PATCH 11/11] Add retention --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5d2e629..a11df0d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,7 @@ jobs: with: name: working-dir-build-cache path: ./wdarch/working-dir-build-cache.tar.gz + retention-days: 1 # No need to waste space when this artifact is only used as part of the build process - name: Agent artifact uses: actions/upload-artifact@v4