#127 Review SBT files and CI yamls (#136)

* #127 Review SBT files and CI yamls

- few whitespace cleanups and moves
- rename limitVersion to truncateVerions and reuse it
- fix logging of spark and scala version per modulea
- block publishing of war files to maven central
- update project description
- update jacoco build to use variables instead of matrix
- update release yaml to publish WAR to a draft PR
- if jacoco fails it updates jacoco comments as failed

---------

Co-authored-by: miroslavpojer <[email protected]>
Co-authored-by: David Benedeki <[email protected]>

.github/workflows/jacoco_check.yml

@@ -21,52 +21,131 @@ on:
     branches: [ master ]
     types: [ opened, edited, synchronize, reopened ]
 
+env:
+  scalaLong12: 2.12.18
+  scalaShort12: "2.12"
+  scalaLong13: 2.13.11
+  scalaShort13: "2.13"
+  overall: 80.0
+  changed: 80.0
+
 jobs:
   test:
     name: Build and test
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          # The project supports Scala 2.11, 2.12 and the default version of spark2
-          # The CI runs all tests suites for all supported Scala versions at build.yml
-          # The codebase for all Scala versions is the same, so the coverage is calculated only once
-          # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and
-          # vendor distributions.
-          - scala: 2.12.18
-            scalaShort: "2.12"
-            overall: 80.0
-            changed: 80.0
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: atum_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+        uses: olafurpg/setup-scala@v14
         with:
           java-version: "[email protected]"
+      - name: Prepare testing database
+        run: sbt flywayMigrate
       - name: Build and run tests
-        run: sbt "++${{matrix.scala}}; project agent; jacoco"
-     #  Agent module code coverage
+        continue-on-error: true
+        id: jacocorun
+        run: sbt jacoco
+      #  server module code coverage
       - name: Add coverage to PR
+        if: steps.jacocorun.outcome == 'success'
+        id: jacoco-server
+        uses: madrapps/[email protected]
+        with:
+          paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{env.overall }}
+          min-coverage-changed-files: ${{ env.changed }}
+          title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }}
+          update-comment: true
+      #  agent module code coverage
+      - name: Add coverage to PR
+        if: steps.jacocorun.outcome == 'success'
         id: jacoco-agent
         uses: madrapps/[email protected]
         with:
-          paths: ${{ github.workspace }}/agent/target/spark2-jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
+          name: agent-jacoco-report
+          paths: ${{ github.workspace }}/agent/target/jvm-${{ env.scalaShort12 }}/jacoco/report/jacoco.xml
           token: ${{ secrets.GITHUB_TOKEN }}
-          min-coverage-overall: ${{matrix.overall }}
-          min-coverage-changed-files: ${{ matrix.changed }}
-          title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }}
+          min-coverage-overall: ${{ env.overall }}
+          min-coverage-changed-file: ${{ env.changed }}
+          title: JaCoCo agent module code coverage report - scala ${{ env.scalaLong12 }}
+          update-comment: true
+      #  model module code coverage
+      - name: Add coverage to PR
+        if: steps.jacocorun.outcome == 'success'
+        id: jacoco-model
+        uses: madrapps/[email protected]
+        with:
+          name: model-jacoco-report
+          paths: ${{ github.workspace }}/model/target/jvm-${{ env.scalaShort12 }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{ env.overall }}
+          min-coverage-changed-file: ${{ env.changed }}
+          title: JaCoCo model module code coverage report - scala ${{ env.scalaLong12 }}
           update-comment: true
       - name: Get the Coverage info
+        if: steps.jacocorun.outcome == 'success'
         run: |
+          echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}"
           echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}"
           echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}"
-      - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}%
-        if: |
-          ${{ steps.jacoco-agent.outputs.coverage-changed-files < 80.0 }}
+          echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco-model.outputs.coverage-changed-files }}"
+      - name: Fail PR if changed files coverage is less than ${{ env.changed }}%
+        if: steps.jacocorun.outcome == 'success'
         uses: actions/github-script@v6
         with:
           script: |
-            core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!')
+            const coverageCheckFailed =
+              Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') ||
+              Number('${{ steps.jacoco-agent.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') ||
+              Number('${{ steps.jacoco-model.outputs.coverage-changed-files }}') < Number('${{ env.changed }}');
+            if (coverageCheckFailed) {
+              core.setFailed('Changed files coverage is less than ${{ env.changed }}%!');
+            }
+      - name: Edit JaCoCo comments on build failure
+        if: steps.jacocorun.outcome != 'success'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const issue_number = context.issue.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const jacocoReportRegExp = /^### JaCoCo .* code coverage report .*/;
+
+            const comments = await github.rest.issues.listComments({
+              owner,
+              repo,
+              issue_number,
+            });
 
+            for (const comment of comments.data) {
+              const lines = comment.body.split('\n');
+              if (lines.length > 0 && jacocoReportRegExp.test(lines[0])) {
+                await github.rest.issues.updateComment({
+                  owner,
+                  repo,
+                  comment_id: comment.id,
+                  body: lines[0] + "\n\n### Build Failed",
+                });
+              }
+            }
 
+            core.setFailed('JaCoCo test coverage report generation failed, and related PR comments were updated.');

.github/workflows/release_draft.yml

@@ -0,0 +1,115 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Release - create draft release
+on:
+  workflow_dispatch:
+    inputs:
+      tagName:
+        description: 'Name of git tag to be created, and then draft release created. Syntax: "v[0-9]+.[0-9]+.[0-9]+".'
+        required: true
+
+jobs:
+  tag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Validate format of received tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const newTag = core.getInput('tag-name');
+            const regex = /^v[0-9]+\.[0-9]+\.[0-9]+$/;
+
+            if (!regex.test(newTag)) {
+              core.setFailed('Tag does not match the required format "v[0-9]+.[0-9]+.[0-9]+"');
+              return;
+            }
+
+            // get all tags
+            const { data: refs } = await github.rest.git.listMatchingRefs({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: 'tags/'
+            });
+
+            const latestTag = refs.sort((a, b) => new Date(b.object.date) - new Date(a.object.date))[0].ref.replace('refs/tags/', '');
+            const latestVersion = latestTag.replace('v', '').split('.').map(Number);
+            const newVersion = newTag.replace('v', '').split('.').map(Number);
+
+            // check tag's correct version increase
+            const isValidVersion = (latestVersion[0] === newVersion[0] && latestVersion[1] === newVersion[1] && newVersion[2] === latestVersion[2] + 1) ||
+              (latestVersion[0] === newVersion[0] && newVersion[1] === latestVersion[1] + 1 && newVersion[2] === 0) ||
+              (newVersion[0] === latestVersion[0] + 1 && newVersion[1] === 0 && newVersion[2] === 0);
+
+            if (!isValidVersion) {
+              core.setFailed('New tag is not one version higher than the latest tag');
+              return;
+            }
+          tag-name: ${{ github.event.inputs.tagName }}
+
+      - name: Create and push tag
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = core.getInput('tag-name')
+            const ref = `refs/tags/${tag}`;
+            const sha = context.sha; // The SHA of the commit to tag
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: ref,
+              sha: sha
+            });
+
+            console.log(`Tag created: ${tag}`);
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          tag-name: ${{ github.event.inputs.tagName }}
+
+  release-draft:
+    needs: tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: refs/tags/${{ github.event.inputs.tagName }}
+      - name: Generate release notes
+        id: generate_release_notes
+        uses: AbsaOSS/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          chapters: |
+            [
+              {"title": "Breaking Changes 💥", "label": "breaking-change"},
+              {"title": "New Features 🎉", "label": "enhancement"},
+              {"title": "Bugfixes 🛠", "label": "bug"}
+            ]
+          warnings: true
+      - name: Create draft release
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          name: ${{ github.event.inputs.tagName }}
+          body: ${{ steps.generate_release_notes.outputs.releaseNotes }}
+          tag_name: ${{ github.event.inputs.tagName }}
+          draft: true
+          prerelease: false

.github/workflows/release.yml → .github/workflows/release_publish.yml

@@ -14,21 +14,32 @@
 # limitations under the License.
 #
 
-name: Release
+name: Release - publish artifacts
 on:
-  workflow_dispatch:
+  release:
+    types: [released]
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: olafurpg/setup-scala@v13
-      - run: sbt ci-release
+      - uses: olafurpg/setup-scala@v14
+      - name: Run sbt ci-release (produces war as well)
+        run: sbt ci-release
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+
+      - name: Find WAR file
+        id: find_war
+        run: echo "WAR_PATH=$(find . -name '*.war' | head -n 1)" >> $GITHUB_ENV
+
+      - name: Upload WAR file to GitHub Release
+        run: gh release upload ${{ github.event.release.tag_name }} ${{ env.WAR_PATH }} --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}