diff --git a/.github/workflows/assign_issue_to_project.yml b/.github/workflows/assign_issue_to_project.yml index 860ef1e24..2010447c0 100644 --- a/.github/workflows/assign_issue_to_project.yml +++ b/.github/workflows/assign_issue_to_project.yml @@ -13,7 +13,7 @@ # limitations under the License. # -name: Auto Assign Issue to Project +name: Assign Issue to Project on: issues: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34f6d55a1..e0aeb448a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: Build all modules +name: Build All Modules on: pull_request: @@ -23,7 +23,8 @@ on: types: [ opened, synchronize, reopened ] jobs: - build-agent-and-model: + test-agent-and-model: + name: Test Agent and Model runs-on: ubuntu-latest steps: - name: Checkout code @@ -36,55 +37,13 @@ jobs: java-version: "adopt@1.8" - name: Build and run unit tests - run: sbt "project agent" test doc "project model" test doc - - build-server-unit: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - uses: coursier/cache-action@v5 - - - name: Setup Scala - uses: olafurpg/setup-scala@v14 - with: - java-version: "adopt@1.11.0-11" - - - name: Build and run unit tests - run: sbt "project server" test - - build-server-integration: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: atum_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - name: Checkout code - uses: actions/checkout@v2 - - uses: coursier/cache-action@v5 - - - name: Setup Scala - uses: olafurpg/setup-scala@v14 - with: - java-version: "adopt@1.11.0-11" - - - name: Prepare testing database - run: sbt flywayMigrate + run: sbt "project model" test doc "project agent_spark3" test doc - name: Build and run integration tests - run: sbt "project server" testIT doc + run: sbt "project model" testIT "project agent_spark3" testIT - build-database: + test-database-and-server: + name: Test Database and Server runs-on: ubuntu-latest services: postgres: @@ -107,14 +66,13 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v14 with: - java-version: "adopt@1.8" + java-version: "adopt@1.11.0-11" - # Fails on balta dbConnection error issue: https://github.com/AbsaOSS/balta/issues/23 -# - name: Build and run unit tests -# run: sbt "project database" test + - name: Build and run unit tests + run: sbt "project database" test doc "project server" test doc - name: Prepare testing database run: sbt flywayMigrate - - name: Build and run database/integration tests - run: sbt testDB doc + - name: Build and run integration tests + run: sbt "project database" testIT "project server" testIT diff --git a/.github/workflows/dependent_items.yml b/.github/workflows/dependent_items.yml index 1d690f2ee..bb35a2e95 100644 --- a/.github/workflows/dependent_items.yml +++ b/.github/workflows/dependent_items.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: Dependent Issues +name: Dependent Items on: issues: @@ -40,7 +40,8 @@ on: - cron: '0 0 * * *' jobs: - check: + dependent-items-check: + name: Dependent Items Check runs-on: ubuntu-latest steps: - uses: z0al/dependent-issues@v1.5.2 diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 1f1375d6b..a93ce1783 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -14,17 +14,17 @@ # limitations under the License. # -name: FormatCheck +name: Format Check on: pull_request: - branches: [ master, develop ] - types: [ assigned, opened, synchronize, reopened, labeled ] + branches: [ master ] + types: [ assigned, opened, synchronize ] jobs: - scalafmt: + scala-format-check: + name: Scala Format Check runs-on: ubuntu-latest - name: Scalafmt Check steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml deleted file mode 100644 index caf5eade6..000000000 --- a/.github/workflows/jacoco_check_server.yml +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: JaCoCo report server - -on: - pull_request: - branches: [ master ] - types: [ opened, edited, synchronize, reopened ] - -env: - scalaLong: 2.13.11 - scalaShort: "2.13" - overall: 75.0 - changed: 75.0 - -jobs: - test: - name: Build and test - runs-on: ubuntu-latest - - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: atum_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Setup Scala - uses: olafurpg/setup-scala@v14 - with: - java-version: "adopt@1.11.0-11" - - name: Prepare testing database - run: sbt flywayMigrate - - name: Build and run tests - continue-on-error: true - id: jacocorun - run: sbt "project server" jacoco - # server module code coverage - - name: Add coverage to PR - if: steps.jacocorun.outcome == 'success' - id: jacoco-server - uses: madrapps/jacoco-report@v1.6.1 - with: - paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{env.overall }} - min-coverage-changed-files: ${{ env.changed }} - title: JaCoCo server module code coverage report - scala ${{ env.scalaLong }} - update-comment: true - - name: Get the Coverage info - if: steps.jacocorun.outcome == 'success' - run: | - echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - - name: Fail PR if changed files coverage is less than ${{ env.changed }}% - if: steps.jacocorun.outcome == 'success' - uses: actions/github-script@v6 - with: - script: | - const coverageCheckFailed = - Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}'); - if (coverageCheckFailed) { - core.setFailed('Changed files coverage is less than ${{ env.changed }}%!'); - } - - name: Edit JaCoCo comments on build failure - if: steps.jacocorun.outcome != 'success' - uses: actions/github-script@v6 - with: - script: | - const issue_number = context.issue.number; - const owner = context.repo.owner; - const repo = context.repo.repo; - const jacocoReportRegExp = /^### JaCoCo .* code coverage report .*/; - - const comments = await github.rest.issues.listComments({ - owner, - repo, - issue_number, - }); - - for (const comment of comments.data) { - const lines = comment.body.split('\n'); - if (lines.length > 0 && jacocoReportRegExp.test(lines[0])) { - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: comment.id, - body: lines[0] + "\n\n### Build Failed", - }); - } - } - - core.setFailed('JaCoCo test coverage report generation failed, and related PR comments were updated.'); diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_report.yml similarity index 71% rename from .github/workflows/jacoco_check.yml rename to .github/workflows/jacoco_report.yml index 4757c1160..003ec5506 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_report.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: JaCoCo report agent and module +name: JaCoCo Report on: pull_request: @@ -28,48 +28,69 @@ env: changed: 80.0 jobs: - test: - name: Build and test + build-test-and-measure: + name: Build, Test and Measure runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: atum_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup Scala uses: olafurpg/setup-scala@v14 with: - java-version: "adopt@1.8" + java-version: "adopt@1.11.0-11" + - name: Prepare testing database + run: sbt flywayMigrate - name: Build and run tests continue-on-error: true id: jacocorun - run: | - sbt "project agent; jacoco" - sbt "project model; jacoco" - # agent module code coverage - - name: Add coverage to PR + run: sbt jacoco + - name: Add coverage to PR (model) + if: steps.jacocorun.outcome == 'success' + id: jacoco-model + uses: madrapps/jacoco-report@v1.6.1 + with: + paths: ${{ github.workspace }}/model/target/jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{ env.overall }} + min-coverage-changed-files: ${{ env.changed }} + title: JaCoCo model module code coverage report - scala ${{ env.scalaLong }} + update-comment: true + - name: Add coverage to PR (agent) if: steps.jacocorun.outcome == 'success' id: jacoco-agent uses: madrapps/jacoco-report@v1.6.1 with: - name: agent-jacoco-report - paths: ${{ github.workspace }}/agent/target/jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/agent/target/spark3-jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{ env.overall }} - min-coverage-changed-file: ${{ env.changed }} + min-coverage-changed-files: ${{ env.changed }} title: JaCoCo agent module code coverage report - scala ${{ env.scalaLong }} update-comment: true - # model module code coverage - - name: Add coverage to PR + - name: Add coverage to PR (server) if: steps.jacocorun.outcome == 'success' - id: jacoco-model + id: jacoco-server uses: madrapps/jacoco-report@v1.6.1 with: - name: model-jacoco-report - paths: ${{ github.workspace }}/model/target/jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{ env.overall }} - min-coverage-changed-file: ${{ env.changed }} - title: JaCoCo model module code coverage report - scala ${{ env.scalaLong }} + min-coverage-overall: ${{env.overall }} + min-coverage-changed-files: ${{ env.changed }} + title: JaCoCo server module code coverage report - scala ${{ env.scalaLong }} update-comment: true - name: Get the Coverage info if: steps.jacocorun.outcome == 'success' @@ -78,6 +99,8 @@ jobs: echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}" echo "Changed Files coverage ${{ steps.jacoco-model.outputs.coverage-changed-files }}" + echo "Total server module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ env.changed }}% if: steps.jacocorun.outcome == 'success' uses: actions/github-script@v6 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index 965dd287f..36a4f4d5f 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -22,7 +22,8 @@ on: types: [ opened, synchronize, reopened ] jobs: - license-test: + license-check: + name: License Check runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/README.md b/README.md index 4868c4d25..1d14d9d15 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ It provides functionality for computing and pushing control metrics to the API l For more information, see the [Vocabulary section](#Vocabulary) or `agent/README.md` for more technical documentation. #### Spark 2.4 support -Because there are some java level incompatibilities between Spark 2.4 and Spark 3.x whe build on Java 11+, we have to +Because there are some java level incompatibilities between Spark 2.4 and Spark 3.x when build on Java 11+, we have to drop support for Spark 2.4. If you need the agent to work with Spark 2.4 follow these steps: * Switch to Java 8 * In `'build.sbt'` change the matrix rows, to be Spark 2.4 and Scala 2.11 for modules _agent_ and _model_ @@ -127,7 +127,7 @@ executing the calculation against a given Spark DataFrame. Some `Measures` define no columns (such as `count`), some require exactly one column (such as `sum` of values for particular column), and some require more columns (such as `hash` function). -### Measurement +### Measurement Practically speaking, a single `Measurement` contains a `Measure` and result associated with it.