Feature/#63 create build matrix and build action

.github/workflows/assign_issue_to_project.yml

@@ -1,3 +1,18 @@
+#
+#   Copyright 2021 ABSA Group Limited
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
 name: Auto Assign Issue to Project
 
 on:

.github/workflows/build.yml

@@ -0,0 +1,39 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build
+
+on:
+  pull_request:
+    branches: [ master ]
+    types: [ opened, synchronize, reopened ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: coursier/cache-action@v5
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: "[email protected]"
+      - name: Build and run tests
+        run: sbt test doc

.github/workflows/dependent_items.yml

@@ -1,3 +1,19 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 name: Dependent Issues
 
 on:

.github/workflows/format_check.yml

@@ -0,0 +1,42 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: FormatCheck
+
+on:
+  pull_request:
+    branches: [ master, develop ]
+    types: [ assigned, opened, synchronize, reopened, labeled ]
+
+jobs:
+  scalafmt:
+    runs-on: ubuntu-latest
+    name: Scalafmt Check
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: "[email protected]"
+
+      - name: Run scalafmt And Print Diff
+        continue-on-error: true
+        run: sbt scalafmt scalafmtSbt && git diff --exit-code

.github/workflows/jacoco_check.yml

@@ -0,0 +1,88 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: JaCoCo report
+
+on:
+  pull_request:
+    branches: [ master ]
+    types: [ opened, edited, synchronize, reopened ]
+
+jobs:
+  test:
+    name: Build and test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          # The project supports Scala 2.11, 2.12 and the default version of spark2
+          # The CI runs all tests suites for all supported Scala versions at build.yml
+          # The codebase for all Scala versions is the same, so the coverage is calculated only once
+          # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and
+          # vendor distributions.
+          - scala: 2.12.12
+            scalaShort: "2.12"
+            overall: 80.0
+            changed: 80.0
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: "[email protected]"
+      - name: Build and run tests
+        run: sbt ++${{matrix.scala}} jacoco
+     #  Agent module code coverage
+      - name: Add coverage to PR
+        id: jacoco-agent
+        uses: madrapps/[email protected]
+        with:
+          name: agent-jacoco-report
+          paths: ${{ github.workspace }}/agent/target/spark2-jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{matrix.overall }}
+          min-coverage-changed-file: ${{ matrix.changed }}
+          title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }}
+          update-comment: true
+     #  server module code coverage
+      - name: Add coverage to PR
+        id: jacoco-server
+        uses: madrapps/[email protected]
+        with:
+          name: server-jacoco-report
+          paths: ${{ github.workspace }}/server/target/jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          min-coverage-overall: ${{matrix.overall }}
+          min-coverage-changed-file: ${{ matrix.changed }}
+          title: JaCoCo server module code coverage report - scala ${{ matrix.scala }}
+          update-comment: true
+      - name: Get the Coverage info
+        run: |
+          echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}"
+          echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}"
+          echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}"
+      - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}%
+        if: |
+          ${{ steps.jacoco-agent.outputs.coverage-changed-files < 80.0 }} &&
+          ${{ steps.jacoco-server.outputs.coverage-changed-files < 80.0 }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!')
+
+

.github/workflows/release.yml

@@ -0,0 +1,34 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Release
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - uses: olafurpg/setup-scala@v13
+      - run: sbt ci-release
+        env:
+          PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+          PGP_SECRET: ${{ secrets.PGP_SECRET }}
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}

README.md

@@ -10,3 +10,18 @@ See `agent/README.md`.
 ### Server `server/`
 An API under construction that communicates with AtumAgent and with the persisting storage. It also provides measure configuration to the `AtumAgent`.
 See `server/README.md`.
+
+## How to generate Code coverage report
+```sbt
+sbt jacoco
+```
+Code coverage wil be generated on path:
+```
+{project-root}/atum-service/target/spark{spark_version}-jvm-{scala_version}/jacoco/report/html
+{project-root}/atum-service-test/target/jvm-{scala_version}/jacoco/report/html
+```
+
+
+## How to Release
+
+Please see [this file](RELEASE.md) for more details.

RELEASE.md

@@ -0,0 +1,23 @@
+# Release
+
+Releases of this library are currently handled by [sbt-ci-release](https://github.com/sbt/sbt-ci-release).
+Please see its documentation for more details about how it works if you are interested to know more.
+
+The actual deployments are triggered manually by the maintainers of this repository, using `workflow_dispatch` event
+trigger.
+
+Once changes from a PR were reviewed and merged into the master branch, follow these steps:
+1. Create a new Git Tag and push it to the repository, to the master branch. For example,
+   if you want to release a version 0.4.0 (note that we are using [Semantic Versioning](https://semver.org/)):
+
+    ```shell
+    git tag -a v0.4.0 -m "v0.4.0"
+    git push origin v0.4.0
+    ```
+
+2. In GitHub UI, go to the repository's **Actions** -> **Release** -> **Run workflow**, and under **Use workflow from**
+   use **Tags** and find the tag you created in the previous step.
+
+   > **Important note**: don't run the workflow against the master branch, but against the tag.
+   > `sbt-ci-release` plugin won't be able to correctly find tag, and it will think that you are trying
+   > to do a snapshot release, not an actual release that should be synchronized with Maven Central.

agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent
 
 import za.co.absa.atum.agent.model.MeasureResult

agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent
 
 import org.apache.spark.sql.DataFrame

agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.core
 
 import org.apache.spark.sql.DataFrame

agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.model
 
 case class MeasureResult(measurement: Measurement, result: String)

agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.model
 
 import org.apache.spark.sql.functions._