From 1bcaa3d5200b962279a1146509eaddb739db6fb6 Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:15:14 +0200 Subject: [PATCH] Feature/#63 create build matrix and build action (#64) * fixes #63 - added some configs variables for stb and jacoco code coverage. * fixes #63 - added more configs variables for stb * fixes #63 - added GitHub workflow Actions * fixes #63 - added license * fixes #63 * fixes #63 * * Fixed `ProjectMatrix` * * fixed bugs preventing project load and compilation * Fixes #63 - Emplementing jacoco setup * Fixes #63 - implementing jacoco setup * Fixes #63 - implementing jacoco setup * Fixes #63 - implementing jacoco setup * Fixes #63 - Fixing license * Fixes #63 - Fixing license * Fixes #63 - removed license_check and CODEOWNERS files * Update project/plugins.sbt Co-authored-by: miroslavpojer * Update project/plugins.sbt Co-authored-by: miroslavpojer * Fixes #63 - Added jacoco workflow * Fixes #63 - Added dependent_items.yml * Fixes #63 - Added dependent_items.yml * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * * Proposal of adding Jacoco support for server module. * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * * Proposal for change. * * Return back the title with spark major version now. * Fixes #63 * Fixes #63 - added license * Update .github/workflows/jacoco_check.yml Co-authored-by: Ladislav Sulak * Update .github/workflows/jacoco_check.yml Co-authored-by: Ladislav Sulak * Fixes #63 - update build.sbt * Fixes #63 - update publish.sbt * Fixes #63 - update publish.sbt * Fixes #63 * Fixes #63 * Fixes #63 * Fixes #63 * Closes #63 * Closes #63 - Added notes for jacoco coverage and realse in the readme file --------- Co-authored-by: David Benedeki Co-authored-by: miroslavpojer Co-authored-by: Ladislav Sulak --- .github/workflows/assign_issue_to_project.yml | 15 ++++ .github/workflows/build.yml | 39 ++++++++ .github/workflows/dependent_items.yml | 16 ++++ .github/workflows/format_check.yml | 42 +++++++++ .github/workflows/jacoco_check.yml | 88 +++++++++++++++++++ .github/workflows/release.yml | 34 +++++++ README.md | 15 ++++ RELEASE.md | 23 +++++ .../za/co/absa/atum/agent/AtumAgent.scala | 16 ++++ .../za/co/absa/atum/agent/AtumContext.scala | 16 ++++ .../agent/core/MeasurementProcessor.scala | 16 ++++ .../absa/atum/agent/model/MeasureResult.scala | 16 ++++ .../absa/atum/agent/model/Measurement.scala | 16 ++++ build.sbt | 74 ++++++++++++---- project/Dependencies.scala | 53 ++++++----- project/JacocoSetup.scala | 44 ++++++++++ project/SparkVersionAxis.scala | 56 ++++++++++++ project/build.properties | 15 ++++ project/plugins.sbt | 23 +++++ publish.sbt | 41 +++++++++ .../za/co/absa/atum/web/api/package.scala | 2 +- version.sbt | 1 - 22 files changed, 620 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/format_check.yml create mode 100644 .github/workflows/jacoco_check.yml create mode 100644 .github/workflows/release.yml create mode 100644 RELEASE.md create mode 100644 project/JacocoSetup.scala create mode 100644 project/SparkVersionAxis.scala delete mode 100644 version.sbt diff --git a/.github/workflows/assign_issue_to_project.yml b/.github/workflows/assign_issue_to_project.yml index 382b3b33b..860ef1e24 100644 --- a/.github/workflows/assign_issue_to_project.yml +++ b/.github/workflows/assign_issue_to_project.yml @@ -1,3 +1,18 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Auto Assign Issue to Project on: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..ff833ea67 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,39 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Build + +on: + pull_request: + branches: [ master ] + types: [ opened, synchronize, reopened ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: coursier/cache-action@v5 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt test doc diff --git a/.github/workflows/dependent_items.yml b/.github/workflows/dependent_items.yml index fd0349ceb..1d690f2ee 100644 --- a/.github/workflows/dependent_items.yml +++ b/.github/workflows/dependent_items.yml @@ -1,3 +1,19 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Dependent Issues on: diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml new file mode 100644 index 000000000..1f1375d6b --- /dev/null +++ b/.github/workflows/format_check.yml @@ -0,0 +1,42 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: FormatCheck + +on: + pull_request: + branches: [ master, develop ] + types: [ assigned, opened, synchronize, reopened, labeled ] + +jobs: + scalafmt: + runs-on: ubuntu-latest + name: Scalafmt Check + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + + - name: Run scalafmt And Print Diff + continue-on-error: true + run: sbt scalafmt scalafmtSbt && git diff --exit-code diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml new file mode 100644 index 000000000..398a32ddf --- /dev/null +++ b/.github/workflows/jacoco_check.yml @@ -0,0 +1,88 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: JaCoCo report + +on: + pull_request: + branches: [ master ] + types: [ opened, edited, synchronize, reopened ] + +jobs: + test: + name: Build and test + runs-on: ubuntu-latest + strategy: + matrix: + include: + # The project supports Scala 2.11, 2.12 and the default version of spark2 + # The CI runs all tests suites for all supported Scala versions at build.yml + # The codebase for all Scala versions is the same, so the coverage is calculated only once + # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and + # vendor distributions. + - scala: 2.12.12 + scalaShort: "2.12" + overall: 80.0 + changed: 80.0 + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt ++${{matrix.scala}} jacoco + # Agent module code coverage + - name: Add coverage to PR + id: jacoco-agent + uses: madrapps/jacoco-report@v1.3 + with: + name: agent-jacoco-report + paths: ${{ github.workspace }}/agent/target/spark2-jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{matrix.overall }} + min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }} + update-comment: true + # server module code coverage + - name: Add coverage to PR + id: jacoco-server + uses: madrapps/jacoco-report@v1.3 + with: + name: server-jacoco-report + paths: ${{ github.workspace }}/server/target/jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{matrix.overall }} + min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} + update-comment: true + - name: Get the Coverage info + run: | + echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" + echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" + - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% + if: | + ${{ steps.jacoco-agent.outputs.coverage-changed-files < 80.0 }} && + ${{ steps.jacoco-server.outputs.coverage-changed-files < 80.0 }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') + + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..2f418b92c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,34 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Release +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v13 + - run: sbt ci-release + env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} diff --git a/README.md b/README.md index 640d93c46..66738d954 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,18 @@ See `agent/README.md`. ### Server `server/` An API under construction that communicates with AtumAgent and with the persisting storage. It also provides measure configuration to the `AtumAgent`. See `server/README.md`. + +## How to generate Code coverage report +```sbt +sbt jacoco +``` +Code coverage wil be generated on path: +``` +{project-root}/atum-service/target/spark{spark_version}-jvm-{scala_version}/jacoco/report/html +{project-root}/atum-service-test/target/jvm-{scala_version}/jacoco/report/html +``` + + +## How to Release + +Please see [this file](RELEASE.md) for more details. \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..5e94dbc0b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,23 @@ +# Release + +Releases of this library are currently handled by [sbt-ci-release](https://github.com/sbt/sbt-ci-release). +Please see its documentation for more details about how it works if you are interested to know more. + +The actual deployments are triggered manually by the maintainers of this repository, using `workflow_dispatch` event +trigger. + +Once changes from a PR were reviewed and merged into the master branch, follow these steps: +1. Create a new Git Tag and push it to the repository, to the master branch. For example, + if you want to release a version 0.4.0 (note that we are using [Semantic Versioning](https://semver.org/)): + + ```shell + git tag -a v0.4.0 -m "v0.4.0" + git push origin v0.4.0 + ``` + +2. In GitHub UI, go to the repository's **Actions** -> **Release** -> **Run workflow**, and under **Use workflow from** + use **Tags** and find the tag you created in the previous step. + + > **Important note**: don't run the workflow against the master branch, but against the tag. + > `sbt-ci-release` plugin won't be able to correctly find tag, and it will think that you are trying + > to do a snapshot release, not an actual release that should be synchronized with Maven Central. diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index c63db3f63..64e683723 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import za.co.absa.atum.agent.model.MeasureResult diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index bf39f85f8..e1e4cab6d 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala index e0a864734..c21a45a61 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.core import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index d4154df7a..377294f9b 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model case class MeasureResult(measurement: Measurement, result: String) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala index 27250ab79..c1a4df86f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model import org.apache.spark.sql.functions._ diff --git a/build.sbt b/build.sbt index dc65d02cc..7afc24a68 100644 --- a/build.sbt +++ b/build.sbt @@ -4,6 +4,7 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at + * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -14,37 +15,74 @@ */ import Dependencies._ +import SparkVersionAxis._ +import JacocoSetup._ -ThisBuild / organization := "za.co.absa" +ThisBuild / organization := "za.co.absa.atum-service" lazy val scala211 = "2.11.12" lazy val scala212 = "2.12.12" +lazy val spark2 = "2.4.7" +lazy val spark3 = "3.3.1" + +ThisBuild / crossScalaVersions := Seq(scala211, scala212) +ThisBuild / scalaVersion := scala212 -Test / parallelExecution := false +ThisBuild / versionScheme := Some("early-semver") Global / onChangedBuildSource := ReloadOnSourceChanges -lazy val root = (project in file(".")) +lazy val printScalaVersion = taskKey[Unit]("Print Scala versions for atum-service is being built for.") + +ThisBuild / printScalaVersion := { + val log = streams.value.log + log.info(s"Building with Scala ${scalaVersion.value}") +} + +lazy val commonSettings = Seq( + libraryDependencies ++= commonDependencies, + scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + Test / parallelExecution := false +) + +lazy val parent = (project in file(".")) + .aggregate(atumServer.projectRefs ++ atumAgent.projectRefs: _*) + .settings( + name := "atum-service-parent", + publish / skip := true + ) + +lazy val atumAgent = (projectMatrix in file("agent")) .settings( - name := "atum-root", - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint") + commonSettings ++ Seq( + name := "atum-agent", + (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, + scalafmtOnCompile := true + ) ) + .enablePlugins(ScalafmtPlugin) + .sparkRow(SparkVersionAxis(spark2), scalaVersions = Seq(scala211, scala212)) + .sparkRow(SparkVersionAxis(spark3), scalaVersions = Seq(scala212)) -lazy val server = project +lazy val atumServer = (projectMatrix in file("server")) + .settings( + commonSettings ++ Seq( + name := "atum-server", + libraryDependencies ++= Dependencies.serverDependencies, + (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, + webappWebInfClasses := true, + inheritJarManifest := true + ): _* + ) .settings( - name := "atum-server", - scalaVersion := scala212, - libraryDependencies ++= Dependencies.serverDependencies, - webappWebInfClasses := true, - inheritJarManifest := true + jacocoReportSettings := jacocoSettings( scalaVersion.value, "atum-server"), + jacocoExcludes := jacocoProjectExcludes() ) .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) + .jvmPlatform(scalaVersions = Seq(scala212)) + + + -lazy val agent = project - .settings( - name := "atum-agent", - scalaVersion := scala212, - libraryDependencies ++= Dependencies.agentDependencies, - scalafmtOnCompile := true - ).enablePlugins(ScalafmtPlugin) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d5322d731..623c982f2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,8 +17,25 @@ import sbt._ object Dependencies { + private def limitVersion(version: String, parts: Int): String = { + version.split("\\.", parts + 1).take(parts).mkString(".") + } + + def getVersionUpToMinor(version: String): String = { + limitVersion(version, 2) + } + + def getVersionUpToMajor(version: String): String = { + limitVersion(version, 1) + } + + def commonDependencies: Seq[ModuleID] = Seq( + "org.scalatest" %% "scalatest" % "3.2.2" % Test, + "org.mockito" %% "mockito-scala" % "1.17.12" % Test + ) + + def serverDependencies: Seq[ModuleID] = { - val serverDependencies: Seq[ModuleID] = { val springVersion = "2.6.1" val springOrg = "org.springframework.boot" @@ -53,30 +70,24 @@ object Dependencies { ) } + def agentDependencies(sparkVersion: String): Seq[ModuleID] = { - val agentDependencies: Seq[ModuleID] = { - - val spark3Version = "3.3.2" - val scala212 = "2.12.12" - val scalatestVersion = "3.2.15" - val specs2Version = "4.19.2" - val typesafeConfigVersion = "1.4.2" - - lazy val sparkCore = "org.apache.spark" %% "spark-core" % spark3Version - - lazy val sparkCommons = "za.co.absa" % "spark-commons-spark3.3_2.12" % "0.5.0" - - lazy val sparkCommonsTest = "za.co.absa" %% "spark-commons-test" % "0.5.0" % Test - - lazy val sparkSql ="org.apache.spark" %% "spark-sql" % spark3Version - lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestVersion % Test - lazy val specs2core = "org.specs2" %% "specs2-core" % specs2Version % Test - lazy val typeSafeConfig = "com.typesafe" % "config" % typesafeConfigVersion + val typesafeVersion = "1.4.2" + val sparkCommonsVersion = "0.6.0" + val sparkMinorVersion = getVersionUpToMinor(sparkVersion) + val specs2CoreVersion = "4.19.2" + Seq( + "org.apache.spark" %% "spark-core" % sparkVersion % Provided, + "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, + "com.typesafe" % "config" % typesafeVersion, + "za.co.absa" %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion, + "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, + ) - Seq(sparkCore, sparkCommons, sparkCommonsTest, sparkSql, scalaTest, specs2core, typeSafeConfig) + } - }} +} diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala new file mode 100644 index 000000000..3395256af --- /dev/null +++ b/project/JacocoSetup.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.github.sbt.jacoco.JacocoKeys.JacocoReportFormats +import com.github.sbt.jacoco.report.JacocoReportSettings + +import java.time.format.DateTimeFormatter +import java.time.{ZoneId, ZonedDateTime} + +object JacocoSetup { + + private val jacocoReportCommonSettings: JacocoReportSettings = JacocoReportSettings( + formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) + ) + + def jacocoSettings(sparkVersion: String, scalaVersion: String, moduleName: String): JacocoReportSettings = { + val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) + val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `$moduleName` for spark:$sparkVersion - scala:$scalaVersion [$now]") + } + + def jacocoSettings(scalaVersion: String, moduleName: String): JacocoReportSettings = { + val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) + val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `$moduleName` for scala:$scalaVersion [$now]") + } + + def jacocoProjectExcludes(): Seq[String] = { + Seq() + } + +} diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala new file mode 100644 index 000000000..e548e4057 --- /dev/null +++ b/project/SparkVersionAxis.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sbt.{Def, VirtualAxis} +import sbt.Keys.{libraryDependencies, moduleName, name} +import sbt.internal.ProjectMatrix +import Dependencies.agentDependencies +import JacocoSetup.{jacocoProjectExcludes, jacocoSettings} +import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} + +case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { + + val sparkVersionMajor: String = Dependencies.getVersionUpToMajor(sparkVersion) + override val directorySuffix = s"-spark$sparkVersionMajor" + override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") +} + +object SparkVersionAxis { + private def camelCaseToLowerDashCase(origName: String): String = { + origName + .replaceAll("([A-Z])", "-$1") + .toLowerCase() + } + + implicit class ProjectExtension(val projectMatrix: ProjectMatrix) extends AnyVal { + + def sparkRow(sparkAxis: SparkVersionAxis, scalaVersions: Seq[String], settings: Def.SettingsDefinition*): ProjectMatrix = { + val sparkVersion = sparkAxis.sparkVersion + scalaVersions.foldLeft(projectMatrix)((currentProjectMatrix, scalaVersion) => + currentProjectMatrix.customRow( + scalaVersions = Seq(scalaVersion), + axisValues = Seq(sparkAxis, VirtualAxis.jvm), + _.settings( + moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix), + libraryDependencies ++= agentDependencies(sparkAxis.sparkVersion), + jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion, "atum-agent"), + jacocoExcludes := jacocoProjectExcludes() + ).settings(settings: _*) + ) + ) + } + } +} + diff --git a/project/build.properties b/project/build.properties index 40b3b8e7b..7a0a1ed1d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1,16 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + sbt.version=1.9.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index dbd38948a..be661ebd4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -17,7 +17,30 @@ addSbtPlugin("com.earldouglas" % "xsbt-web-plugin" % "4.2.4") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.7.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") +addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") +// To add release plugin +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") + +// sbt-jacoco dependency downloading +lazy val ow2Version = "9.5" +lazy val jacocoVersion = "0.8.11-absa.1" + +def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" + +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") + +addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) + +addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") +addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") diff --git a/publish.sbt b/publish.sbt index 23943f3a5..68766db9a 100644 --- a/publish.sbt +++ b/publish.sbt @@ -14,6 +14,47 @@ * limitations under the License. */ +ThisBuild / scmInfo := Some( + ScmInfo( + browseUrl = url("https://github.com/AbsaOSS/atum-service/tree/master"), + connection = "scm:git:git://github.com/AbsaOSS/atum-service.git", + devConnection = "scm:git:ssh://github.com/AbsaOSS/atum-service.git" + ) +) + +ThisBuild / developers := List( + Developer( + id = "Zejnilovic", + name = "Saša Zejnilović", + email = "sasa.zejnilovic@absa.africa", + url = url("https://github.com/Zejnilovic") + ), + Developer( + id = "benedeki", + name = "David Benedeki", + email = "david.benedeki@absa.africa", + url = url("https://github.com/benedeki") + ), + Developer( + id = "miroslavpojer", + name = "Miroslav Pojer", + email = "miroslav.pojer@absa.africa", + url = url("https://github.com/miroslavpojer") + ), + Developer( + id = "lsulak", + name = "Ladislav Sulak", + email = "ladislav.sulak@absa.africa", + url = url("https://github.com/lsulak") + ), + Developer( + id = "TebaleloS", + name = "Tebalelo Sekhula", + email = "tebalelo.sekhula@absa.africa", + url = url("https://github.com/TebaleloS") + ) +) + ThisBuild / organizationName := "ABSA Group Limited" ThisBuild / organizationHomepage := Some(url("https://www.absa.africa")) diff --git a/server/src/main/scala/za/co/absa/atum/web/api/package.scala b/server/src/main/scala/za/co/absa/atum/web/api/package.scala index 3d198a87e..56c2934e8 100644 --- a/server/src/main/scala/za/co/absa/atum/web/api/package.scala +++ b/server/src/main/scala/za/co/absa/atum/web/api/package.scala @@ -19,8 +19,8 @@ package za.co.absa.atum.web import java.util.Optional import java.util.concurrent.CompletableFuture import scala.concurrent.Future - import scala.compat.java8.FutureConverters._ +import scala.language.implicitConversions package object api { diff --git a/version.sbt b/version.sbt deleted file mode 100644 index a3a28d6cf..000000000 --- a/version.sbt +++ /dev/null @@ -1 +0,0 @@ -ThisBuild / version := "0.0.1-SNAPSHOT"