From da7282340608f30361e9f7d2594c4d20118a5900 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Jan 2022 19:31:46 +0000 Subject: [PATCH 01/95] Bump shelljs from 0.8.4 to 0.8.5 in /flowman-server-ui Bumps [shelljs](https://github.com/shelljs/shelljs) from 0.8.4 to 0.8.5. - [Release notes](https://github.com/shelljs/shelljs/releases) - [Changelog](https://github.com/shelljs/shelljs/blob/master/CHANGELOG.md) - [Commits](https://github.com/shelljs/shelljs/compare/v0.8.4...v0.8.5) --- updated-dependencies: - dependency-name: shelljs dependency-type: indirect ... Signed-off-by: dependabot[bot] --- flowman-server-ui/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flowman-server-ui/package-lock.json b/flowman-server-ui/package-lock.json index a7435d279..513447c1a 100644 --- a/flowman-server-ui/package-lock.json +++ b/flowman-server-ui/package-lock.json @@ -13576,9 +13576,9 @@ "dev": true }, "node_modules/shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "dependencies": { "glob": "^7.0.0", @@ -27880,9 +27880,9 @@ "dev": true }, "shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "requires": { "glob": "^7.0.0", From ca96b2af5c90cd14dc3d24f0547cc960248b1eda Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 28 Jan 2022 16:37:31 +0100 Subject: [PATCH 02/95] Update for next development version --- docker/pom.xml | 2 +- flowman-client/pom.xml | 2 +- flowman-common/pom.xml | 2 +- flowman-core/pom.xml | 2 +- flowman-dist/pom.xml | 2 +- flowman-dsl/pom.xml | 2 +- flowman-hub/pom.xml | 2 +- flowman-parent/pom.xml | 2 +- flowman-plugins/aws/pom.xml | 2 +- flowman-plugins/azure/pom.xml | 2 +- flowman-plugins/delta/pom.xml | 2 +- flowman-plugins/impala/pom.xml | 2 +- flowman-plugins/json/pom.xml | 2 +- flowman-plugins/kafka/pom.xml | 2 +- flowman-plugins/mariadb/pom.xml | 2 +- flowman-plugins/mssqlserver/pom.xml | 2 +- flowman-plugins/mysql/pom.xml | 2 +- flowman-plugins/openapi/pom.xml | 2 +- flowman-plugins/swagger/pom.xml | 2 +- flowman-scalatest-compat/pom.xml | 2 +- flowman-server-ui/pom.xml | 2 +- flowman-server/pom.xml | 2 +- flowman-spark-extensions/pom.xml | 2 +- flowman-spark-testing/pom.xml | 2 +- flowman-spec/pom.xml | 2 +- flowman-studio-ui/pom.xml | 2 +- flowman-studio/pom.xml | 2 +- flowman-testing/pom.xml | 2 +- flowman-tools/pom.xml | 2 +- pom.xml | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index 215e56311..058a705eb 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-client/pom.xml b/flowman-client/pom.xml index 34560b135..cf3d9614e 100644 --- a/flowman-client/pom.xml +++ b/flowman-client/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-common/pom.xml b/flowman-common/pom.xml index be15351d6..cd1112f72 100644 --- a/flowman-common/pom.xml +++ b/flowman-common/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-core/pom.xml b/flowman-core/pom.xml index d7d196e80..21f606b68 100644 --- a/flowman-core/pom.xml +++ b/flowman-core/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-dist/pom.xml b/flowman-dist/pom.xml index b7ae5ddd8..358d4dae1 100644 --- a/flowman-dist/pom.xml +++ b/flowman-dist/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-dsl/pom.xml b/flowman-dsl/pom.xml index 9f05c9891..78bd6f133 100644 --- a/flowman-dsl/pom.xml +++ b/flowman-dsl/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-hub/pom.xml b/flowman-hub/pom.xml index af6f160ec..ffef1a3f3 100644 --- a/flowman-hub/pom.xml +++ b/flowman-hub/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-parent/pom.xml b/flowman-parent/pom.xml index 9dd5f540f..0abd89f7d 100644 --- a/flowman-parent/pom.xml +++ b/flowman-parent/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-plugins/aws/pom.xml b/flowman-plugins/aws/pom.xml index c50019433..80306b290 100644 --- a/flowman-plugins/aws/pom.xml +++ b/flowman-plugins/aws/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/azure/pom.xml b/flowman-plugins/azure/pom.xml index 3939100dd..a0dac5f24 100644 --- a/flowman-plugins/azure/pom.xml +++ b/flowman-plugins/azure/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/delta/pom.xml b/flowman-plugins/delta/pom.xml index 18e1d1abc..7cf72bfd8 100644 --- a/flowman-plugins/delta/pom.xml +++ b/flowman-plugins/delta/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/impala/pom.xml b/flowman-plugins/impala/pom.xml index 7cb2969c6..420ae0bfc 100644 --- a/flowman-plugins/impala/pom.xml +++ b/flowman-plugins/impala/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/json/pom.xml b/flowman-plugins/json/pom.xml index fc0d5c863..874b9fb9d 100644 --- a/flowman-plugins/json/pom.xml +++ b/flowman-plugins/json/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/kafka/pom.xml b/flowman-plugins/kafka/pom.xml index 28d0f3eb8..3447e8e3b 100644 --- a/flowman-plugins/kafka/pom.xml +++ b/flowman-plugins/kafka/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mariadb/pom.xml b/flowman-plugins/mariadb/pom.xml index 3d0321fe1..526b59d44 100644 --- a/flowman-plugins/mariadb/pom.xml +++ b/flowman-plugins/mariadb/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index a2caaabb1..5d0111b93 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mysql/pom.xml b/flowman-plugins/mysql/pom.xml index ca161e973..791741adc 100644 --- a/flowman-plugins/mysql/pom.xml +++ b/flowman-plugins/mysql/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/openapi/pom.xml b/flowman-plugins/openapi/pom.xml index de43703a8..2df8fdb62 100644 --- a/flowman-plugins/openapi/pom.xml +++ b/flowman-plugins/openapi/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/swagger/pom.xml b/flowman-plugins/swagger/pom.xml index 490df8902..7b4b2323c 100644 --- a/flowman-plugins/swagger/pom.xml +++ b/flowman-plugins/swagger/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../../pom.xml diff --git a/flowman-scalatest-compat/pom.xml b/flowman-scalatest-compat/pom.xml index c7c0931b1..a660600fe 100644 --- a/flowman-scalatest-compat/pom.xml +++ b/flowman-scalatest-compat/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-server-ui/pom.xml b/flowman-server-ui/pom.xml index 9bda56927..e03cbd525 100644 --- a/flowman-server-ui/pom.xml +++ b/flowman-server-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-server/pom.xml b/flowman-server/pom.xml index 117c4fcc5..8bcaf18bb 100644 --- a/flowman-server/pom.xml +++ b/flowman-server/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-spark-extensions/pom.xml b/flowman-spark-extensions/pom.xml index 71f51e441..6d606958e 100644 --- a/flowman-spark-extensions/pom.xml +++ b/flowman-spark-extensions/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-spark-testing/pom.xml b/flowman-spark-testing/pom.xml index be280c407..1a64bb373 100644 --- a/flowman-spark-testing/pom.xml +++ b/flowman-spark-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-spec/pom.xml b/flowman-spec/pom.xml index 9046bbeda..868bbe48a 100644 --- a/flowman-spec/pom.xml +++ b/flowman-spec/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-studio-ui/pom.xml b/flowman-studio-ui/pom.xml index 51cfee95e..dad5d6711 100644 --- a/flowman-studio-ui/pom.xml +++ b/flowman-studio-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-studio/pom.xml b/flowman-studio/pom.xml index ab2ab9fd5..a79414a6b 100644 --- a/flowman-studio/pom.xml +++ b/flowman-studio/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-testing/pom.xml b/flowman-testing/pom.xml index b2e3f6869..5e89263dc 100644 --- a/flowman-testing/pom.xml +++ b/flowman-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/flowman-tools/pom.xml b/flowman-tools/pom.xml index e04344483..7e3bdeadf 100644 --- a/flowman-tools/pom.xml +++ b/flowman-tools/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 6315c494d..e6add6094 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.dimajix.flowman flowman-root - 0.21.1 + 0.21.2-SNAPSHOT pom Flowman root pom A Spark based ETL tool From a7e8a3a7f6cfd9ebc38eb3d08efe9fc46d68b773 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 29 Jan 2022 12:28:55 +0100 Subject: [PATCH 03/95] Improve weather example --- docs/spec/relation/file.md | 11 ++++++++--- examples/weather/model/aggregates.yml | 6 ++++-- examples/weather/model/measurements.yml | 1 - .../dimajix/flowman/spec/relation/FileRelation.scala | 5 +++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/spec/relation/file.md b/docs/spec/relation/file.md index 8f5393227..c69b0a048 100644 --- a/docs/spec/relation/file.md +++ b/docs/spec/relation/file.md @@ -12,7 +12,9 @@ relations: format: "csv" # Specify the base directory where all data is stored. This location does not include the partition pattern location: "${export_dir}" - # Specify the pattern how to identify files and/or partitions. This pattern is relative to the `location` + # You could specify the pattern how to identify files and/or partitions. This pattern is relative to the `location`. + # Actually, it is highly recommended NOT to explicitly specify a partition pattern for outgoing relations + # and let Spark generate this according to the Hive standard. pattern: "${export_pattern}" # Set format specific options options: @@ -67,7 +69,7 @@ relations: column has a name and a type and optionally a granularity. Normally the partition columns are separate from the schema, but you *may* also include the partition column in the schema, although this is not considered to be best practice. But it turns out to be quite useful in combination with dynamically writing to multiple partitions. - + * `pattern` **(optional)** *(string)* *(default: empty)*: This field specifies the directory and/or file name pattern to access specific partitions. Please see the section [Partitioning](#Partitioning) below. @@ -124,7 +126,10 @@ in all situations where only schema information is required. ### Partitioning -Flowman also supports partitioning, i.e. written to different sub directories. +Flowman also supports partitioning, i.e. written to different sub directories. You can explicitly specify a *partition +pattern* via the `pattern` field, but it is highly recommended to NOT explicitly set this field and let Spark manage +partitions itself. This way Spark can infer partition values from directory names and will also list directories more +efficiently. ### Writing to Dynamic Partitions diff --git a/examples/weather/model/aggregates.yml b/examples/weather/model/aggregates.yml index aba6ea933..a5c405c24 100644 --- a/examples/weather/model/aggregates.yml +++ b/examples/weather/model/aggregates.yml @@ -5,8 +5,10 @@ relations: format: parquet # Specify the base directory where all data is stored. This location does not include the partition pattern location: "$basedir/aggregates/" - # Specify the pattern how to identify files and/or partitions. This pattern is relative to the `location` - pattern: "${year}" + # You could specify the pattern how to identify files and/or partitions. This pattern is relative to the `location`. + # Actually, it is highly recommended NOT to explicitly specify a partition pattern for outgoing relations + # and let Spark generate this according to the Hive standard. + #pattern: "${year}" # Add partition column, which can be used in the `pattern` partitions: - name: year diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index 8e069aa1a..7a3157ed5 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -3,7 +3,6 @@ relations: kind: file format: parquet location: "$basedir/measurements/" - pattern: "${year}" partitions: - name: year type: integer diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala index e5bda9971..c4e2a812c 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala @@ -197,8 +197,9 @@ case class FileRelation( appendPartitionColumns(df1) } private def readSpark(execution:Execution, partitions:Map[String,FieldValue]) : DataFrame = { - val df = this.reader(execution, format, options) - .load(qualifiedLocation.toString) + val reader = this.reader(execution, format, options) + val reader1 = if (execution.fs.file(qualifiedLocation).isDirectory()) reader.option("basePath", qualifiedLocation.toString) else reader + val df = reader1.load(qualifiedLocation.toString) // Filter partitions val parts = MapIgnoreCase(this.partitions.map(p => p.name -> p)) From 8e98855509aab8cbb54f276bda5d81dff51600db Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 31 Jan 2022 12:13:16 +0100 Subject: [PATCH 04/95] Fix example configuration for History Server --- docker/conf/history-server.yml | 20 +++++++++++++++++++ flowman-dist/conf/history-server.yml.template | 4 ---- 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 docker/conf/history-server.yml diff --git a/docker/conf/history-server.yml b/docker/conf/history-server.yml new file mode 100644 index 000000000..6e2b18221 --- /dev/null +++ b/docker/conf/history-server.yml @@ -0,0 +1,20 @@ +# The following definition provides a "run history" stored in a database. If nothing else is specified, the database +# is stored locally as a Derby database. If you do not want to use the history, you can simply remove the whole +# 'history' block from this file. +history: + kind: jdbc + connection: flowman_state + retries: 3 + timeout: 1000 + +connections: + flowman_state: + driver: $System.getenv('FLOWMAN_LOGDB_DRIVER', 'org.apache.derby.jdbc.EmbeddedDriver') + url: $System.getenv('FLOWMAN_LOGDB_URL', $String.concat('jdbc:derby:', $System.getenv('FLOWMAN_HOME'), '/logdb;create=true')) + username: $System.getenv('FLOWMAN_LOGDB_USER', '') + password: $System.getenv('FLOWMAN_LOGDB_PASSWORD', '') + +plugins: + - flowman-mariadb + - flowman-mysql + - flowman-mssqlserver diff --git a/flowman-dist/conf/history-server.yml.template b/flowman-dist/conf/history-server.yml.template index 90df9b7ce..eab638f25 100644 --- a/flowman-dist/conf/history-server.yml.template +++ b/flowman-dist/conf/history-server.yml.template @@ -16,10 +16,6 @@ connections: password: $System.getenv('FLOWMAN_HISTORY_PASSWORD', '') -# This section contains global configuration properties. -config: - - # This section enables plugins. You may want to remove plugins which are of no use for you. plugins: - flowman-mariadb From caf4ccc20e4ed1d532c43c66d37b807d3c60a69d Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 31 Jan 2022 18:32:43 +0100 Subject: [PATCH 05/95] Fix missing metrics and graphs in Flowman History --- .../main/scala/com/dimajix/flowman/model/Metadata.scala | 3 +++ .../src/main/scala/com/dimajix/flowman/spec/Spec.scala | 1 - .../com/dimajix/flowman/spec/assertion/AssertionSpec.scala | 1 + .../dimajix/flowman/spec/connection/ConnectionSpec.scala | 3 +++ .../scala/com/dimajix/flowman/spec/hook/HookSpec.scala | 2 +- .../main/scala/com/dimajix/flowman/spec/job/JobSpec.scala | 2 +- .../com/dimajix/flowman/spec/mapping/MappingSpec.scala | 7 ++++--- .../com/dimajix/flowman/spec/measure/MeasureSpec.scala | 1 + .../com/dimajix/flowman/spec/relation/RelationSpec.scala | 1 + .../scala/com/dimajix/flowman/spec/target/TargetSpec.scala | 1 + .../com/dimajix/flowman/spec/template/TemplateSpec.scala | 1 + .../scala/com/dimajix/flowman/spec/test/TestSpec.scala | 2 +- 12 files changed, 18 insertions(+), 7 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala index 4b99e475f..b61e71e84 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala @@ -40,6 +40,9 @@ final case class Metadata( kind: String, labels: Map[String,String] = Map() ) { + require(name != null) + require(category != null && category.nonEmpty) + require(kind != null && kind.nonEmpty) def asMap : Map[String,String] = { Map( "name" -> name, diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Spec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Spec.scala index f62951613..2ceadecfc 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Spec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Spec.scala @@ -62,7 +62,6 @@ final class MetadataSpec { abstract class NamedSpec[T] extends Spec[T] { - @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="name", required = false) protected[spec] var name:String = "" @JsonProperty(value="metadata", required=false) protected var metadata:Option[MetadataSpec] = None diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/assertion/AssertionSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/assertion/AssertionSpec.scala index 5f7671923..a2a3f4b82 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/assertion/AssertionSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/assertion/AssertionSpec.scala @@ -47,6 +47,7 @@ object AssertionSpec extends TypeRegistry[AssertionSpec] { new JsonSubTypes.Type(name = "uniqueKey", value = classOf[UniqueKeyAssertionSpec]) )) abstract class AssertionSpec extends NamedSpec[Assertion] { + @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="description", required = false) private var description: Option[String] = None override def instantiate(context: Context): Assertion diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/connection/ConnectionSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/connection/ConnectionSpec.scala index 711ca1293..e07284754 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/connection/ConnectionSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/connection/ConnectionSpec.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.spec.connection +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import com.fasterxml.jackson.databind.annotation.JsonTypeResolver @@ -46,6 +47,8 @@ object ConnectionSpec extends TypeRegistry[ConnectionSpec] { new JsonSubTypes.Type(name = "sftp", value = classOf[SshConnectionSpec]) )) abstract class ConnectionSpec extends NamedSpec[Connection] { + @JsonProperty(value="kind", required = true) protected var kind: String = "jdbc" + /** * Creates an instance of this specification and performs the interpolation of all variables * diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/HookSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/HookSpec.scala index a468debb8..677f3ac94 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/HookSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/HookSpec.scala @@ -34,7 +34,7 @@ object HookSpec extends TypeRegistry[HookSpec] { } -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind", visible=true) @JsonSubTypes(value = Array( new JsonSubTypes.Type(name = "simpleReport", value = classOf[SimpleReportHookSpec]), new JsonSubTypes.Type(name = "report", value = classOf[ReportHookSpec]), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/job/JobSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/job/JobSpec.scala index 82496851d..a41f7da75 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/job/JobSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/job/JobSpec.scala @@ -94,7 +94,7 @@ final class JobSpec extends NamedSpec[Job] { val name = context.evaluate(this.name) Job.Properties( context, - metadata.map(_.instantiate(context, name, Category.JOB, kind)).getOrElse(Metadata(context, name, Category.JOB, kind)), + metadata.map(_.instantiate(context, name, Category.JOB, "job")).getOrElse(Metadata(context, name, Category.JOB, "job")), description.map(context.evaluate) ) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala index 9d4d6e2f7..b01f91223 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala @@ -89,9 +89,10 @@ object MappingSpec extends TypeRegistry[MappingSpec] { new JsonSubTypes.Type(name = "values", value = classOf[ValuesMappingSpec]) )) abstract class MappingSpec extends NamedSpec[Mapping] { - @JsonProperty("broadcast") protected var broadcast:String = "false" - @JsonProperty("checkpoint") protected var checkpoint:String = "false" - @JsonProperty("cache") protected var cache:String = "NONE" + @JsonProperty(value="kind", required = true) protected var kind: String = _ + @JsonProperty(value="broadcast", required = false) protected var broadcast:String = "false" + @JsonProperty(value="checkpoint", required = false) protected var checkpoint:String = "false" + @JsonProperty(value="cache", required = false) protected var cache:String = "NONE" /** * Creates an instance of this specification and performs the interpolation of all variables diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/measure/MeasureSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/measure/MeasureSpec.scala index 86c403173..730f55613 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/measure/MeasureSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/measure/MeasureSpec.scala @@ -43,6 +43,7 @@ object MeasureSpec extends TypeRegistry[MeasureSpec] { new JsonSubTypes.Type(name = "sql", value = classOf[SqlMeasureSpec]) )) abstract class MeasureSpec extends NamedSpec[Measure] { + @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="description", required = false) private var description: Option[String] = None override def instantiate(context: Context): Measure diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala index 091705e8f..72bb0f402 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala @@ -61,6 +61,7 @@ object RelationSpec extends TypeRegistry[RelationSpec] { new JsonSubTypes.Type(name = "view", value = classOf[HiveViewRelationSpec]) )) abstract class RelationSpec extends NamedSpec[Relation] { + @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="description", required = false) private var description: Option[String] = None override def instantiate(context:Context) : Relation diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala index dedd38218..e10888626 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala @@ -67,6 +67,7 @@ object TargetSpec extends TypeRegistry[TargetSpec] { new JsonSubTypes.Type(name = "verify", value = classOf[VerifyTargetSpec]) )) abstract class TargetSpec extends NamedSpec[Target] { + @JsonProperty(value = "kind", required = true) protected var kind: String = _ @JsonProperty(value = "before", required=false) protected[spec] var before:Seq[String] = Seq() @JsonProperty(value = "after", required=false) protected[spec] var after:Seq[String] = Seq() diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/template/TemplateSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/template/TemplateSpec.scala index df88f3385..a167b07b7 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/template/TemplateSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/template/TemplateSpec.scala @@ -83,6 +83,7 @@ object TemplateSpec { new JsonSubTypes.Type(name = "target", value = classOf[TargetTemplateSpec]) )) abstract class TemplateSpec extends NamedSpec[Template[_]] { + @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="parameters", required=false) protected var parameters : Seq[TemplateSpec.Parameter] = Seq() protected def instanceProperties(context:Context) : Template.Properties = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/test/TestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/test/TestSpec.scala index 3ba4bf846..b45b8a3b4 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/test/TestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/test/TestSpec.scala @@ -80,7 +80,7 @@ class TestSpec extends NamedSpec[Test] { val name = context.evaluate(this.name) Test.Properties( context, - metadata.map(_.instantiate(context, name, Category.TEST, kind)).getOrElse(Metadata(context, name, Category.TEST, kind)), + metadata.map(_.instantiate(context, name, Category.TEST, "test")).getOrElse(Metadata(context, name, Category.TEST, "test")), description.map(context.evaluate) ) } From 3c8c5dfd4d9d5c42269531c12b5db8d69d9ea7fe Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 31 Jan 2022 19:51:46 +0100 Subject: [PATCH 06/95] Fix build --- CHANGELOG.md | 6 + docs/spec/relation/sqlserver.md | 128 ++++++++++++++++++ .../com/dimajix/flowman/model/Metadata.scala | 2 +- flowman-plugins/mssqlserver/pom.xml | 88 +++++++++++- .../src/main/assembly/assembly.xml | 10 +- .../mssqlserver/src/main/resources/plugin.yml | 1 + .../spec/relation/SqlServerRelation.scala | 99 ++++++++++++++ .../spec/relation/SqlServerRelationTest.scala | 73 ++++++++++ .../flowman/spec/relation/JdbcRelation.scala | 12 +- .../flowman/spec/schema/SchemaSpec.scala | 2 +- .../flowman/spec/target/NullTarget.scala | 3 +- .../flowman/spec/schema/SchemaTest.scala | 12 +- 12 files changed, 414 insertions(+), 22 deletions(-) create mode 100644 docs/spec/relation/sqlserver.md create mode 100644 flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala create mode 100644 flowman-plugins/mssqlserver/src/test/scala/com/dimajix/flowman/spec/relation/SqlServerRelationTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index d1342466d..b9f2eccae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Version 0.22.0 + +* Add new 'sqlserver' relation + + + # Version 0.21.1 - 2022-01-28 * flowexec now returns different exit codes depending on the processing result diff --git a/docs/spec/relation/sqlserver.md b/docs/spec/relation/sqlserver.md new file mode 100644 index 000000000..3ecc0865a --- /dev/null +++ b/docs/spec/relation/sqlserver.md @@ -0,0 +1,128 @@ +# SQL Server Relations + +The SQL Server relation allows you to access MS SQL Server and Azure SQL databases using a JDBC driver. It uses the +`spark-sql-connector` from Microsoft to speed up processing. + + +## Example + +```yaml +# First specify a connection. This can be used by multiple SQL Server relations +connections: + frontend: + kind: jdbc + url: "$frontend_db_url" + username: "$frontend_db_username" + password: "$frontend_db_password" + +relations: + frontend_users: + kind: sqlserver + # Specify the name of the connection to use + connection: frontend + # Specify the table + table: "users" + schema: + kind: avro + file: "${project.basedir}/schema/users.avsc" + primaryKey: + - user_id +``` +It is also possible to directly embed the connection as follows: +```yaml +relations: + frontend_users: + kind: sqlserver + # Specify the name of the connection to use + connection: + kind: jdbc + url: "$frontend_db_url" + username: "$frontend_db_username" + password: "$frontend_db_password" + # Specify the table + table: "users" +``` +For most cases, it is recommended not to embed the connection, since this prevents reusing the same connection in +multiple places. + + +## Fields + * `kind` **(mandatory)** *(type: string)*: `jdbc` + + * `schema` **(optional)** *(type: schema)* *(default: empty)*: + Explicitly specifies the schema of the JDBC source. Alternatively Flowman will automatically + try to infer the schema. + + * `primaryKey` **(optional)** *(type: list)* *(default: empty)*: +List of columns which form the primary key. This will be used when Flowman creates the table, and this will also be used +as the fallback for merge/upsert operations, when no `mergeKey` and no explicit merge condition is specified. + + * `mergeKey` **(optional)** *(type: list)* *(default: empty)*: + List of columns which will be used as default condition for merge and upsert operations. The main difference to + `primaryKey` is that these columns will not be used as a primary key for creating the table. + + * `description` **(optional)** *(type: string)* *(default: empty)*: + A description of the relation. This is purely for informational purpose. + + * `connection` **(mandatory)** *(type: string)*: + The *connection* field specifies the name of a [JDBC Connection](../connection/jdbc.md) + object which has to be defined elsewhere. + + * `database` **(optional)** *(type: string)* *(default: empty)*: + Defines the Hive database where the table is defined. When no database is specified, the table is accessed without any +specific qualification, meaning that the default database will be used or the one specified in the connection. + + * `table` **(mandatory)** *(type: string)*: + Specifies the name of the table in the relational database. + + * `properties` **(optional)** *(type: map:string)* *(default: empty)*: + Specifies any additional properties passed to the JDBC connection. Note that both the JDBC + relation and the JDBC connection can define properties. So it is advisable to define all + common properties in the connection and more table specific properties in the relation. + The connection properties are applied first, then the relation properties. This means that + a relation property can overwrite a connection property if it has the same name. + + +## Automatic Migrations +Flowman supports some automatic migrations, specifically with the migration strategies `ALTER`, `ALTER_REPLACE` +and `REPLACE` (those can be set via the global config variable `flowman.default.relation.migrationStrategy`, +see [configuration](../../config.md) for more details). + +The migration strategy `ALTER` supports the following alterations for JDBC relations: +* Changing nullability +* Adding new columns +* Dropping columns +* Changing the column type + +Note that although Flowman will try to apply these changes, not all SQL databases support all of these changes in +all variations. Therefore it may well be the case, that the SQL database will fail performing these changes. If +the migration strategy is set to `ALTER_REPLACE`, then Flowman will fall back to trying to replace the whole table +altogether on *any* non-recoverable exception during migration. + + +## Schema Conversion +The JDBC relation fully supports automatic schema conversion on input and output operations as described in the +corresponding section of [relations](index.md). + + +## Output Modes +The `jdbc` relation supports the following output modes in a [`relation` target](../target/relation.md): + +| Output Mode | Supported | Comments | +|---------------------|-----------|-------------------------------------------------------| +| `errorIfExists` | yes | Throw an error if the JDBC table already exists | +| `ignoreIfExists` | yes | Do nothing if the JDBC table already exists | +| `overwrite` | yes | Overwrite the whole table or the specified partitions | +| `overwrite_dynamic` | no | - | +| `append` | yes | Append new records to the existing table | +| `update` | no | - | +| `merge` | no | - | + + +## Remarks + +Note that Flowman will rely on schema inference in some important situations, like [mocking](mock.md) and generally +for describing the schema of a relation. This might create unwanted connections to the physical data source, +particular in case of self-contained tests. To prevent Flowman from creating a connection to the physical data +source, you simply need to explicitly specify a schema, which will then be used instead of the physical schema +in all situations where only schema information is required. diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala index b61e71e84..513172d5d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Metadata.scala @@ -42,7 +42,7 @@ final case class Metadata( ) { require(name != null) require(category != null && category.nonEmpty) - require(kind != null && kind.nonEmpty) + require(kind != null) def asMap : Map[String,String] = { Map( "name" -> name, diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index 5d0111b93..c7ebecae8 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -18,8 +18,48 @@ ${project.version} ${project.build.finalName}.jar 9.2.1.jre8 + 1.2.0 + + + CDH-6.3 + + 1.0.2 + + + + CDP-7.1 + + 1.0.2 + + + + spark-2.4 + + 1.0.2 + + + + spark-3.0 + + 1.1.0 + + + + spark-3.1 + + 1.2.0 + + + + spark-3.2 + + 1.2.0 + + + + @@ -28,6 +68,14 @@ + + net.alchim31.maven + scala-maven-plugin + + + org.scalatest + scalatest-maven-plugin + org.apache.maven.plugins maven-assembly-plugin @@ -38,8 +86,38 @@ com.dimajix.flowman - flowman-core - provided + flowman-spec + + + + com.dimajix.flowman + flowman-spark-testing + test + + + + org.apache.spark + spark-core_${scala.api_version} + + + + org.apache.spark + spark-sql_${scala.api_version} + + + + org.apache.hadoop + hadoop-client + + + + org.apache.spark + spark-hive_${scala.api_version} + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml @@ -47,6 +125,12 @@ mssql-jdbc ${mssqlserver-java-client.version} + + + com.microsoft.azure + spark-mssql-connector_${scala.api_version} + ${spark-mssql-connector.version} + diff --git a/flowman-plugins/mssqlserver/src/main/assembly/assembly.xml b/flowman-plugins/mssqlserver/src/main/assembly/assembly.xml index 9dc35b6db..89fea5c4e 100644 --- a/flowman-plugins/mssqlserver/src/main/assembly/assembly.xml +++ b/flowman-plugins/mssqlserver/src/main/assembly/assembly.xml @@ -22,11 +22,17 @@ plugins/${plugin.name} - false - false + true + true false runtime true + + com.dimajix.flowman:flowman-spec + org.scala-lang.modules:scala-collection-compat_${scala.api_version} + org.apache.hadoop:hadoop-client-api + org.apache.hadoop:hadoop-client-runtime + diff --git a/flowman-plugins/mssqlserver/src/main/resources/plugin.yml b/flowman-plugins/mssqlserver/src/main/resources/plugin.yml index e2f82485b..02c34fb9e 100644 --- a/flowman-plugins/mssqlserver/src/main/resources/plugin.yml +++ b/flowman-plugins/mssqlserver/src/main/resources/plugin.yml @@ -3,4 +3,5 @@ description: ${project.name} version: ${plugin.version} isolation: false jars: + - ${plugin.jar} - mssql-jdbc-${mssqlserver-java-client.version}.jar diff --git a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala new file mode 100644 index 000000000..057ca7e59 --- /dev/null +++ b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.relation + +import scala.collection.mutable + +import com.fasterxml.jackson.annotation.JsonProperty +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions + +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.model.Connection +import com.dimajix.flowman.model.PartitionField +import com.dimajix.flowman.model.Reference +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.Schema +import com.dimajix.flowman.spec.annotation.RelationType +import com.dimajix.flowman.spec.connection.ConnectionReferenceSpec +import com.dimajix.flowman.spec.connection.JdbcConnection + + +class SqlServerRelation( + instanceProperties:Relation.Properties, + schema:Option[Schema] = None, + partitions: Seq[PartitionField] = Seq(), + connection: Reference[Connection], + properties: Map[String,String] = Map(), + database: Option[String] = None, + table: Option[String] = None, + query: Option[String] = None, + mergeKey: Seq[String] = Seq(), + primaryKey: Seq[String] = Seq() +) extends JdbcRelation(instanceProperties, schema, partitions, connection, properties, database, table, query, mergeKey, primaryKey) { + override protected def doWrite(execution: Execution, df:DataFrame): Unit = { + val (_,props) = createConnectionProperties() + this.writer(execution, df, "com.microsoft.sqlserver.jdbc.spark", Map(), SaveMode.Append) + .options(props) + .option(JDBCOptions.JDBC_TABLE_NAME, tableIdentifier.unquotedString) + .save() + } + + override protected def createConnectionProperties() : (String,Map[String,String]) = { + val connection = this.connection.value.asInstanceOf[JdbcConnection] + val props = mutable.Map[String,String]() + props.put(JDBCOptions.JDBC_URL, connection.url) + props.put(JDBCOptions.JDBC_DRIVER_CLASS, "com.microsoft.sqlserver.jdbc.SQLServerDriver") + connection.username.foreach(props.put("user", _)) + connection.password.foreach(props.put("password", _)) + + connection.properties.foreach(kv => props.put(kv._1, kv._2)) + properties.foreach(kv => props.put(kv._1, kv._2)) + + (connection.url,props.toMap) + } +} + + + +@RelationType(kind="sqlserver") +class SqlServerRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec { + @JsonProperty(value = "connection", required = true) private var connection: ConnectionReferenceSpec = _ + @JsonProperty(value = "properties", required = false) private var properties: Map[String, String] = Map() + @JsonProperty(value = "database", required = false) private var database: Option[String] = None + @JsonProperty(value = "table", required = false) private var table: Option[String] = None + @JsonProperty(value = "query", required = false) private var query: Option[String] = None + @JsonProperty(value = "mergeKey", required = false) private var mergeKey: Seq[String] = Seq() + @JsonProperty(value = "primaryKey", required = false) private var primaryKey: Seq[String] = Seq() + + override def instantiate(context: Context): SqlServerRelation = { + new SqlServerRelation( + instanceProperties(context), + schema.map(_.instantiate(context)), + partitions.map(_.instantiate(context)), + connection.instantiate(context), + context.evaluate(properties), + database.map(context.evaluate).filter(_.nonEmpty), + table.map(context.evaluate).filter(_.nonEmpty), + query.map(context.evaluate).filter(_.nonEmpty), + mergeKey.map(context.evaluate), + primaryKey.map(context.evaluate) + ) + } +} diff --git a/flowman-plugins/mssqlserver/src/test/scala/com/dimajix/flowman/spec/relation/SqlServerRelationTest.scala b/flowman-plugins/mssqlserver/src/test/scala/com/dimajix/flowman/spec/relation/SqlServerRelationTest.scala new file mode 100644 index 000000000..d2ee55a74 --- /dev/null +++ b/flowman-plugins/mssqlserver/src/test/scala/com/dimajix/flowman/spec/relation/SqlServerRelationTest.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.relation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.model.ConnectionIdentifier +import com.dimajix.flowman.model.Schema +import com.dimajix.flowman.model.ValueConnectionReference +import com.dimajix.flowman.spec.ObjectMapper +import com.dimajix.flowman.spec.schema.EmbeddedSchema +import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.IntegerType +import com.dimajix.flowman.types.StringType + + +class SqlServerRelationTest extends AnyFlatSpec with Matchers { + "The JdbcRelation" should "support embedding the connection" in { + val spec = + s""" + |kind: sqlserver + |name: some_relation + |description: "This is a test table" + |connection: + | kind: jdbc + | name: some_connection + | url: some_url + |table: lala_001 + |schema: + | kind: inline + | fields: + | - name: str_col + | type: string + | - name: int_col + | type: integer + """.stripMargin + + val relationSpec = ObjectMapper.parse[RelationSpec](spec).asInstanceOf[SqlServerRelationSpec] + + val session = Session.builder().disableSpark().build() + val context = session.context + + val relation = relationSpec.instantiate(context) + relation shouldBe a[SqlServerRelation] + relation.name should be ("some_relation") + relation.schema should be (Some(EmbeddedSchema( + Schema.Properties(context, name="embedded", kind="inline"), + fields = Seq( + Field("str_col", StringType), + Field("int_col", IntegerType) + ) + ))) + relation.connection shouldBe a[ValueConnectionReference] + relation.connection.identifier should be (ConnectionIdentifier("some_connection")) + relation.connection.name should be ("some_connection") + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 70f46b4d8..69f845b29 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -168,7 +168,7 @@ case class JdbcRelation( require(partitions != null) // Get Connection - val (_,props) = createProperties() + val (_,props) = createConnectionProperties() // Read from database. We do not use this.reader, because Spark JDBC sources do not support explicit schemas val reader = execution.spark.read @@ -247,8 +247,8 @@ case class JdbcRelation( "Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") } } - private def doWrite(execution: Execution, df:DataFrame): Unit = { - val (_,props) = createProperties() + protected def doWrite(execution: Execution, df:DataFrame): Unit = { + val (_,props) = createConnectionProperties() this.writer(execution, df, "jdbc", Map(), SaveMode.Append) .options(props) .option(JDBCOptions.JDBC_TABLE_NAME, tableIdentifier.unquotedString) @@ -290,7 +290,7 @@ case class JdbcRelation( val sourceColumns = collectColumns(mergeCondition.expr, "source") ++ clauses.flatMap(c => collectColumns(df.schema, c, "source")) val sourceDf = df.select(sourceColumns.toSeq.map(col):_*) - val (url, props) = createProperties() + val (url, props) = createConnectionProperties() val options = new JDBCOptions(url, tableIdentifier.unquotedString, props) val targetSchema = outputSchema(execution) JdbcUtils.mergeTable(tableIdentifier, "target", targetSchema, sourceDf, "source", mergeCondition, clauses, options) @@ -553,7 +553,7 @@ case class JdbcRelation( } } - private def createProperties() : (String,Map[String,String]) = { + protected def createConnectionProperties() : (String,Map[String,String]) = { val connection = this.connection.value.asInstanceOf[JdbcConnection] val props = mutable.Map[String,String]() props.put(JDBCOptions.JDBC_URL, connection.url) @@ -568,7 +568,7 @@ case class JdbcRelation( } private def withConnection[T](fn:(java.sql.Connection,JDBCOptions) => T) : T = { - val (url,props) = createProperties() + val (url,props) = createConnectionProperties() logger.debug(s"Connecting to jdbc source at $url") val options = new JDBCOptions(url, tableIdentifier.unquotedString, props) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/SchemaSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/SchemaSpec.scala index ba9e7b160..353fda254 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/SchemaSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/SchemaSpec.scala @@ -51,7 +51,7 @@ object SchemaSpec extends TypeRegistry[SchemaSpec] { new JsonSubTypes.Type(name = "union", value = classOf[UnionSchemaSpec]) )) abstract class SchemaSpec extends Spec[Schema] { - @JsonProperty(value="kind", required = true) protected var kind: String = _ + @JsonProperty(value="kind", required = true) protected var kind: String = "inline" override def instantiate(context:Context) : Schema diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/NullTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/NullTarget.scala index 0b950ec0a..440168c09 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/NullTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/NullTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -68,6 +68,7 @@ object NullTargetSpec { val spec = new NullTargetSpec spec.name = name spec.partition = partition + spec.kind = "null" spec } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/schema/SchemaTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/schema/SchemaTest.scala index 0174cd74d..b66a1c394 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/schema/SchemaTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/schema/SchemaTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,21 +16,15 @@ package com.dimajix.flowman.spec.schema -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.execution.RootContext +import com.dimajix.flowman.spec.ObjectMapper class SchemaTest extends AnyFlatSpec with Matchers { - lazy val mapper = { - val mapper = new ObjectMapper(new YAMLFactory()) - mapper.registerModule(DefaultScalaModule) - mapper - } + lazy val mapper = ObjectMapper.mapper "A Schema" should "default to the embedded schema" in { val spec = From 0c188df8635cbd33c5c7b5874600bd0814fbc477 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Feb 2022 06:58:53 +0100 Subject: [PATCH 07/95] Fix build for Spark 2.4 --- flowman-plugins/mssqlserver/pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index c7ebecae8..ec506414c 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -19,6 +19,7 @@ ${project.build.finalName}.jar 9.2.1.jre8 1.2.0 + _${scala.api_version} @@ -26,36 +27,42 @@ CDH-6.3 1.0.2 + CDP-7.1 1.0.2 + spark-2.4 1.0.2 + spark-3.0 1.1.0 + _${scala.api_version} spark-3.1 1.2.0 + _${scala.api_version} spark-3.2 1.2.0 + _${scala.api_version} @@ -128,7 +135,7 @@ com.microsoft.azure - spark-mssql-connector_${scala.api_version} + spark-mssql-connector${spark-mssql-connector.suffix} ${spark-mssql-connector.version} From 9504e6137c0796c2f6d26bf9386e725aca3444a8 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Feb 2022 17:13:27 +0100 Subject: [PATCH 08/95] Fix documentation --- docs/spec/assertion/schema.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/spec/assertion/schema.md b/docs/spec/assertion/schema.md index 1d21bce01..3ba9ac637 100644 --- a/docs/spec/assertion/schema.md +++ b/docs/spec/assertion/schema.md @@ -17,7 +17,7 @@ columns: ```yaml kind: schema mapping: some_mapping -ignoreNullable: false +ignoreNullability: false schema: kind: inline fields: @@ -35,7 +35,7 @@ targets: assertions: - kind: schema mapping: some_mapping - ignoreNullable: true + ignoreNullability: true ignoreCase: false ignoreOrder: true schema: From 36ff6b58ca16767b6b292ead429bef2f79115ee2 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Feb 2022 18:46:11 +0100 Subject: [PATCH 09/95] Update mssql plugin documentation --- docs/plugins/mssql.md | 3 +++ docs/spec/relation/sqlserver.md | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/docs/plugins/mssql.md b/docs/plugins/mssql.md index e456e271f..c2b2b5eb4 100644 --- a/docs/plugins/mssql.md +++ b/docs/plugins/mssql.md @@ -1 +1,4 @@ # MS SQL Server Plugin + +## Provided Entities +* [`sqlserver` relation](../spec/relation/sqlserver.md) diff --git a/docs/spec/relation/sqlserver.md b/docs/spec/relation/sqlserver.md index 3ecc0865a..7988e4a52 100644 --- a/docs/spec/relation/sqlserver.md +++ b/docs/spec/relation/sqlserver.md @@ -3,6 +3,11 @@ The SQL Server relation allows you to access MS SQL Server and Azure SQL databases using a JDBC driver. It uses the `spark-sql-connector` from Microsoft to speed up processing. +## Plugin + +This relation type is provided as part of the [`flowman-mssql` plugin](../../plugins/mssql.md), which needs to be enabled +in your `namespace.yml` file. See [namespace documentation](../namespace.md) for more information for configuring plugins. + ## Example From ac31a995730a8c218d59c7deca80b08ac959e3ab Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 3 Feb 2022 08:46:22 +0100 Subject: [PATCH 10/95] Fix inheritance of ProjectResolver in derived RootContext --- .../com/dimajix/flowman/execution/AbstractContext.scala | 4 ++-- .../com/dimajix/flowman/execution/ProjectContext.scala | 2 +- .../scala/com/dimajix/flowman/execution/RootContext.scala | 8 ++++---- .../com/dimajix/flowman/execution/ScopeContext.scala | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractContext.scala index 3225ccda9..92abc2bed 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractContext.scala @@ -31,7 +31,7 @@ import com.dimajix.flowman.model.Prototype object AbstractContext { - abstract class Builder[B <: Builder[B,C], C <: Context](parent:Context, defaultSettingLevel:SettingLevel) { this:B => + abstract class Builder[B <: Builder[B,C], C <: Context](parent:Option[Context], defaultSettingLevel:SettingLevel) { this:B => private var _environment = Seq[(String,Any,SettingLevel)]() private var _config = Seq[(String,String,SettingLevel)]() private var _connections = Seq[(String, Prototype[Connection], SettingLevel)]() @@ -48,7 +48,7 @@ object AbstractContext { val rawConnections = mutable.Map[String, (Prototype[Connection], Int)]() // Fetch environment from parent - if (parent != null) { + parent.foreach { parent => parent.rawEnvironment.foreach(kv => rawEnvironment.update(kv._1, kv._2)) parent.rawConfig.foreach(kv => rawConfig.update(kv._1, kv._2)) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ProjectContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ProjectContext.scala index 0ced97fe4..b9c7a8add 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ProjectContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ProjectContext.scala @@ -43,7 +43,7 @@ import com.dimajix.flowman.model.TestIdentifier object ProjectContext { - class Builder private[ProjectContext](parent:Context, project:Project) extends AbstractContext.Builder[Builder,ProjectContext](parent, SettingLevel.PROJECT_SETTING) { + class Builder private[ProjectContext](parent:Context, project:Project) extends AbstractContext.Builder[Builder,ProjectContext](Some(parent), SettingLevel.PROJECT_SETTING) { require(parent != null) require(project != null) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala index 199931139..9a3f9e9c7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala @@ -48,8 +48,8 @@ import com.dimajix.flowman.model.TestIdentifier object RootContext { - class Builder private[RootContext](namespace:Option[Namespace], profiles:Set[String], parent:Context = null) extends AbstractContext.Builder[Builder,RootContext](parent, SettingLevel.NAMESPACE_SETTING) { - private var projectResolver:Option[String => Option[Project]] = None + class Builder private[RootContext](namespace:Option[Namespace], profiles:Set[String], parent:Option[Context] = None) extends AbstractContext.Builder[Builder,RootContext](parent, SettingLevel.NAMESPACE_SETTING) { + private var projectResolver:Option[String => Option[Project]] = parent.map(_.root).flatMap(_.projectResolver) private var overrideMappings:Map[MappingIdentifier, Prototype[Mapping]] = Map() private var overrideRelations:Map[RelationIdentifier, Prototype[Relation]] = Map() private var execution:Option[Execution] = None @@ -107,13 +107,13 @@ object RootContext { def builder() = new Builder(None, Set()) def builder(namespace:Namespace) = new Builder(Some(namespace), Set()) def builder(namespace:Option[Namespace], profiles:Set[String]) = new Builder(namespace, profiles) - def builder(parent:Context) = new Builder(parent.namespace, Set(), parent) + def builder(parent:Context) = new Builder(parent.namespace, Set(), Some(parent)) } final class RootContext private[execution]( _namespace:Option[Namespace], - projectResolver:Option[String => Option[Project]], + private val projectResolver:Option[String => Option[Project]], _profiles:Set[String], _env:Map[String,(Any, Int)], _config:Map[String,(String, Int)], diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ScopeContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ScopeContext.scala index 9d707f69c..32208db4a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ScopeContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ScopeContext.scala @@ -41,7 +41,7 @@ import com.dimajix.flowman.model.TestIdentifier object ScopeContext { - class Builder(parent:Context) extends AbstractContext.Builder[Builder,ScopeContext](parent, SettingLevel.SCOPE_OVERRIDE) { + class Builder(parent:Context) extends AbstractContext.Builder[Builder,ScopeContext](Some(parent), SettingLevel.SCOPE_OVERRIDE) { require(parent != null) private var mappings = Map[String, Prototype[Mapping]]() From b6ab21a5de845da6a4d501568cb88cd93b5fa9a3 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 3 Feb 2022 09:43:26 +0100 Subject: [PATCH 11/95] Add project cache to Flowman session --- .../com/dimajix/flowman/execution/Session.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala index 8fb8dfc24..66066baca 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.execution +import scala.collection.mutable + import org.apache.hadoop.conf.{Configuration => HadoopConf} import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession @@ -400,11 +402,15 @@ class Session private[execution]( private val rootExecution : RootExecution = new RootExecution(this) private val operationsManager = new OperationManager - private lazy val rootContext : RootContext = { - def loadProject(name:String) : Option[Project] = { - Some(store.loadProject(name)) + private val cachedProjects : mutable.Map[String,Project] = mutable.Map() + private def loadProject(name:String) : Option[Project] = { + val project = cachedProjects.synchronized { + cachedProjects.getOrElseUpdate(name, store.loadProject(name)) } + Some(project) + } + private lazy val rootContext : RootContext = { val builder = RootContext.builder(_namespace, _profiles) .withEnvironment(_environment, SettingLevel.GLOBAL_OVERRIDE) .withConfig(_config, SettingLevel.GLOBAL_OVERRIDE) From c533a05d65df6210b48ae58ac004e80cee4718cd Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 4 Feb 2022 17:18:37 +0100 Subject: [PATCH 12/95] Improve errors and exceptions in ProjectTransformer --- .../dimajix/flowman/transforms/ProjectTransformer.scala | 8 ++++++-- .../scala/com/dimajix/flowman/transforms/exceptions.scala | 6 ++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/ProjectTransformer.scala b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/ProjectTransformer.scala index 19726a05d..6ad1857d9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/ProjectTransformer.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/ProjectTransformer.scala @@ -46,7 +46,9 @@ final case class ProjectTransformer( val tree = ColumnTree.ofSchema(df.schema) def col(spec:ProjectTransformer.Column) = { - val input = tree.find(spec.column).get.mkValue() + val input = tree.find(spec.column) + .getOrElse(throw new NoSuchColumnException(spec.column.toString)) + .mkValue() val typed = spec.dtype match { case None => input case Some(ft) => input.cast(ft.sparkType) @@ -71,7 +73,9 @@ final case class ProjectTransformer( val tree = SchemaTree.ofSchema(schema) def col(spec:ProjectTransformer.Column) = { - val input = tree.find(spec.column).get.mkValue() + val input = tree.find(spec.column) + .getOrElse(throw new NoSuchColumnException(spec.column.toString)) + .mkValue() val typed = spec.dtype match { case None => input case Some(ft) => input.copy(ftype = ft) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/exceptions.scala b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/exceptions.scala index cf259f34b..961192fd5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/exceptions.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/exceptions.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,7 @@ package com.dimajix.flowman.transforms class AnalysisException(val message: String,val cause: Option[Throwable] = None) - extends Exception(message, cause.orNull) { - -} + extends IllegalArgumentException(message, cause.orNull) class NoSuchColumnException(column:String) extends AnalysisException(s"Column '$column' not found") From d43b6a39c77e43114bfd2fa434a0f49c4768a0d7 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 7 Feb 2022 13:15:15 +0100 Subject: [PATCH 13/95] Apply filter first before droppging columns in DropMapping --- .../dimajix/flowman/spec/mapping/DropMapping.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala index f2f505a6d..4531fc80d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala @@ -56,13 +56,14 @@ case class DropMapping( require(deps != null) val df = deps(input) - val asm = assembler - val result = asm.reassemble(df) - // Apply optional filter - val filteredResult = filter.map(result.filter).getOrElse(result) + // Apply optional filter, before dropping columns! + val filtered = filter.map(df.filter).getOrElse(df) + + val asm = assembler + val result = asm.reassemble(filtered) - Map("main" -> filteredResult) + Map("main" -> result) } /** @@ -83,7 +84,7 @@ case class DropMapping( private def assembler : Assembler = { val builder = Assembler.builder() - .columns(_.drop(columns)) + .columns(_.drop(columns)) builder.build() } } From afe766de630cccc823014881ba341830a79d77db Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 8 Feb 2022 20:45:37 +0100 Subject: [PATCH 14/95] Initial version of documentation subsystem --- docs/index.md | 1 + .../{cookbook/testing.md => testing/index.md} | 2 +- ...com.dimajix.flowman.spi.ColumnTestExecutor | 1 + ...com.dimajix.flowman.spi.SchemaTestExecutor | 1 + .../flowman/documentation/Category.scala | 46 +++++++ .../flowman/documentation/Collector.scala | 25 ++++ .../flowman/documentation/ColumnDoc.scala | 78 +++++++++++ .../flowman/documentation/ColumnTest.scala | 87 ++++++++++++ .../flowman/documentation/Documenter.scala | 56 ++++++++ .../flowman/documentation/EntityDoc.scala | 21 +++ .../documentation/ExecuteTestCollector.scala | 26 ++++ .../flowman/documentation/Fragment.scala | 58 ++++++++ .../flowman/documentation/Generator.scala | 30 +++++ .../documentation/MappingCollector.scala | 87 ++++++++++++ .../flowman/documentation/MappingDoc.scala | 94 +++++++++++++ .../flowman/documentation/ProjectDoc.scala | 63 +++++++++ .../flowman/documentation/Reference.scala | 23 ++++ .../documentation/RelationCollector.scala | 63 +++++++++ .../flowman/documentation/RelationDoc.scala | 59 ++++++++ .../flowman/documentation/SchemaDoc.scala | 91 +++++++++++++ .../flowman/documentation/SchemaTest.scala | 47 +++++++ .../documentation/TargetCollector.scala | 60 +++++++++ .../flowman/documentation/TargetDoc.scala | 80 +++++++++++ .../flowman/documentation/TestResult.scala | 49 +++++++ .../flowman/documentation/velocity.scala | 70 ++++++++++ .../dimajix/flowman/execution/Runner.scala | 45 +++---- .../dimajix/flowman/execution/Session.scala | 1 + .../com/dimajix/flowman/graph/Graph.scala | 2 + .../com/dimajix/flowman/model/Mapping.scala | 19 ++- .../com/dimajix/flowman/model/Relation.scala | 19 ++- .../com/dimajix/flowman/model/Schema.scala | 10 +- .../com/dimajix/flowman/model/Target.scala | 19 ++- .../{templating.scala => velocity.scala} | 0 .../flowman/spi/ColumnTestExecutor.scala | 40 ++++++ .../flowman/spi/SchemaTestExecutor.scala | 39 ++++++ .../spec/annotation/ColumnTestType.java | 37 +++++ .../spec/annotation/GeneratorType.java | 37 +++++ .../spec/annotation/SchemaTestType.java | 37 +++++ ...dimajix.flowman.spi.ClassAnnotationHandler | 3 + .../flowman/documentation/text/project.vtl | 15 +++ .../dimajix/flowman/spec/ObjectMapper.scala | 9 ++ .../spec/documentation/ColumnDocSpec.scala | 53 ++++++++ .../spec/documentation/ColumnTestSpec.scala | 69 ++++++++++ .../spec/documentation/FileGenerator.scala | 54 ++++++++ .../spec/documentation/GeneratorSpec.scala | 48 +++++++ .../spec/documentation/MappingDocSpec.scala | 126 ++++++++++++++++++ .../spec/documentation/RelationDocSpec.scala | 68 ++++++++++ .../spec/documentation/SchemaDocSpec.scala | 47 +++++++ .../spec/documentation/SchemaTestSpec.scala | 49 +++++++ .../spec/documentation/TargetDocSpec.scala | 41 ++++++ .../documentation/TemplateGenerator.scala | 49 +++++++ .../flowman/spec/mapping/MappingSpec.scala | 5 +- .../spec/mapping/TemplateMapping.scala | 7 + .../flowman/spec/relation/RelationSpec.scala | 5 +- .../spec/relation/TemplateRelation.scala | 7 + .../flowman/spec/target/TargetSpec.scala | 5 +- .../flowman/spec/target/TemplateTarget.scala | 7 + .../spec/documentation/ColumnTestTest.scala | 44 ++++++ .../spec/documentation/MappingDocTest.scala | 51 +++++++ .../spec/documentation/RelationDocTest.scala | 49 +++++++ .../flowman/tools/exec/Arguments.scala | 4 +- .../documentation/DocumentationCommand.scala | 34 +++++ .../exec/documentation/GenerateCommand.scala | 90 +++++++++++++ .../tools/exec/project/ProjectCommand.scala | 2 - .../flowman/tools/shell/ParsedCommand.scala | 4 +- 65 files changed, 2429 insertions(+), 39 deletions(-) rename docs/{cookbook/testing.md => testing/index.md} (99%) create mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor create mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Category.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Collector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/EntityDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Fragment.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Generator.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala rename flowman-core/src/main/scala/com/dimajix/flowman/model/{templating.scala => velocity.scala} (100%) create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala create mode 100644 flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java create mode 100644 flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/GeneratorType.java create mode 100644 flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java create mode 100644 flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/GeneratorSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala create mode 100644 flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumentationCommand.scala create mode 100644 flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala diff --git a/docs/index.md b/docs/index.md index bbddbb24b..ba6b0cc7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -96,6 +96,7 @@ Flowman also provides optional plugins which extend functionality. You can find installation lifecycle spec/index + testing/index cli/index history-server/index cookbook/index diff --git a/docs/cookbook/testing.md b/docs/testing/index.md similarity index 99% rename from docs/cookbook/testing.md rename to docs/testing/index.md index fd2c4687d..e8d542b51 100644 --- a/docs/cookbook/testing.md +++ b/docs/testing/index.md @@ -1,4 +1,4 @@ -# Testing +# Testing with Flowman Testing data pipelines often turns out to be a difficult undertaking, since the pipeline relies on external data sources which need to be mocked. Fortunately, Flowman natively supports writing `tests`, which in turn support simple diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor new file mode 100644 index 000000000..da5f5a7af --- /dev/null +++ b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor @@ -0,0 +1 @@ +com.dimajix.flowman.documentation.DefaultColumnTestExecutor diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor new file mode 100644 index 000000000..5e1404ad8 --- /dev/null +++ b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor @@ -0,0 +1 @@ +com.dimajix.flowman.documentation.DefaultSchemaTestExecutor diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Category.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Category.scala new file mode 100644 index 000000000..ec005a106 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Category.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2018-2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import java.util.Locale + + +sealed abstract class Category extends Product with Serializable { + def lower : String = toString.toLowerCase(Locale.ROOT) + def upper : String = toString.toUpperCase(Locale.ROOT) +} + +object Category { + case object PROJECT extends Category + case object COLUMN extends Category + case object MAPPING extends Category + case object RELATION extends Category + case object SCHEMA extends Category + case object TARGET extends Category + + def ofString(category:String) : Category = { + category.toLowerCase(Locale.ROOT) match { + case "column" => COLUMN + case "mapping" => MAPPING + case "project" => PROJECT + case "relation" => RELATION + case "schema" => SCHEMA + case "target" => TARGET + case _ => throw new IllegalArgumentException(s"No such category $category") + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Collector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Collector.scala new file mode 100644 index 000000000..87c330215 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Collector.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph + + +abstract class Collector { + def collect(execution: Execution, graph:Graph, documentation:ProjectDoc) : ProjectDoc +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala new file mode 100644 index 000000000..8b67f6e66 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.common.MapIgnoreCase +import com.dimajix.flowman.types.Field + + + +final case class ColumnReference( + override val parent:Option[Reference], + name:String +) extends Reference + +object ColumnDoc { + def merge(thisCols:Seq[ColumnDoc], otherCols:Seq[ColumnDoc]) :Seq[ColumnDoc] = { + val thisColsByName = MapIgnoreCase(thisCols.map(c => c.name -> c)) + val otherColsByName = MapIgnoreCase(otherCols.map(c => c.name -> c)) + val mergedColumns = thisCols.map { column => + otherColsByName.get(column.name) match { + case Some(other) => column.merge(other) + case None => column + } + } + mergedColumns ++ otherCols.filter(c => !thisColsByName.contains(c.name)) + } + +} +final case class ColumnDoc( + parent:Option[Reference], + name:String, + field:Field, + description:Option[String], + children:Seq[ColumnDoc], + tests:Seq[ColumnTest] +) extends EntityDoc { + override def reference: ColumnReference = ColumnReference(parent, name) + override def fragments: Seq[Fragment] = children + override def reparent(parent: Reference): ColumnDoc = { + val ref = ColumnReference(Some(parent), name) + copy( + parent = Some(parent), + children = children.map(_.reparent(ref)), + tests = tests.map(_.reparent(ref)) + ) + } + + def nullable : Boolean = field.nullable + def typeName : String = field.typeName + def sqlType : String = field.sqlType + def sparkType : String = field.sparkType.sql + def catalogType : String = field.catalogType.sql + + def merge(other:ColumnDoc) : ColumnDoc = { + val childs = + if (this.children.nonEmpty && other.children.nonEmpty) + ColumnDoc.merge(children, other.children) + else + this.children ++ other.children + val desc = description.orElse(description) + val tsts = tests ++ other.tests + copy(description=desc, children=childs, tests=tsts) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala new file mode 100644 index 000000000..cde1b7869 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -0,0 +1,87 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.apache.spark.sql.DataFrame + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.spi.ColumnTestExecutor + + +final case class ColumnTestReference( + override val parent:Option[Reference] +) extends Reference + + +abstract class ColumnTest extends Fragment with Product with Serializable { + def result : Option[TestResult] + def withResult(result:TestResult) : ColumnTest + + override def reparent(parent: Reference): ColumnTest + + override def parent: Some[Reference] + override def reference: ColumnTestReference = ColumnTestReference(parent) + override def fragments: Seq[Fragment] = result.toSeq +} + + +case class NotNullColumnTest( + parent:Some[Reference], + description: Option[String] = None, + result:Option[TestResult] = None +) extends ColumnTest { + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} + +case class UniqueColumnTest( + parent:Some[Reference], + description: Option[String] = None, + result:Option[TestResult] = None +) extends ColumnTest { + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) +} + +case class RangeColumnTest( + parent:Some[Reference], + description: Option[String] = None, + result:Option[TestResult] = None +) extends ColumnTest { + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) +} + +case class ValuesColumnTest( + parent:Some[Reference], + description: Option[String] = None, + result:Option[TestResult] = None +) extends ColumnTest { + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) +} + +//case class ForeignKeyColumnTest() extends ColumnTest +//case class ExpressionColumnTest() extends ColumnTest + + +class DefaultColumnTestExecutor extends ColumnTestExecutor { + override def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = ??? +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala new file mode 100644 index 000000000..80615ed03 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Job +import com.dimajix.flowman.model.Project + + +case class Documenter( + collectors:Seq[Collector], + generators:Seq[Generator] +) { + def execute(session:Session, job:Job, args:Map[String,Any]) : Unit = { + val runner = session.runner + runner.withExecution(isolated=true) { execution => + runner.withJobContext(job, args, Some(execution)) { (context, arguments) => + execute(context, execution, job.project.get) + } + } + } + private def execute(context:Context, execution: Execution, project:Project) : Unit = { + // 1. Get Project documentation + val projectDoc = ProjectDoc( + project.name, + project.version + ) + + // 2. Apply all other collectors + val graph = Graph.ofProject(context, project, Phase.BUILD) + val finalDoc = collectors.foldLeft(projectDoc)((doc, collector) => collector.collect(execution, graph, doc)) + + // 3. Generate documentation + generators.foreach { gen => + gen.generate(context, execution, finalDoc) + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/EntityDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/EntityDoc.scala new file mode 100644 index 000000000..b9551bc3d --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/EntityDoc.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +abstract class EntityDoc extends Fragment { + +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala new file mode 100644 index 000000000..434fbfd23 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Project + + +class ExecuteTestCollector extends Collector { + override def collect(execution: Execution, graph:Graph, documentation:ProjectDoc) : ProjectDoc = ??? +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Fragment.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Fragment.scala new file mode 100644 index 000000000..e0fb2bab2 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Fragment.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + + +/** + * A [[Fragment]] represents a piece of documentation. The full documentation then is a tree structure build from many + * fragments. + */ +abstract class Fragment { + /** + * Optional textual description of the fragment to be shown in the documentation + * @return + */ + def description : Option[String] + + /** + * A resolvable reference to the fragment itself + * @return + */ + def reference : Reference + + /** + * A reference to the parent of this [[Fragment]] + * @return + */ + def parent : Option[Reference] + + /** + * List of child fragments + * @return + */ + def fragments : Seq[Fragment] + + def reparent(parent:Reference) : Fragment + + def resolve(path:Seq[Reference]) : Option[Fragment] = { + path match { + case head :: tail => + fragments.find(_.reference == head).flatMap(_.resolve(tail)) + case Nil => Some(this) + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Generator.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Generator.scala new file mode 100644 index 000000000..0b4d1d331 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Generator.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution + + +abstract class Generator { + def generate(context:Context, execution: Execution, documentation: ProjectDoc) : Unit +} + + +abstract class BaseGenerator extends Generator { + +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala new file mode 100644 index 000000000..aaae4a008 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -0,0 +1,87 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import scala.collection.mutable + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.types.StructType + + +class MappingCollector extends Collector { + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { + val mappings = mutable.Map[MappingIdentifier, MappingDoc]() + val parent = documentation.reference + + def getMappingDoc(mapping:Mapping) : MappingDoc = { + mappings.getOrElseUpdate(mapping.identifier, genDoc(mapping)) + } + def getOutputDoc(mappingOutput:MappingOutputIdentifier) : Option[MappingOutputDoc] = { + val mapping = mappingOutput.mapping + val doc = mappings.getOrElseUpdate(mapping, genDoc(graph.mapping(mapping).mapping)) + val output = mappingOutput.output + doc.outputs.find(_.identifier.output == output) + } + def genDoc(mapping:Mapping) : MappingDoc = { + val inputs = mapping.inputs.flatMap(in => getOutputDoc(in).map(in -> _)).toMap + document(execution, parent, mapping, inputs) + } + + val docs = graph.mappings.map { mapping => + mapping.mapping.identifier -> getMappingDoc(mapping.mapping) + }.toMap + + documentation.copy(mappings=docs) + } + + /** + * Generates a documentation for this mapping + * @param execution + * @param parent + * @param inputs + * @return + */ + private def document(execution: Execution, parent:Reference, mapping:Mapping, inputs:Map[MappingOutputIdentifier,MappingOutputDoc]) : MappingDoc = { + val inputSchemas = inputs.map(kv => kv._1 -> kv._2.schema.map(_.toStruct).getOrElse(StructType(Seq()))) + val schemas = mapping.describe(execution, inputSchemas) + val doc = MappingDoc( + Some(parent), + mapping.identifier, + None, + inputs.map(_._2.reference).toSeq, + Seq() + ) + val ref = doc.reference + + val outputs = schemas.map { case(output,schema) => + val doc = MappingOutputDoc( + Some(ref), + MappingOutputIdentifier(mapping.identifier, output), + None, + None + ) + val schemaDoc = SchemaDoc.ofStruct(doc.reference, schema) + doc.copy(schema = Some(schemaDoc)) + } + + doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala new file mode 100644 index 000000000..6164422d0 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier + + +final case class MappingOutputReference( + override val parent:Option[Reference], + name:String +) extends Reference + + +final case class MappingOutputDoc( + parent:Some[Reference], + identifier: MappingOutputIdentifier, + description: Option[String], + schema:Option[SchemaDoc] +) extends Fragment { + override def reference: Reference = MappingOutputReference(parent, identifier.output) + override def fragments: Seq[Fragment] = schema.toSeq + override def reparent(parent: Reference): MappingOutputDoc = { + val ref = MappingOutputReference(Some(parent), identifier.output) + copy( + parent=Some(parent), + schema=schema.map(_.reparent(ref)) + ) + } + + def merge(other:MappingOutputDoc) : MappingOutputDoc = { + val id = if (identifier.mapping.isEmpty) other.identifier else identifier + val desc = other.description.orElse(this.description) + val schm = schema.map(_.merge(other.schema)).orElse(other.schema) + val result = copy(identifier=id, description=desc, schema=schm) + parent.orElse(other.parent) + .map(result.reparent) + .getOrElse(result) + } +} + + +final case class MappingReference( + override val parent:Option[Reference], + name:String +) extends Reference + + +final case class MappingDoc( + parent:Option[Reference], + identifier:MappingIdentifier, + description:Option[String], + inputs:Seq[Reference], + outputs:Seq[MappingOutputDoc] +) extends EntityDoc { + override def reference: MappingReference = MappingReference(parent, identifier.name) + override def fragments: Seq[Fragment] = outputs + override def reparent(parent: Reference): MappingDoc = { + val ref = MappingOutputReference(Some(parent), identifier.name) + copy( + parent=Some(parent), + outputs=outputs.map(_.reparent(ref)) + ) + } + + def merge(other:Option[MappingDoc]) : MappingDoc = other.map(merge).getOrElse(this) + def merge(other:MappingDoc) : MappingDoc = { + val id = if (identifier.isEmpty) other.identifier else identifier + val desc = other.description.orElse(this.description) + val in = inputs.toSet ++ other.inputs.toSet + val out = outputs.map { out => + other.outputs.find(_.identifier.output == out.identifier.output).map(out.merge).getOrElse(out) + } ++ + other.outputs.filter(out => !outputs.exists(_.identifier.output == out.identifier.output)) + val result = copy(identifier=id, description=desc, inputs=in.toSeq, outputs=out) + parent.orElse(other.parent) + .map(result.reparent) + .getOrElse(result) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala new file mode 100644 index 000000000..ce8c725f2 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.TargetIdentifier + + +final case class ProjectReference( + name:String +) extends Reference { + override def parent: Option[Reference] = None +} + + +final case class ProjectDoc( + name: String, + version: Option[String], + description: Option[String] = None, + targets:Map[TargetIdentifier,TargetDoc] = Map(), + relations:Map[RelationIdentifier,RelationDoc] = Map(), + mappings:Map[MappingIdentifier,MappingDoc] = Map() +) extends EntityDoc { + override def reference: Reference = ProjectReference(name) + override def parent: Option[Reference] = None + override def fragments: Seq[Fragment] = (targets.values ++ relations.values ++ mappings.values).toSeq + + override def resolve(path:Seq[Reference]) : Option[Fragment] = { + if (path.isEmpty) + Some(this) + else + None + } + + override def reparent(parent: Reference): ProjectDoc = ??? + + def resolve(ref:Reference) : Option[Fragment] = { + ref.path match { + case head :: tail => + if (head != reference) + None + else + resolve(head).flatMap(_.resolve(tail)) + case Nil => + None + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala new file mode 100644 index 000000000..e27e86503 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + + +abstract class Reference extends Product with Serializable { + def parent : Option[Reference] + def path : Seq[Reference] = parent.toSeq.flatMap(_.path) :+ this +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala new file mode 100644 index 000000000..46961b0f6 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Relation + + +class RelationCollector extends Collector { + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { + val parent = documentation.reference + val docs = graph.relations.map(t => t.relation.identifier -> document(execution, parent, t.relation)).toMap + documentation.copy(relations = docs) + } + + + /** + * Create a documentation for the relation. + * @param execution + * @param parent + * @return + */ + private def document(execution:Execution, parent:Reference, relation:Relation) : RelationDoc = { + val doc = RelationDoc( + Some(parent), + relation.identifier, + relation.description, + None, + relation.provides.toSeq, + Map() + ) + val ref = doc.reference + + val desc = SchemaDoc.ofStruct(ref, relation.describe(execution)) + val schemaDoc = relation.schema.map { schema => + val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) + SchemaDoc( + Some(ref), + schema.description, + fieldsDoc.columns, + Seq() + ) + } + val mergedDoc = desc.merge(schemaDoc) + + doc.copy(schema = Some(mergedDoc)).merge(relation.documentation) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala new file mode 100644 index 000000000..33f9bdea7 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.types.FieldValue + + +final case class RelationReference( + parent:Option[Reference], + name:String +) extends Reference + + +final case class RelationDoc( + parent:Option[Reference], + identifier:RelationIdentifier, + description:Option[String], + schema:Option[SchemaDoc], + provides:Seq[ResourceIdentifier], + partitions:Map[String,FieldValue] = Map() +) extends EntityDoc { + override def reference: RelationReference = RelationReference(parent, identifier.name) + override def fragments: Seq[Fragment] = schema.toSeq + override def reparent(parent: Reference): RelationDoc = { + val ref = RelationReference(Some(parent), identifier.name) + copy( + parent = Some(parent), + schema = schema.map(_.reparent(ref)) + ) + } + + def merge(other:Option[RelationDoc]) : RelationDoc = other.map(merge).getOrElse(this) + def merge(other:RelationDoc) : RelationDoc = { + val id = if (identifier.isEmpty) other.identifier else identifier + val desc = other.description.orElse(this.description) + val schm = schema.map(_.merge(other.schema)).orElse(other.schema) + val prov = provides.toSet ++ other.provides.toSet + val result = copy(identifier=id, description=desc, schema=schm, provides=prov.toSeq) + parent.orElse(other.parent) + .map(result.reparent) + .getOrElse(result) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala new file mode 100644 index 000000000..30151cb89 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -0,0 +1,91 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.types.ArrayType +import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.FieldType +import com.dimajix.flowman.types.MapType +import com.dimajix.flowman.types.StructType + + +final case class SchemaReference( + override val parent:Option[Reference] +) extends Reference + + +object SchemaDoc { + def ofStruct(parent:Reference, struct:StructType) : SchemaDoc = ofFields(parent, struct.fields) + def ofFields(parent:Reference, fields:Seq[Field]) : SchemaDoc = { + val doc = SchemaDoc(Some(parent), None, Seq(), Seq()) + + def genColumns(parent:Reference, fields:Seq[Field]) : Seq[ColumnDoc] = { + fields.map(f => genColumn(parent, f)) + } + def genChildren(parent:Reference, ftype:FieldType) : Seq[ColumnDoc] = { + ftype match { + case s:StructType => + genColumns(parent, s.fields) + case m:MapType => + genChildren(parent, m.valueType) + case a:ArrayType => + genChildren(parent, a.elementType) + case _ => + Seq() + } + + } + def genColumn(parent:Reference, field:Field) : ColumnDoc = { + val doc = ColumnDoc(Some(parent), field.name, field, field.description, Seq(), Seq()) + val children = genChildren(doc.reference, field.ftype) + doc.copy(children = children) + } + val columns = genColumns(doc.reference, fields) + doc.copy(columns = columns) + } +} + + +final case class SchemaDoc( + parent:Option[Reference], + description:Option[String], + columns:Seq[ColumnDoc], + tests:Seq[SchemaTest] +) extends EntityDoc { + override def reference: SchemaReference = SchemaReference(parent) + override def fragments: Seq[Fragment] = columns ++ tests + override def reparent(parent: Reference): SchemaDoc = { + val ref = SchemaReference(Some(parent)) + copy( + parent = Some(parent), + columns = columns.map(_.reparent(ref)), + tests = tests.map(_.reparent(ref)) + ) + } + + def toStruct : StructType = StructType(columns.map(_.field)) + def merge(other:Option[SchemaDoc]) : SchemaDoc = other.map(merge).getOrElse(this) + def merge(other:SchemaDoc) : SchemaDoc = { + val desc = other.description.orElse(this.description) + val tsts = tests ++ other.tests + val cols = ColumnDoc.merge(columns, other.columns) + val result = copy(description=desc, columns=cols, tests=tsts) + parent.orElse(other.parent) + .map(result.reparent) + .getOrElse(result) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala new file mode 100644 index 000000000..71e2ae596 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.apache.spark.sql.DataFrame + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.spi.SchemaTestExecutor + + +final case class SchemaTestReference( + override val parent:Option[SchemaReference] +) extends Reference + + +sealed abstract class SchemaTest extends Fragment with Product with Serializable { + def result : Option[TestResult] + def withResult(result:TestResult) : SchemaTest + + override def parent: Option[SchemaReference] + override def reference: SchemaTestReference = SchemaTestReference(parent) + override def fragments: Seq[Fragment] = result.toSeq + override def reparent(parent: Reference): SchemaTest +} + + +//case class ExpressionSchemaTest( +//) extends SchemaTest + + +class DefaultSchemaTestExecutor extends SchemaTestExecutor { + override def execute(execution: Execution, df: DataFrame, test: SchemaTest): Option[TestResult] = ??? +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala new file mode 100644 index 000000000..70a526fb0 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Target + + +class TargetCollector extends Collector { + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { + val parent = documentation.reference + val docs = graph.targets.map(t => t.target.identifier -> document(execution, parent, t.target)).toMap + documentation.copy(targets = docs) + } + + /** + * Creates a documentation of this target + * @param execution + * @param parent + * @return + */ + private def document(execution: Execution, parent:Reference, target:Target) : TargetDoc = { + val doc = TargetDoc( + Some(parent), + target.identifier, + None, + Seq(), + Seq(), + Seq() + ) + val ref = doc.reference + + val phaseDocs = target.phases.toSeq.map { p => + TargetPhaseDoc( + Some(ref), + p, + None, + target.provides(p).toSeq, + target.requires(p).toSeq + ) + } + + doc.copy(phases=phaseDocs).merge(target.documentation) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala new file mode 100644 index 000000000..5454e9dcc --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.model.TargetIdentifier + + +final case class TargetPhaseReference( + override val parent:Option[Reference], + phase:Phase +) extends Reference + + +final case class TargetPhaseDoc( + parent:Option[Reference], + phase:Phase, + description:Option[String], + provides:Seq[ResourceIdentifier], + requires:Seq[ResourceIdentifier] +) extends Fragment { + override def reference: Reference = TargetPhaseReference(parent, phase) + override def fragments: Seq[Fragment] = Seq() + override def reparent(parent: Reference): TargetPhaseDoc = { + copy(parent = Some(parent)) + } +} + + +final case class TargetReference( + override val parent:Option[Reference], + name:String +) extends Reference + + +final case class TargetDoc( + parent:Option[Reference], + identifier:TargetIdentifier, + description:Option[String], + phases:Seq[TargetPhaseDoc], + inputs:Seq[Reference], + outputs:Seq[Reference] +) extends EntityDoc { + override def reference: TargetReference = TargetReference(parent, identifier.name) + override def fragments: Seq[Fragment] = phases + override def reparent(parent: Reference): TargetDoc = { + val ref = TargetReference(Some(parent), identifier.name) + copy( + parent = Some(parent), + phases = phases.map(_.reparent(ref)), + ) + } + + def merge(other:Option[TargetDoc]) : TargetDoc = other.map(merge).getOrElse(this) + def merge(other:TargetDoc) : TargetDoc = { + val id = if (identifier.isEmpty) other.identifier else identifier + val desc = other.description.orElse(this.description) + val in = inputs.toSet ++ other.inputs.toSet + val out = outputs.toSet ++ other.outputs.toSet + val result = copy(identifier=id, description=desc, inputs=in.toSeq, outputs=out.toSeq) + parent.orElse(other.parent) + .map(result.reparent) + .getOrElse(result) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala new file mode 100644 index 000000000..b859d6565 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +sealed abstract class TestStatus extends Product with Serializable + +object TestStatus { + final case object FAILED extends TestStatus + final case object SUCCESS extends TestStatus + final case object ERROR extends TestStatus +} + + +final case class TestResultReference( + parent:Option[Reference] +) extends Reference + + +final case class TestResult( + parent:Some[Reference], + status:TestStatus, + description:Option[String], + details:Option[Fragment] +) extends Fragment { + override def reference: TestResultReference = TestResultReference(parent) + override def fragments: Seq[Fragment] = details.toSeq + + override def reparent(parent:Reference) : TestResult = { + val ref = TestResultReference(Some(parent)) + copy( + parent = Some(parent), + details = details.map(_.reparent(ref)) + ) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala new file mode 100644 index 000000000..464c78252 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import scala.collection.JavaConverters._ + + +final case class ColumnDocWrapper(column:ColumnDoc) { + override def toString: String = column.name + + def getName() : String = column.name + def getNullable() : Boolean = column.nullable + def getType() : String = column.typeName + def getSqlType() : String = column.sqlType + def getSparkType() : String = column.sparkType + def getCatalogType() : String = column.catalogType + def getDescription() : String = column.description.getOrElse("") + def getColumns() : java.util.List[ColumnDocWrapper] = column.children.map(ColumnDocWrapper).asJava +} + + +final case class SchemaDocWrapper(schema:SchemaDoc) { + def getDescription() : String = schema.description.getOrElse("") + def getColumns() : java.util.List[ColumnDocWrapper] = schema.columns.map(ColumnDocWrapper).asJava +} + + +final case class MappingOutputDocWrapper(output:MappingOutputDoc) { + override def toString: String = output.identifier.toString + + def getMapping() : String = output.identifier.name + def getOutput() : String = output.identifier.output + def getName() : String = output.identifier.output + def getDescription() : String = output.description.getOrElse("") + def getSchema() : SchemaDocWrapper = output.schema.map(SchemaDocWrapper).orNull +} + + +final case class MappingDocWrapper(mapping:MappingDoc) { + override def toString: String = mapping.identifier.toString + + def getName() : String = mapping.identifier.name + def getDescription() : String = mapping.description.getOrElse("") + def getOutputs() : java.util.List[MappingOutputDocWrapper] = mapping.outputs.map(MappingOutputDocWrapper).asJava +} + + +final case class ProjectDocWrapper(project:ProjectDoc) { + override def toString: String = project.name + + def getName() : String = project.name + def getVersion() : String = project.version.getOrElse("") + def getDescription() : String = project.description.getOrElse("") + + def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.values.map(MappingDocWrapper).toSeq.asJava +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala index 549c99b83..bcff74137 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala @@ -97,25 +97,6 @@ private[execution] sealed class RunnerImpl { result } - def withExecution[T](parent:Execution, isolated:Boolean=false)(fn:Execution => T) : T = { - val execution : Execution = new ScopedExecution(parent, isolated) - val result = fn(execution) - - // Wait for any running background operation, and do not perform a cleanup - val ops = execution.operations - val activeOps = ops.listActive() - if (activeOps.nonEmpty) { - logger.info("Some background operations are still active:") - activeOps.foreach(o => logger.info(s" - s${o.name}")) - logger.info("Waiting for termination...") - ops.awaitTermination() - } - - // Finally release any resources - execution.cleanup() - result - } - protected val lineSize = 109 protected val separator = boldWhite(StringUtils.repeat('-', lineSize)) protected val doubleSeparator = boldWhite(StringUtils.repeat('=', lineSize)) @@ -237,7 +218,6 @@ private[execution] sealed class RunnerImpl { private[execution] final class JobRunnerImpl(runner:Runner) extends RunnerImpl { private val stateStore = runner.stateStore private val stateStoreListener = new StateStoreAdaptorListener(stateStore) - private val parentExecution = runner.parentExecution /** * Executes a single job using the given execution and a map of parameters. The Runner may decide not to @@ -257,7 +237,7 @@ private[execution] final class JobRunnerImpl(runner:Runner) extends RunnerImpl { val startTime = Instant.now() val isolated2 = isolated || job.parameters.nonEmpty || job.environment.nonEmpty - withExecution(parentExecution, isolated2) { execution => + runner.withExecution(isolated2) { execution => runner.withJobContext(job, args, Some(execution), force, dryRun, isolated2) { (context, arguments) => val title = s"lifecycle for job '${job.identifier}' ${arguments.map(kv => kv._1 + "=" + kv._2).mkString(", ")}" val listeners = if (!dryRun) stateStoreListener +: (runner.hooks ++ job.hooks).map(_.instantiate(context)) else Seq() @@ -452,10 +432,8 @@ private[execution] final class JobRunnerImpl(runner:Runner) extends RunnerImpl { * @param runner */ private[execution] final class TestRunnerImpl(runner:Runner) extends RunnerImpl { - private val parentExecution = runner.parentExecution - def executeTest(test:Test, keepGoing:Boolean=false, dryRun:Boolean=false) : Status = { - withExecution(parentExecution, true) { execution => + runner.withExecution(true) { execution => runner.withTestContext(test, Some(execution), dryRun) { context => val title = s"Running test '${test.identifier}'" logTitle(title) @@ -656,6 +634,25 @@ final class Runner( } } + def withExecution[T](isolated:Boolean=false)(fn:Execution => T) : T = { + val execution : Execution = new ScopedExecution(parentExecution, isolated) + val result = fn(execution) + + // Wait for any running background operation, and do not perform a cleanup + val ops = execution.operations + val activeOps = ops.listActive() + if (activeOps.nonEmpty) { + logger.info("Some background operations are still active:") + activeOps.foreach(o => logger.info(s" - s${o.name}")) + logger.info("Waiting for termination...") + ops.awaitTermination() + } + + // Finally release any resources + execution.cleanup() + result + } + /** * Provides a context for the given job. This will apply all environment variables of the job and add * additional variables like a `force` flag. diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala index 66066baca..8ae2eafe5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Session.scala @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory import com.dimajix.flowman.catalog.HiveCatalog import com.dimajix.flowman.config.Configuration import com.dimajix.flowman.config.FlowmanConf +import com.dimajix.flowman.documentation.Documenter import com.dimajix.flowman.execution.Session.builder import com.dimajix.flowman.hadoop.FileSystem import com.dimajix.flowman.history.NullStateStore diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala index 64c967600..59c5d94e0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala @@ -75,6 +75,8 @@ final case class Graph( relations:Seq[RelationRef], targets:Seq[TargetRef] ) { + def project : Option[Project] = context.project + def nodes : Seq[Node] = mappings ++ relations ++ targets def edges : Seq[Edge] = nodes.flatMap(_.outgoing) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala index a783e7bf3..3f18cf0b7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala @@ -19,6 +19,7 @@ package com.dimajix.flowman.model import org.apache.spark.sql.DataFrame import org.apache.spark.storage.StorageLevel +import com.dimajix.flowman.documentation.MappingDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.NoSuchMappingOutputException @@ -35,7 +36,8 @@ object Mapping { Metadata(context, name, Category.MAPPING, kind), false, false, - StorageLevel.NONE + StorageLevel.NONE, + None ) } } @@ -44,7 +46,8 @@ object Mapping { metadata:Metadata, broadcast:Boolean, checkpoint:Boolean, - cache:StorageLevel + cache:StorageLevel, + documentation:Option[MappingDoc] ) extends Instance.Properties[Properties] { override val namespace : Option[Namespace] = context.namespace override val project : Option[Project] = context.project @@ -70,6 +73,12 @@ trait Mapping extends Instance { */ def identifier : MappingIdentifier + /** + * Returns a (static) documentation of this mapping + * @return + */ + def documentation : Option[MappingDoc] + /** * This method should return true, if the resulting dataframe should be broadcast for map-side joins * @return @@ -169,6 +178,12 @@ abstract class BaseMapping extends AbstractInstance with Mapping { */ override def identifier : MappingIdentifier = instanceProperties.identifier + /** + * Returns a (static) documentation of this mapping + * @return + */ + override def documentation : Option[MappingDoc] = instanceProperties.documentation + /** * This method should return true, if the resulting dataframe should be broadcast for map-side joins * @return diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala index e4d5324db..226173236 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala @@ -34,6 +34,7 @@ import com.dimajix.common.MapIgnoreCase import com.dimajix.common.SetIgnoreCase import com.dimajix.common.Trilean import com.dimajix.flowman.config.FlowmanConf +import com.dimajix.flowman.documentation.RelationDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MergeClause @@ -56,6 +57,7 @@ object Relation { Properties( context, Metadata(context, name, Category.RELATION, kind), + None, None ) } @@ -63,7 +65,8 @@ object Relation { final case class Properties( context:Context, metadata:Metadata, - description:Option[String] + description:Option[String], + documentation:Option[RelationDoc] ) extends Instance.Properties[Properties] { override val namespace : Option[Namespace] = context.namespace @@ -99,6 +102,12 @@ trait Relation extends Instance { */ def description : Option[String] + /** + * Returns a (static) documentation of this relation + * @return + */ + def documentation : Option[RelationDoc] + /** * Returns the list of all resources which will be created by this relation. This method mainly refers to the * CREATE and DESTROY execution phase. @@ -276,6 +285,12 @@ abstract class BaseRelation extends AbstractInstance with Relation { */ override def description : Option[String] = instanceProperties.description + /** + * Returns a (static) documentation of this relation + * @return + */ + override def documentation : Option[RelationDoc] = instanceProperties.documentation + /** * Returns the schema of the relation, excluding partition columns * @return @@ -322,7 +337,7 @@ abstract class BaseRelation extends AbstractInstance with Relation { * Creates all known links for building a descriptive graph of the whole data flow * Params: linker - The linker object to use for creating new edges */ - def link(linker:Linker) : Unit = {} + override def link(linker:Linker) : Unit = {} /** * Creates a DataFrameReader which is already configured with the schema diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Schema.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Schema.scala index 36582bfa8..5818d798d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Schema.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Schema.scala @@ -86,9 +86,13 @@ trait Schema extends Instance { */ def sparkSchema : org.apache.spark.sql.types.StructType + /** + * Returns a Spark schema useable for Catalog entries. This Schema may include VARCHAR(n) and CHAR(n) entries + * @return + */ def catalogSchema : org.apache.spark.sql.types.StructType - /** + /** * Provides a human readable string representation of the schema */ def printTree() : Unit = { @@ -118,6 +122,10 @@ abstract class BaseSchema extends AbstractInstance with Schema { org.apache.spark.sql.types.StructType(fields.map(_.sparkField)) } + /** + * Returns a Spark schema useable for Catalog entries. This Schema may include VARCHAR(n) and CHAR(n) entries + * @return + */ override def catalogSchema : org.apache.spark.sql.types.StructType = { org.apache.spark.sql.types.StructType(fields.map(_.catalogField)) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala index 8a8238c9b..86f451ce5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala @@ -20,6 +20,7 @@ import org.apache.spark.sql.DataFrame import com.dimajix.common.Trilean import com.dimajix.common.Unknown +import com.dimajix.flowman.documentation.TargetDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.Phase @@ -60,7 +61,8 @@ object Target { context, Metadata(context, name, Category.TARGET, kind), Seq(), - Seq() + Seq(), + None ) } } @@ -68,7 +70,8 @@ object Target { context:Context, metadata:Metadata, before: Seq[TargetIdentifier], - after: Seq[TargetIdentifier] + after: Seq[TargetIdentifier], + documentation: Option[TargetDoc] ) extends Instance.Properties[Properties] { override val namespace : Option[Namespace] = context.namespace override val project : Option[Project] = context.project @@ -94,6 +97,12 @@ trait Target extends Instance { */ def identifier : TargetIdentifier + /** + * Returns a (static) documentation of this target + * @return + */ + def documentation : Option[TargetDoc] + /** * Returns an instance representing this target with the context * @return @@ -169,6 +178,12 @@ abstract class BaseTarget extends AbstractInstance with Target { */ override def identifier : TargetIdentifier = instanceProperties.identifier + /** + * Returns a (static) documentation of this target + * @return + */ + override def documentation : Option[TargetDoc] = instanceProperties.documentation + /** * Returns an instance representing this target with the context * @return diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/templating.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala similarity index 100% rename from flowman-core/src/main/scala/com/dimajix/flowman/model/templating.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala new file mode 100644 index 000000000..68ed556bb --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spi + +import java.util.ServiceLoader + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Column +import org.apache.spark.sql.DataFrame + +import com.dimajix.flowman.documentation.ColumnTest +import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.execution.Execution + + +object ColumnTestExecutor { + def executors : Seq[ColumnTestExecutor] = { + val loader = ServiceLoader.load(classOf[ColumnTestExecutor]) + loader.iterator().asScala.toSeq + } +} + +trait ColumnTestExecutor { + def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala new file mode 100644 index 000000000..00837f734 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spi + +import java.util.ServiceLoader + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.DataFrame + +import com.dimajix.flowman.documentation.SchemaTest +import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.execution.Execution + + +object SchemaTestExecutor { + def executors : Seq[SchemaTestExecutor] = { + val loader = ServiceLoader.load(classOf[SchemaTestExecutor]) + loader.iterator().asScala.toSeq + } +} + +trait SchemaTestExecutor { + def execute(execution: Execution, df:DataFrame, test:SchemaTest) : Option[TestResult] +} diff --git a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java new file mode 100644 index 000000000..e16e73e03 --- /dev/null +++ b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * This annotation marks a specific class as a [[ColumnTest]] to be used in a data flow spec. The specific ColumnTest itself has + * to derive from the ColumnTest class + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ColumnTestType { + /** + * Specifies the kind of the column test which is used in data flow specifications. + * @return + */ + String kind(); +} diff --git a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/GeneratorType.java b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/GeneratorType.java new file mode 100644 index 000000000..ad6dbe071 --- /dev/null +++ b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/GeneratorType.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * This annotation marks a specific class as a [[Generator]] to be used in a data flow spec. The specific Generator itself has + * to derive from the Generator class + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface GeneratorType { + /** + * Specifies the kind of the relation which is used in data flow specifications. + * @return + */ + String kind(); +} diff --git a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java new file mode 100644 index 000000000..85b6a5847 --- /dev/null +++ b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * This annotation marks a specific class as a [[SchemaTest]] to be used in a data flow spec. The specific SchemaTest itself has + * to derive from the SchemaTest class + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface SchemaTestType { + /** + * Specifies the kind of the schema test which is used in data flow specifications. + * @return + */ + String kind(); +} diff --git a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler index 32c433818..1c947a9f1 100644 --- a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler +++ b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler @@ -11,3 +11,6 @@ com.dimajix.flowman.spec.storage.StoreSpecAnnotationHandler com.dimajix.flowman.spec.target.TargetSpecAnnotationHandler com.dimajix.flowman.spec.assertion.AssertionSpecAnnotationHandler com.dimajix.flowman.spec.storage.ParcelSpecAnnotationHandler +com.dimajix.flowman.spec.documentation.GeneratorSpecAnnotationHandler +com.dimajix.flowman.spec.documentation.ColumnTestSpecAnnotationHandler +com.dimajix.flowman.spec.documentation.SchemaTestSpecAnnotationHandler diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl new file mode 100644 index 000000000..82a4a1850 --- /dev/null +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -0,0 +1,15 @@ +Project: ${project.name} version ${project.version} + +Mappings: +#foreach($mapping in ${project.mappings}) +Mapping '${mapping}' +${mapping.description} +#foreach($output in ${mapping.outputs}) + - Output '${output.name}' +#foreach($column in ${output.schema.columns}) + ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end +#end +#end + + +#end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala index 27969c3d0..b907709fb 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala @@ -23,6 +23,9 @@ import com.dimajix.flowman.spec.assertion.AssertionSpec import com.dimajix.flowman.spec.catalog.CatalogSpec import com.dimajix.flowman.spec.connection.ConnectionSpec import com.dimajix.flowman.spec.dataset.DatasetSpec +import com.dimajix.flowman.spec.documentation.ColumnTestSpec +import com.dimajix.flowman.spec.documentation.GeneratorSpec +import com.dimajix.flowman.spec.documentation.SchemaTestSpec import com.dimajix.flowman.spec.history.HistorySpec import com.dimajix.flowman.spec.mapping.MappingSpec import com.dimajix.flowman.spec.measure.MeasureSpec @@ -61,6 +64,9 @@ object ObjectMapper extends CoreObjectMapper { val datasetTypes = DatasetSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val metricSinkTypes = MetricSinkSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val parcelTypes = ParcelSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val generatorTypes = GeneratorSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val columnTestTypes = ColumnTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val schemaTestTypes = SchemaTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val mapper = super.mapper mapper.registerSubtypes(stateStoreTypes:_*) mapper.registerSubtypes(catalogTypes:_*) @@ -75,6 +81,9 @@ object ObjectMapper extends CoreObjectMapper { mapper.registerSubtypes(datasetTypes:_*) mapper.registerSubtypes(metricSinkTypes:_*) mapper.registerSubtypes(parcelTypes:_*) + mapper.registerSubtypes(generatorTypes:_*) + mapper.registerSubtypes(columnTestTypes:_*) + mapper.registerSubtypes(schemaTestTypes:_*) mapper } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala new file mode 100644 index 000000000..c13d0ebba --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.ColumnDoc +import com.dimajix.flowman.documentation.Reference +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.NullType + + +class ColumnDocSpec { + @JsonProperty(value="name", required=true) private var name:String = _ + @JsonProperty(value="description", required=false) private var description:Option[String] = None + @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[ColumnTestSpec] = Seq() + + def instantiate(context: Context, parent:Reference): ColumnDoc = { + val doc = ColumnDoc( + Some(parent), + context.evaluate(name), + Field(name, NullType), + context.evaluate(description), + Seq(), + Seq() + ) + def ref = doc.reference + + val cols = columns.map(_.instantiate(context, ref)) + val tests = this.tests.map(_.instantiate(context, ref)) + + doc.copy( + children = cols, + tests = tests + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala new file mode 100644 index 000000000..1972456ac --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala @@ -0,0 +1,69 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +import com.dimajix.common.TypeRegistry +import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.documentation.ColumnTest +import com.dimajix.flowman.documentation.NotNullColumnTest +import com.dimajix.flowman.documentation.RangeColumnTest +import com.dimajix.flowman.documentation.UniqueColumnTest +import com.dimajix.flowman.documentation.ValuesColumnTest +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.spec.annotation.ColumnTestType +import com.dimajix.flowman.spi.ClassAnnotationHandler + + +object ColumnTestSpec extends TypeRegistry[ColumnTestSpec] { +} + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(name = "notNull", value = classOf[NotNullColumnTestSpec]), + new JsonSubTypes.Type(name = "unique", value = classOf[UniqueColumnTestSpec]), + new JsonSubTypes.Type(name = "range", value = classOf[RangeColumnTestSpec]), + new JsonSubTypes.Type(name = "values", value = classOf[ValuesColumnTestSpec]) +)) +abstract class ColumnTestSpec { + def instantiate(context: Context, parent:ColumnReference): ColumnTest +} + + +class ColumnTestSpecAnnotationHandler extends ClassAnnotationHandler { + override def annotation: Class[_] = classOf[ColumnTestType] + + override def register(clazz: Class[_]): Unit = + ColumnTestSpec.register(clazz.getAnnotation(classOf[ColumnTestType]).kind(), clazz.asInstanceOf[Class[_ <: ColumnTestSpec]]) +} + + +class NotNullColumnTestSpec extends ColumnTestSpec { + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = NotNullColumnTest(Some(parent)) +} +class UniqueColumnTestSpec extends ColumnTestSpec { + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = UniqueColumnTest(Some(parent)) +} +class RangeColumnTestSpec extends ColumnTestSpec { + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = RangeColumnTest(Some(parent)) +} +class ValuesColumnTestSpec extends ColumnTestSpec { + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = ValuesColumnTest(Some(parent)) +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala new file mode 100644 index 000000000..76e5aeb92 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation +import java.net.URL +import java.nio.charset.Charset + +import com.fasterxml.jackson.annotation.JsonProperty +import com.google.common.io.Resources +import org.apache.hadoop.fs.Path +import org.slf4j.LoggerFactory + +import com.dimajix.flowman.documentation.Generator +import com.dimajix.flowman.execution.Context + + +object FileGenerator { + val textTemplate : URL = Resources.getResource(classOf[FileGenerator], "/com/dimajix/flowman/documentation/text") + val defaultTemplate : URL = textTemplate +} + + +case class FileGenerator( + location:Path, + template:URL = FileGenerator.defaultTemplate +) extends TemplateGenerator(template) { + private val logger = LoggerFactory.getLogger(classOf[FileGenerator]) +} + + +class FileGeneratorSpec extends GeneratorSpec { + @JsonProperty(value="location", required=true) private var location:String = _ + @JsonProperty(value="template", required=false) private var template:String = FileGenerator.defaultTemplate.toString + + override def instantiate(context: Context): Generator = { + FileGenerator( + new Path(context.evaluate(location)), + new URL(context.evaluate(template)) + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/GeneratorSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/GeneratorSpec.scala new file mode 100644 index 000000000..cbb6395d3 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/GeneratorSpec.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +import com.dimajix.common.TypeRegistry +import com.dimajix.flowman.documentation.Generator +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.spec.Spec +import com.dimajix.flowman.spec.annotation.GeneratorType +import com.dimajix.flowman.spi.ClassAnnotationHandler + + +object GeneratorSpec extends TypeRegistry[GeneratorSpec] { +} + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(name = "file", value = classOf[FileGeneratorSpec]) +)) +abstract class GeneratorSpec extends Spec[Generator] { + def instantiate(context:Context): Generator +} + + +class GeneratorSpecAnnotationHandler extends ClassAnnotationHandler { + override def annotation: Class[_] = classOf[GeneratorType] + + override def register(clazz: Class[_]): Unit = + GeneratorSpec.register(clazz.getAnnotation(classOf[GeneratorType]).kind(), clazz.asInstanceOf[Class[_ <: GeneratorSpec]]) +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala new file mode 100644 index 000000000..bae0f0df2 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala @@ -0,0 +1,126 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.MappingDoc +import com.dimajix.flowman.documentation.MappingOutputDoc +import com.dimajix.flowman.documentation.MappingReference +import com.dimajix.flowman.documentation.SchemaDoc +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.spec.Spec + + +class MappingOutputDocSpec { + @JsonProperty(value="description", required=false) private var description:Option[String] = None + @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + + def instantiate(context: Context, parent:MappingReference, name:String): MappingOutputDoc = { + val doc = MappingOutputDoc( + Some(parent), + MappingOutputIdentifier.empty.copy(output=name), + context.evaluate(description), + None + ) + val ref = doc.reference + + val schema = + if (columns.nonEmpty || tests.nonEmpty) { + val schema = SchemaDoc( + Some(ref), + None, + Seq(), + Seq() + ) + val ref2 = schema.reference + val cols = columns.map(_.instantiate(context, ref2)) + val tests = this.tests.map(_.instantiate(context, ref2)) + Some(schema.copy( + columns=cols, + tests=tests + )) + } + else { + None + } + + doc.copy( + schema = schema + ) + } +} + + +class MappingDocSpec extends Spec[MappingDoc] { + @JsonProperty(value="description", required=false) private var description:Option[String] = None + @JsonProperty(value="outputs", required=false) private var outputs:Map[String,MappingOutputDocSpec] = _ + @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + + def instantiate(context: Context): MappingDoc = { + val doc = MappingDoc( + None, + MappingIdentifier.empty, + context.evaluate(description), + Seq(), + Seq() + ) + val ref = doc.reference + + val output = + if (columns.nonEmpty || tests.nonEmpty) { + val output = MappingOutputDoc( + Some(ref), + MappingOutputIdentifier.empty.copy(output="main"), + None, + None + ) + val ref2 = output.reference + + val schema = SchemaDoc( + Some(ref2), + None, + Seq(), + Seq() + ) + val ref3 = schema.reference + val cols = columns.map(_.instantiate(context, ref3)) + val tests = this.tests.map(_.instantiate(context, ref3)) + Some( + output.copy( + schema = Some(schema.copy( + columns=cols, + tests=tests + )) + ) + ) + } + else { + None + } + + val outputs = this.outputs.map { case(name,output) => + output.instantiate(context, ref, name) + } ++ output.toSeq + + doc.copy(outputs=outputs.toSeq) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala new file mode 100644 index 000000000..b388f6c98 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.RelationDoc +import com.dimajix.flowman.documentation.SchemaDoc +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.spec.Spec + + +class RelationDocSpec extends Spec[RelationDoc] { + @JsonProperty(value="description", required=false) private var description:Option[String] = None + @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + + override def instantiate(context: Context): RelationDoc = { + val doc = RelationDoc( + None, + RelationIdentifier.empty, + context.evaluate(description), + None, + Seq(), + Map() + ) + val ref = doc.reference + + val schema = + if (columns.nonEmpty || tests.nonEmpty) { + val schema = SchemaDoc( + Some(ref), + None, + Seq(), + Seq() + ) + val ref2 = schema.reference + val cols = columns.map(_.instantiate(context, ref2)) + val tests = this.tests.map(_.instantiate(context, ref2)) + Some(schema.copy( + columns=cols, + tests=tests + )) + } + else { + None + } + + doc.copy( + schema = schema + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala new file mode 100644 index 000000000..b45eeb24f --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.Reference +import com.dimajix.flowman.documentation.SchemaDoc +import com.dimajix.flowman.execution.Context + + +class SchemaDocSpec { + @JsonProperty(value="description", required=false) private var description:Option[String] = None + @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + + def instantiate(context: Context, parent:Reference): SchemaDoc = { + val doc = SchemaDoc( + Some(parent), + context.evaluate(description), + Seq(), + Seq() + ) + val ref = doc.reference + + val cols = columns.map(_.instantiate(context, ref)) + val tests = this.tests.map(_.instantiate(context, ref)) + doc.copy( + columns = cols, + tests = tests + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala new file mode 100644 index 000000000..f36bba463 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +import com.dimajix.common.TypeRegistry +import com.dimajix.flowman.documentation.SchemaReference +import com.dimajix.flowman.documentation.SchemaTest +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.spec.annotation.SchemaTestType +import com.dimajix.flowman.spi.ClassAnnotationHandler + + +object SchemaTestSpec extends TypeRegistry[SchemaTestSpec] { +} + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(name = "file", value = classOf[FileGeneratorSpec]) +)) +abstract class SchemaTestSpec { + def instantiate(context: Context, parent:SchemaReference): SchemaTest +} + + + +class SchemaTestSpecAnnotationHandler extends ClassAnnotationHandler { + override def annotation: Class[_] = classOf[SchemaTestType] + + override def register(clazz: Class[_]): Unit = + SchemaTestSpec.register(clazz.getAnnotation(classOf[SchemaTestType]).kind(), clazz.asInstanceOf[Class[_ <: SchemaTestSpec]]) +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala new file mode 100644 index 000000000..cd83cd8e6 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.TargetDoc +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.TargetIdentifier +import com.dimajix.flowman.spec.Spec + + +class TargetDocSpec extends Spec[TargetDoc] { + @JsonProperty(value="description", required=false) private var description:Option[String] = None + + override def instantiate(context: Context): TargetDoc = { + val doc = TargetDoc( + None, + TargetIdentifier.empty, + context.evaluate(description), + Seq(), + Seq(), + Seq() + ) + doc + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala new file mode 100644 index 000000000..679157a9b --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import java.net.URL +import java.nio.charset.Charset + +import com.google.common.io.Resources + +import com.dimajix.flowman.documentation.BaseGenerator +import com.dimajix.flowman.documentation.ProjectDoc +import com.dimajix.flowman.documentation.ProjectDocWrapper +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution + + +abstract class TemplateGenerator( + template:URL +) extends BaseGenerator { + override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + val temp = loadResource("project.vtl") + val result = context.evaluate(temp, Map("project" -> ProjectDocWrapper(documentation))) + println(result) + } + + private def loadResource(name: String): String = { + val path = template.getPath + val url = + if (path.endsWith("/")) + new URL(template.toString + name) + else + new URL(template.toString + "/" + name) + Resources.toString(url, Charset.forName("UTF-8")) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala index b01f91223..65b889ce4 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MappingSpec.scala @@ -29,6 +29,7 @@ import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Metadata import com.dimajix.flowman.spec.NamedSpec import com.dimajix.flowman.spec.annotation.MappingType +import com.dimajix.flowman.spec.documentation.MappingDocSpec import com.dimajix.flowman.spec.template.CustomTypeResolverBuilder import com.dimajix.flowman.spi.ClassAnnotationHandler @@ -93,6 +94,7 @@ abstract class MappingSpec extends NamedSpec[Mapping] { @JsonProperty(value="broadcast", required = false) protected var broadcast:String = "false" @JsonProperty(value="checkpoint", required = false) protected var checkpoint:String = "false" @JsonProperty(value="cache", required = false) protected var cache:String = "NONE" + @JsonProperty(value="documentation", required = false) private var documentation: Option[MappingDocSpec] = None /** * Creates an instance of this specification and performs the interpolation of all variables @@ -114,7 +116,8 @@ abstract class MappingSpec extends NamedSpec[Mapping] { metadata.map(_.instantiate(context, name, Category.MAPPING, kind)).getOrElse(Metadata(context, name, Category.MAPPING, kind)), context.evaluate(broadcast).toBoolean, context.evaluate(checkpoint).toBoolean, - StorageLevel.fromString(context.evaluate(cache)) + StorageLevel.fromString(context.evaluate(cache)), + documentation.map(_.instantiate(context)) ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala index 88343c99f..4c92de174 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.DataFrame import com.dimajix.flowman.common.ParserUtils.splitSettings +import com.dimajix.flowman.documentation.MappingDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.ScopeContext @@ -55,6 +56,12 @@ case class TemplateMapping( } } + /** + * Returns a (static) documentation of this mapping + * + * @return + */ + override def documentation: Option[MappingDoc] = mappingInstance.documentation.map(_.merge(instanceProperties.documentation)) /** * Returns a list of physical resources required by this mapping. This list will only be non-empty for mappings diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala index 72bb0f402..f3072730d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala @@ -29,6 +29,7 @@ import com.dimajix.flowman.model.Metadata import com.dimajix.flowman.model.Relation import com.dimajix.flowman.spec.NamedSpec import com.dimajix.flowman.spec.annotation.RelationType +import com.dimajix.flowman.spec.documentation.RelationDocSpec import com.dimajix.flowman.spec.template.CustomTypeResolverBuilder import com.dimajix.flowman.spi.ClassAnnotationHandler @@ -63,6 +64,7 @@ object RelationSpec extends TypeRegistry[RelationSpec] { abstract class RelationSpec extends NamedSpec[Relation] { @JsonProperty(value="kind", required = true) protected var kind: String = _ @JsonProperty(value="description", required = false) private var description: Option[String] = None + @JsonProperty(value="documentation", required = false) private var documentation: Option[RelationDocSpec] = None override def instantiate(context:Context) : Relation @@ -77,7 +79,8 @@ abstract class RelationSpec extends NamedSpec[Relation] { Relation.Properties( context, metadata.map(_.instantiate(context, name, Category.RELATION, kind)).getOrElse(Metadata(context, name, Category.RELATION, kind)), - description.map(context.evaluate) + description.map(context.evaluate), + documentation.map(_.instantiate(context)) ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/TemplateRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/TemplateRelation.scala index 3a81b74c8..79929d37a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/TemplateRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/TemplateRelation.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.DataFrame import com.dimajix.common.Trilean import com.dimajix.flowman.common.ParserUtils.splitSettings +import com.dimajix.flowman.documentation.RelationDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MergeClause @@ -93,6 +94,12 @@ case class TemplateRelation( */ override def description : Option[String] = relationInstance.description + /** + * Returns a (static) documentation of this relation + * @return + */ + override def documentation : Option[RelationDoc] = relationInstance.documentation.map(_.merge(instanceProperties.documentation)) + /** * Returns the schema of the relation * @return diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala index e10888626..5e1030b87 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala @@ -29,6 +29,7 @@ import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetIdentifier import com.dimajix.flowman.spec.NamedSpec import com.dimajix.flowman.spec.annotation.TargetType +import com.dimajix.flowman.spec.documentation.TargetDocSpec import com.dimajix.flowman.spec.template.CustomTypeResolverBuilder import com.dimajix.flowman.spi.ClassAnnotationHandler @@ -70,6 +71,7 @@ abstract class TargetSpec extends NamedSpec[Target] { @JsonProperty(value = "kind", required = true) protected var kind: String = _ @JsonProperty(value = "before", required=false) protected[spec] var before:Seq[String] = Seq() @JsonProperty(value = "after", required=false) protected[spec] var after:Seq[String] = Seq() + @JsonProperty(value="documentation", required = false) private var documentation: Option[TargetDocSpec] = None override def instantiate(context: Context): Target @@ -85,7 +87,8 @@ abstract class TargetSpec extends NamedSpec[Target] { context, metadata.map(_.instantiate(context, name, Category.TARGET, kind)).getOrElse(Metadata(context, name, Category.TARGET, kind)), before.map(context.evaluate).map(TargetIdentifier.parse), - after.map(context.evaluate).map(TargetIdentifier.parse) + after.map(context.evaluate).map(TargetIdentifier.parse), + documentation.map(_.instantiate(context)) ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala index 497086c6f..bd3f6ed14 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.dimajix.common.Trilean import com.dimajix.flowman.common.ParserUtils.splitSettings +import com.dimajix.flowman.documentation.TargetDoc import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.Phase @@ -55,6 +56,12 @@ case class TemplateTarget( } } + /** + * Returns a (static) documentation of this target + * @return + */ + override def documentation : Option[TargetDoc] = targetInstance.documentation.map(_.merge(instanceProperties.documentation)) + /** * Returns an instance representing this target with the context * @return diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala new file mode 100644 index 000000000..fdb5e815c --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.documentation.UniqueColumnTest +import com.dimajix.flowman.execution.RootContext +import com.dimajix.flowman.spec.ObjectMapper + + +class ColumnTestTest extends AnyFlatSpec with Matchers { + "A ColumnTest" should "be deserializable" in { + val yaml = + """ + |kind: unique + """.stripMargin + + val spec = ObjectMapper.parse[ColumnTestSpec](yaml) + spec shouldBe a[UniqueColumnTestSpec] + + val context = RootContext.builder().build() + val test = spec.instantiate(context, ColumnReference(None, "col0")) + test should be (UniqueColumnTest( + Some(ColumnReference(None, "col0")) + )) + } +} diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala new file mode 100644 index 000000000..10d134fc9 --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.execution.RootContext +import com.dimajix.flowman.spec.ObjectMapper + + +class MappingDocTest extends AnyFlatSpec with Matchers { + "A MappingDocSpec" should "be deserializable" in { + val yaml = + """ + |description: "This is a mapping" + |columns: + | - name: col_a + | description: "This is column a" + | tests: + | - kind: notNull + |outputs: + | other: + | columns: + | - name: col_x + | description: "Column of other output" + |""".stripMargin + + val spec = ObjectMapper.parse[MappingDocSpec](yaml) + + val context = RootContext.builder().build() + val mapping = spec.instantiate(context) + + println(mapping.toString) + } +} diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala new file mode 100644 index 000000000..7e7c2382d --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.RootContext +import com.dimajix.flowman.spec.ObjectMapper + + +class RelationDocTest extends AnyFlatSpec with Matchers { + "A RelationDocSpec" should "be deserializable" in { + val yaml = + """ + |description: "This is a mapping" + |columns: + | - name: col_a + | description: "This is column a" + | tests: + | - kind: notNull + | - name: col_x + | description: "Column of other output" + | columns: + | - name: sub_col + |""".stripMargin + + val spec = ObjectMapper.parse[RelationDocSpec](yaml) + + val context = RootContext.builder().build() + val relation = spec.instantiate(context) + + println(relation.toString) + } +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Arguments.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Arguments.scala index c74558ae9..afee7b4f9 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Arguments.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Arguments.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import org.kohsuke.args4j.spi.SubCommand import org.kohsuke.args4j.spi.SubCommandHandler import org.kohsuke.args4j.spi.SubCommands +import com.dimajix.flowman.tools.exec.documentation.DocumentationCommand import com.dimajix.flowman.tools.exec.history.HistoryCommand import com.dimajix.flowman.tools.exec.info.InfoCommand import com.dimajix.flowman.tools.exec.job.JobCommand @@ -64,6 +65,7 @@ class Arguments(args:Array[String]) { @Argument(required=false,index=0,metaVar="",usage="the object to work with",handler=classOf[SubCommandHandler]) @SubCommands(Array( + new SubCommand(name="documentation",impl=classOf[DocumentationCommand]), new SubCommand(name="history",impl=classOf[HistoryCommand]), new SubCommand(name="info",impl=classOf[InfoCommand]), new SubCommand(name="job",impl=classOf[JobCommand]), diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumentationCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumentationCommand.scala new file mode 100644 index 000000000..a95ab3546 --- /dev/null +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumentationCommand.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.tools.exec.documentation + +import org.kohsuke.args4j.Argument +import org.kohsuke.args4j.spi.SubCommand +import org.kohsuke.args4j.spi.SubCommandHandler +import org.kohsuke.args4j.spi.SubCommands + +import com.dimajix.flowman.tools.exec.Command +import com.dimajix.flowman.tools.exec.NestedCommand + + +class DocumentationCommand extends NestedCommand { + @Argument(required=true,index=0,metaVar="",usage="the subcommand to run",handler=classOf[SubCommandHandler]) + @SubCommands(Array( + new SubCommand(name="generate",impl=classOf[GenerateCommand]) + )) + override var command:Command = _ +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala new file mode 100644 index 000000000..f50618168 --- /dev/null +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.tools.exec.documentation + +import scala.util.Failure +import scala.util.Success +import scala.util.Try +import scala.util.control.NonFatal + +import org.apache.hadoop.fs.Path +import org.kohsuke.args4j.Argument +import org.slf4j.LoggerFactory + +import com.dimajix.common.ExceptionUtils.reasons +import com.dimajix.flowman.common.ParserUtils.splitSettings +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.documentation.MappingCollector +import com.dimajix.flowman.documentation.RelationCollector +import com.dimajix.flowman.documentation.TargetCollector +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.model.Job +import com.dimajix.flowman.model.JobIdentifier +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.spec.documentation.FileGenerator +import com.dimajix.flowman.tools.exec.Command +import com.dimajix.flowman.types.FieldValue + + +class GenerateCommand extends Command { + private val logger = LoggerFactory.getLogger(getClass) + + @Argument(index=0, required=false, usage = "specifies job to document", metaVar = "") + var job: String = "main" + @Argument(index=1, required=false, usage = "specifies job parameters", metaVar = "=") + var args: Array[String] = Array() + + override def execute(session: Session, project: Project, context:Context) : Status = { + val args = splitSettings(this.args).toMap + Try { + context.getJob(JobIdentifier(job)) + } + match { + case Failure(e) => + logger.error(s"Error instantiating job '$job': ${reasons(e)}") + Status.FAILED + case Success(job) => + generateDoc(session, job, job.arguments(args)) + } + } + + private def generateDoc(session: Session, job:Job, args:Map[String,Any]) : Status = { + val collectors = Seq( + new RelationCollector(), + new MappingCollector(), + new TargetCollector() + ) + val generators = Seq( + new FileGenerator(new Path("/tmp/flowman/doc")) + ) + val documenter = Documenter( + collectors, + generators + ) + + try { + documenter.execute(session, job, args) + Status.SUCCESS + } catch { + case NonFatal(ex) => + logger.error("Cannot generate documentation: " + reasons(ex)) + Status.FAILED + } + } +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/project/ProjectCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/project/ProjectCommand.scala index fbf31b1d8..8bb84d816 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/project/ProjectCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/project/ProjectCommand.scala @@ -21,8 +21,6 @@ import org.kohsuke.args4j.spi.SubCommand import org.kohsuke.args4j.spi.SubCommandHandler import org.kohsuke.args4j.spi.SubCommands -import com.dimajix.flowman.execution.Session -import com.dimajix.flowman.model.Project import com.dimajix.flowman.tools.exec.Command import com.dimajix.flowman.tools.exec.NestedCommand diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/shell/ParsedCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/shell/ParsedCommand.scala index 06cc46e44..023de2735 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/shell/ParsedCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/shell/ParsedCommand.scala @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Kaya Kupferschmidt + * Copyright 2020-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import org.kohsuke.args4j.spi.SubCommands import com.dimajix.flowman.tools.exec.Command import com.dimajix.flowman.tools.exec.VersionCommand +import com.dimajix.flowman.tools.exec.documentation.DocumentationCommand import com.dimajix.flowman.tools.exec.info.InfoCommand import com.dimajix.flowman.tools.exec.mapping.MappingCommand import com.dimajix.flowman.tools.exec.model.ModelCommand @@ -38,6 +39,7 @@ import com.dimajix.flowman.tools.exec.history.HistoryCommand class ParsedCommand { @Argument(required=false,index=0,metaVar="",usage="the object to work with",handler=classOf[SubCommandHandler]) @SubCommands(Array( + new SubCommand(name="documentation",impl=classOf[DocumentationCommand]), new SubCommand(name="eval",impl=classOf[EvaluateCommand]), new SubCommand(name="exit",impl=classOf[ExitCommand]), new SubCommand(name="history",impl=classOf[HistoryCommand]), From 8b85d76152f51be60fff3b3a6b5513738e89b426 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 9 Feb 2022 07:47:49 +0100 Subject: [PATCH 15/95] Fix build for Scala 2.11 --- .../scala/com/dimajix/flowman/documentation/TargetDoc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala index 5454e9dcc..8a7e794a0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -62,7 +62,7 @@ final case class TargetDoc( val ref = TargetReference(Some(parent), identifier.name) copy( parent = Some(parent), - phases = phases.map(_.reparent(ref)), + phases = phases.map(_.reparent(ref)) ) } From da0e7cc0d4f294381dbf9baece346503eca402cc Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 9 Feb 2022 09:04:44 +0100 Subject: [PATCH 16/95] Improve documentation subsystem --- examples/weather/mapping/measurements.yml | 29 ++++++++++++++-- examples/weather/model/measurements-raw.yml | 1 + examples/weather/model/measurements.yml | 10 ++++-- examples/weather/schema/measurements.avsc | 12 ++++--- examples/weather/schema/stations.avsc | 33 ++++++++++++------- .../flowman/documentation/ColumnDoc.scala | 12 +++++-- .../flowman/documentation/SchemaDoc.scala | 15 +++++++++ .../flowman/documentation/velocity.scala | 29 +++++++++++++++- .../com/dimajix/flowman/model/Mapping.scala | 25 +++++++++++++- .../flowman/documentation/text/project.vtl | 13 +++++++- .../spec/documentation/MappingDocSpec.scala | 2 +- .../flowman/spec/mapping/AliasMapping.scala | 5 ++- .../spec/mapping/AssembleMapping.scala | 4 ++- .../spec/mapping/CoalesceMapping.scala | 5 ++- .../flowman/spec/mapping/ConformMapping.scala | 4 ++- .../spec/mapping/DeduplicateMapping.scala | 5 ++- .../spec/mapping/DistinctMapping.scala | 5 ++- .../flowman/spec/mapping/DropMapping.scala | 4 ++- .../flowman/spec/mapping/ExplodeMapping.scala | 5 ++- .../spec/mapping/ExtractJsonMapping.scala | 5 ++- .../flowman/spec/mapping/FilterMapping.scala | 4 ++- .../flowman/spec/mapping/FlattenMapping.scala | 4 ++- .../spec/mapping/HistorizeMapping.scala | 4 ++- .../flowman/spec/mapping/MockMapping.scala | 8 +++-- .../flowman/spec/mapping/NullMapping.scala | 4 ++- .../flowman/spec/mapping/ProjectMapping.scala | 4 ++- .../flowman/spec/mapping/RankMapping.scala | 5 ++- .../spec/mapping/ReadHiveMapping.scala | 6 ++-- .../spec/mapping/ReadRelationMapping.scala | 6 ++-- .../spec/mapping/ReadStreamMapping.scala | 6 ++-- .../spec/mapping/RebalanceMapping.scala | 5 ++- .../spec/mapping/RecursiveSqlMapping.scala | 4 ++- .../spec/mapping/RepartitionMapping.scala | 4 ++- .../flowman/spec/mapping/SchemaMapping.scala | 4 ++- .../flowman/spec/mapping/SortMapping.scala | 4 ++- .../flowman/spec/mapping/StackMapping.scala | 4 ++- .../spec/mapping/TemplateMapping.scala | 6 ++-- .../mapping/TransitiveChildrenMapping.scala | 4 ++- .../flowman/spec/mapping/UnionMapping.scala | 4 ++- .../flowman/spec/mapping/UnitMapping.scala | 7 ++-- .../spec/mapping/UnpackJsonMapping.scala | 4 ++- .../flowman/spec/mapping/UpsertMapping.scala | 4 ++- .../flowman/spec/mapping/ValuesMapping.scala | 6 +++- 43 files changed, 271 insertions(+), 63 deletions(-) diff --git a/examples/weather/mapping/measurements.yml b/examples/weather/mapping/measurements.yml index 2d59b83a1..9d12fc69b 100644 --- a/examples/weather/mapping/measurements.yml +++ b/examples/weather/mapping/measurements.yml @@ -5,8 +5,6 @@ mappings: relation: measurements_raw partitions: year: $year - columns: - raw_data: String # Extract multiple columns from the raw measurements data using SQL SUBSTR functions measurements_extracted: @@ -26,6 +24,33 @@ mappings: air_temperature: "CAST(SUBSTR(raw_data,88,5) AS FLOAT)/10" air_temperature_qual: "SUBSTR(raw_data,93,1)" + documentation: + columns: + - name: usaf + description: "The USAF (US Air Force) id of the weather station" + - name: wban + description: "The WBAN id of the weather station" + - name: date + description: "The date when the measurement was made" + - name: time + description: "The time when the measurement was made" + - name: report_type + description: "The report type of the measurement" + - name: wind_direction + description: "The direction from where the wind blows in degrees" + - name: wind_direction_qual + description: "The quality indicator of the wind direction. 1 means trustworthy quality." + - name: wind_observation + description: "" + - name: wind_speed + description: "The wind speed in m/s" + - name: wind_speed_qual + description: "The quality indicator of the wind speed. 1 means trustworthy quality." + - name: air_temperature + description: "The air temperature in degree Celsius" + - name: air_temperature_qual + description: "The quality indicator of the air temperature. 1 means trustworthy quality." + # This mapping refers to the processed data stored as Parquet on the local filesystem measurements: diff --git a/examples/weather/model/measurements-raw.yml b/examples/weather/model/measurements-raw.yml index d114cc6b4..c0d4f28e7 100644 --- a/examples/weather/model/measurements-raw.yml +++ b/examples/weather/model/measurements-raw.yml @@ -8,6 +8,7 @@ relations: - name: year type: integer granularity: 1 + description: "The year when the measurement was made" schema: kind: embedded fields: diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index 7a3157ed5..348e046e4 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -7,6 +7,12 @@ relations: - name: year type: integer granularity: 1 + # The following schema would use an explicitly specified schema + #schema: + # kind: avro + # file: "${project.basedir}/schema/measurements.avsc" + + # We prefer to use the inferred schema of the mapping that is written into the relation schema: - kind: avro - file: "${project.basedir}/schema/measurements.avsc" + kind: mapping + mapping: measurements_extracted diff --git a/examples/weather/schema/measurements.avsc b/examples/weather/schema/measurements.avsc index c81633a53..8e75774b5 100644 --- a/examples/weather/schema/measurements.avsc +++ b/examples/weather/schema/measurements.avsc @@ -5,19 +5,23 @@ "fields": [ { "name": "usaf", - "type": "int" + "type": "int", + "doc": "USAF station id" }, { "name": "wban", - "type": "int" + "type": "int", + "doc": "WBAN station id" }, { "name": "date", - "type": { "type": "int", "logicalType": "date" } + "type": { "type": "int", "logicalType": "date" }, + "doc": "The date when the measurement was made" }, { "name": "time", - "type": "string" + "type": "string", + "doc": "The time when the measurement was made" }, { "name": "wind_direction", diff --git a/examples/weather/schema/stations.avsc b/examples/weather/schema/stations.avsc index 13f208e46..6a7cddd56 100644 --- a/examples/weather/schema/stations.avsc +++ b/examples/weather/schema/stations.avsc @@ -5,47 +5,58 @@ "fields": [ { "name": "usaf", - "type": "int" + "type": "int", + "doc": "USAF station id" }, { "name": "wban", - "type": "int" + "type": "int", + "doc": "WBAN station id" }, { "name": "name", - "type": [ "string", "null" ] + "type": [ "string", "null" ], + "doc": "An optional name for the weather station" }, { "name": "country", - "type": [ "string", "null" ] + "type": [ "string", "null" ], + "doc": "The country the weather station belongs to" }, { "name": "state", - "type": [ "string", "null" ] + "type": [ "string", "null" ], + "doc": "Optional state within the country the weather station belongs to" }, { "name": "icao", - "type": [ "string", "null" ] + "type": [ "string", "null" ], + "doc": "" }, { "name": "latitude", - "type": [ "float", "null" ] + "type": [ "float", "null" ], + "doc": "The latitude of the geo location of the weather station" }, { "name": "longitude", - "type": [ "float", "null" ] + "type": [ "float", "null" ], + "doc": "The longitude of the geo location of the weather station" }, { "name": "elevation", - "type": [ "float", "null" ] + "type": [ "float", "null" ], + "doc": "The elevation above sea level in meters of the weather station" }, { "name": "date_begin", - "type": [ { "type": "int", "logicalType": "date" }, "null" ] + "type": [ { "type": "int", "logicalType": "date" }, "null" ], + "doc": "The date when the weather station went into service" }, { "name": "date_end", - "type": [ { "type": "int", "logicalType": "date" }, "null" ] + "type": [ { "type": "int", "logicalType": "date" }, "null" ], + "doc": "The date when the weather station went out of service" } ] } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 8b67f6e66..7e9d5c867 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -26,6 +26,7 @@ final case class ColumnReference( name:String ) extends Reference + object ColumnDoc { def merge(thisCols:Seq[ColumnDoc], otherCols:Seq[ColumnDoc]) :Seq[ColumnDoc] = { val thisColsByName = MapIgnoreCase(thisCols.map(c => c.name -> c)) @@ -38,7 +39,6 @@ object ColumnDoc { } mergedColumns ++ otherCols.filter(c => !thisColsByName.contains(c.name)) } - } final case class ColumnDoc( parent:Option[Reference], @@ -71,8 +71,16 @@ final case class ColumnDoc( ColumnDoc.merge(children, other.children) else this.children ++ other.children - val desc = description.orElse(description) + val desc = other.description.orElse(description) val tsts = tests ++ other.tests copy(description=desc, children=childs, tests=tsts) } + + /** + * Enriches a Flowman [[Field]] with documentation + */ + def enrich(field:Field) : Field = { + val desc = description.filter(_.nonEmpty).orElse(field.description) + field.copy(description = desc) + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index 30151cb89..cf2c535a1 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.documentation +import com.dimajix.common.MapIgnoreCase import com.dimajix.flowman.types.ArrayType import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.FieldType @@ -88,4 +89,18 @@ final case class SchemaDoc( .map(result.reparent) .getOrElse(result) } + + /** + * Enrich a Flowman struct with information from schema documentation + * @param schema + * @return + */ + def enrich(schema:StructType) : StructType = { + def enrichStruct(columns:Seq[ColumnDoc], struct:StructType) : StructType = { + val columnsByName = MapIgnoreCase(columns.map(c => c.name -> c)) + val fields = struct.fields.map(f => columnsByName.get(f.name).map(_.enrich(f)).getOrElse(f)) + struct.copy(fields = fields) + } + enrichStruct(columns, schema) + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 464c78252..607ffb385 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -42,9 +42,11 @@ final case class SchemaDocWrapper(schema:SchemaDoc) { final case class MappingOutputDocWrapper(output:MappingOutputDoc) { override def toString: String = output.identifier.toString + def getIdentifier() : String = output.identifier.toString + def getProject() : String = output.identifier.project.getOrElse("") + def getName() : String = output.identifier.output def getMapping() : String = output.identifier.name def getOutput() : String = output.identifier.output - def getName() : String = output.identifier.output def getDescription() : String = output.description.getOrElse("") def getSchema() : SchemaDocWrapper = output.schema.map(SchemaDocWrapper).orNull } @@ -53,12 +55,35 @@ final case class MappingOutputDocWrapper(output:MappingOutputDoc) { final case class MappingDocWrapper(mapping:MappingDoc) { override def toString: String = mapping.identifier.toString + def getIdentifier() : String = mapping.identifier.toString + def getProject() : String = mapping.identifier.project.getOrElse("") def getName() : String = mapping.identifier.name def getDescription() : String = mapping.description.getOrElse("") def getOutputs() : java.util.List[MappingOutputDocWrapper] = mapping.outputs.map(MappingOutputDocWrapper).asJava } +final case class RelationDocWrapper(relation:RelationDoc) { + override def toString: String = relation.identifier.toString + + def getIdentifier() : String = relation.identifier.toString + def getProject() : String = relation.identifier.project.getOrElse("") + def getName() : String = relation.identifier.name + def getDescription() : String = relation.description.getOrElse("") + def getSchema() : SchemaDocWrapper = relation.schema.map(SchemaDocWrapper).orNull +} + + +final case class TargetDocWrapper(target:TargetDoc) { + override def toString: String = target.identifier.toString + + def getIdentifier() : String = target.identifier.toString + def getProject() : String = target.identifier.project.getOrElse("") + def getName() : String = target.identifier.name + def getDescription() : String = target.description.getOrElse("") +} + + final case class ProjectDocWrapper(project:ProjectDoc) { override def toString: String = project.name @@ -67,4 +92,6 @@ final case class ProjectDocWrapper(project:ProjectDoc) { def getDescription() : String = project.description.getOrElse("") def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.values.map(MappingDocWrapper).toSeq.asJava + def getRelations() : java.util.List[RelationDocWrapper] = project.relations.values.map(RelationDocWrapper).toSeq.asJava + def getTargets() : java.util.List[TargetDocWrapper] = project.targets.values.map(TargetDocWrapper).toSeq.asJava } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala index 3f18cf0b7..3a3ff029c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala @@ -253,7 +253,10 @@ abstract class BaseMapping extends AbstractInstance with Mapping { val results = execute(execution, replacements) // Extract schemas - results.map { case (name,df) => name -> StructType.of(df.schema)} + val schemas = results.map { case (name,df) => name -> StructType.of(df.schema)} + + // Apply documentation + applyDocumentation(schemas) } /** @@ -279,4 +282,24 @@ abstract class BaseMapping extends AbstractInstance with Mapping { linker.input(in.mapping, in.output) ) } + + /** + * Applies optional documentation to the result of a [[describe]] + * @param schemas + * @return + */ + protected def applyDocumentation(schemas:Map[String,StructType]) : Map[String,StructType] = { + val outputDoc = documentation.map(_.outputs.map(o => o.identifier.output -> o).toMap).getOrElse(Map()) + schemas.map { case (output,schema) => + output -> outputDoc.get(output) + .flatMap(_.schema.map(_.enrich(schema))) + .getOrElse(schema) + } + } + + protected def applyDocumentation(output:String, schema:StructType) : StructType = { + documentation.flatMap(_.outputs.find(_.identifier.output == output)) + .flatMap(_.schema.map(_.enrich(schema))) + .getOrElse(schema) + } } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl index 82a4a1850..3117dd51a 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -7,9 +7,20 @@ ${mapping.description} #foreach($output in ${mapping.outputs}) - Output '${output.name}' #foreach($column in ${output.schema.columns}) - ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end + ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} #end #end +#end + +Relations: +#foreach($relation in ${project.relations}) +Relation '${relation}' + ${relation.description} + #foreach($column in ${relation.schema.columns}) + ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + #end + + #end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala index bae0f0df2..868453461 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala @@ -71,7 +71,7 @@ class MappingOutputDocSpec { class MappingDocSpec extends Spec[MappingDoc] { @JsonProperty(value="description", required=false) private var description:Option[String] = None - @JsonProperty(value="outputs", required=false) private var outputs:Map[String,MappingOutputDocSpec] = _ + @JsonProperty(value="outputs", required=false) private var outputs:Map[String,MappingOutputDocSpec] = Map() @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala index ad62e024b..ed070eb5f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala @@ -61,7 +61,10 @@ case class AliasMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala index edc422b33..d211da657 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala @@ -164,7 +164,9 @@ case class AssembleMapping( val asm = assembler val result = asm.reassemble(schema) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } private def assembler : Assembler = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala index a4dcfa448..6d48096a8 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala @@ -68,7 +68,10 @@ case class CoalesceMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala index 6e8014e23..7519888a8 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala @@ -89,7 +89,9 @@ extends BaseMapping { // Apply all transformations in order val result = transforms.foldLeft(schema)((df,xfs) => xfs.transform(df)) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } private def transforms : Seq[Transformer] = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala index 89f711efe..d7cc89a2e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala @@ -81,7 +81,10 @@ case class DeduplicateMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala index 6dbed2076..235885fc1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala @@ -71,7 +71,10 @@ case class DistinctMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala index 4531fc80d..9a3804758 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala @@ -79,7 +79,9 @@ case class DropMapping( val asm = assembler val result = asm.reassemble(schema) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } private def assembler : Assembler = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala index fe84aeba4..a3d6a7f5d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala @@ -123,7 +123,10 @@ case class ExplodeMapping( lift.transform(exploded) val result = flat.transform(lifted) - Map("main" -> result, "explode" -> exploded) + val schemas = Map("main" -> result, "explode" -> exploded) + + // Apply documentation + applyDocumentation(schemas) } private def explode = ExplodeTransformer(array, outerColumns.keep, outerColumns.drop, outerColumns.rename) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala index 2fd3ff757..019c644de 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala @@ -129,10 +129,13 @@ case class ExtractJsonMapping( val mainSchema = ftypes.StructType(schema.map(_.fields).getOrElse(Seq())) val errorSchema = ftypes.StructType(Seq(Field("record", ftypes.StringType, false))) - Map( + val schemas = Map( "main" -> mainSchema, "error" -> errorSchema ) + + // Apply documentation + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala index d48124927..703ffb392 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala @@ -69,7 +69,9 @@ case class FilterMapping( val result = input(this.input) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala index bf24e541c..325da2546 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala @@ -81,7 +81,9 @@ case class FlattenMapping( val result = xfs.transform(schema) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala index 766826e50..dbae9f5b4 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala @@ -124,7 +124,9 @@ case class HistorizeMapping( ) } - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala index 4f1b29447..aadc8f14a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala @@ -99,7 +99,10 @@ case class MockMapping( * @return */ override def describe(execution: Execution, input: Map[MappingOutputIdentifier, StructType]): Map[String, StructType] = { - mocked.outputs.map(out => out -> describe(execution, Map(), out)).toMap + val schemas = mocked.outputs.map(out => out -> describe(execution, Map(), out)).toMap + + // Apply documentation + applyDocumentation(schemas) } /** @@ -114,7 +117,8 @@ case class MockMapping( require(input != null) require(output != null && output.nonEmpty) - execution.describe(mocked, output) + val schema = execution.describe(mocked, output) + applyDocumentation(output, schema) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala index e00827c65..9a162df5b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala @@ -79,7 +79,9 @@ case class NullMapping( * @return */ override def describe(execution: Execution, input: Map[MappingOutputIdentifier, StructType]): Map[String, StructType] = { - Map("main" -> effectiveSchema) + // Apply documentation + val schemas = Map("main" -> effectiveSchema) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala index 32a76cd8b..6d3582bb3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala @@ -86,7 +86,9 @@ extends BaseMapping { val schema = input(this.input) val result = xfs.transform(schema) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } private def xfs : ProjectTransformer = ProjectTransformer(columns) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala index b7e33a0cb..945507587 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala @@ -103,7 +103,10 @@ case class RankMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala index 017729db6..d3db1b559 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala @@ -98,7 +98,7 @@ extends BaseMapping { require(execution != null) require(input != null) - val schema = if (columns.nonEmpty) { + val result = if (columns.nonEmpty) { // Use user specified schema StructType(columns) } @@ -107,7 +107,9 @@ extends BaseMapping { StructType.of(tableDf.schema) } - Map("main" -> schema) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala index 95f397c71..23b43d6b3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala @@ -105,7 +105,7 @@ case class ReadRelationMapping( require(execution != null) require(input != null) - val schema = if (columns.nonEmpty) { + val result = if (columns.nonEmpty) { // Use user specified schema StructType(columns) } @@ -114,7 +114,9 @@ case class ReadRelationMapping( relation.describe(execution) } - Map("main" -> schema) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala index 1a0eb5ed4..4db6b556f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala @@ -97,7 +97,7 @@ case class ReadStreamMapping ( require(execution != null) require(input != null) - val schema = if (columns.nonEmpty) { + val result = if (columns.nonEmpty) { // Use user specified schema StructType(columns) } @@ -106,7 +106,9 @@ case class ReadStreamMapping ( relation.describe(execution) } - Map("main" -> schema) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala index 64fd789cf..8a93e8bcf 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala @@ -68,7 +68,10 @@ case class RebalanceMapping( require(input != null) val result = input(this.input) - Map("main" -> result) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala index 17ada2fff..af7ba692a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala @@ -142,7 +142,9 @@ extends BaseMapping { firstDf(spark, statement) } - Map("main" -> StructType.of(result.schema)) + // Apply documentation + val schemas = Map("main" -> StructType.of(result.schema)) + applyDocumentation(schemas) } private def statement : String = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala index a8bc9b7c7..446659433 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala @@ -83,7 +83,9 @@ case class RepartitionMapping( val result = input(this.input) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala index a18aac474..f600da62e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala @@ -100,7 +100,9 @@ extends BaseMapping { StructType(columns) } - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala index 6fe5a3c3a..10cbb5626 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala @@ -79,7 +79,9 @@ case class SortMapping( val result = input(this.input) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala index 4cde6f19b..02cef5365 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala @@ -86,7 +86,9 @@ case class StackMapping( val result = xfs.transform(schema) val assembledResult = asm.map(_.reassemble(result)).getOrElse(result) - Map("main" -> assembledResult) + // Apply documentation + val schemas = Map("main" -> assembledResult) + applyDocumentation(schemas) } private lazy val xfs : StackTransformer = diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala index 4c92de174..dacc945d6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala @@ -117,7 +117,8 @@ case class TemplateMapping( require(execution != null) require(input != null) - mappingInstance.describe(execution, input) + val schemas = mappingInstance.describe(execution, input) + applyDocumentation(schemas) } /** @@ -131,7 +132,8 @@ case class TemplateMapping( require(input != null) require(output != null && output.nonEmpty) - mappingInstance.describe(execution, input, output) + val schema = mappingInstance.describe(execution, input, output) + applyDocumentation(output, schema) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala index a9d7a8c79..86c33cb36 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala @@ -132,7 +132,9 @@ case class TransitiveChildrenMapping( childColumns.map(n => fieldsByName(n.toLowerCase(Locale.ROOT))) val result = StructType(columns) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala index d70d1beae..636564c6c 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala @@ -101,7 +101,9 @@ case class UnionMapping( xfs.transformSchemas(schemas) } - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala index e8c86fd97..f3491b909 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala @@ -106,11 +106,13 @@ case class UnitMapping( require(execution != null) require(input != null) - mappingInstances + val schemas = mappingInstances .filter(_._2.outputs.contains("main")) .keys .map(name => name -> describe(execution, input, name)) .toMap + + applyDocumentation(schemas) } /** @@ -138,11 +140,12 @@ case class UnitMapping( .toMap } - mappingInstances + val schema = mappingInstances .filter(_._2.outputs.contains("main")) .get(output) .map(mapping => describe(mapping, "main")) .getOrElse(throw new NoSuchElementException(s"Cannot find output '$output' in unit mapping '$identifier'")) + applyDocumentation(output, schema) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala index 659a27138..2a1672f38 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala @@ -105,7 +105,9 @@ case class UnpackJsonMapping( val fields = schema.fields ++ columns.map(c => Field(Option(c.alias).getOrElse(c.name), StructType(c.schema.fields))) val result = StructType(fields) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala index fe1c204fd..2066bbba3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala @@ -88,7 +88,9 @@ case class UpsertMapping( val result = input(this.input) - Map("main" -> result) + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala index 3142e9e35..b6b949861 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala @@ -96,7 +96,11 @@ case class ValuesMapping( * @return */ override def describe(execution: Execution, input: Map[MappingOutputIdentifier, StructType]): Map[String, StructType] = { - Map("main" -> StructType(schema.map(_.fields).getOrElse(columns))) + val result = StructType(schema.map(_.fields).getOrElse(columns)) + + // Apply documentation + val schemas = Map("main" -> result) + applyDocumentation(schemas) } } From 661816ea02d22dcf7ecc034bce3696621a48d8d3 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 9 Feb 2022 10:01:40 +0100 Subject: [PATCH 17/95] Incorporate documentation with relation descriptions --- .../scala/com/dimajix/flowman/model/Relation.scala | 10 +++++++++- .../dimajix/flowman/spec/relation/KafkaRelation.scala | 4 +++- .../dimajix/flowman/spec/relation/JdbcRelation.scala | 4 +++- .../dimajix/flowman/spec/relation/MockRelation.scala | 4 +++- .../dimajix/flowman/spec/relation/NullRelation.scala | 4 +++- .../dimajix/flowman/spec/relation/ValuesRelation.scala | 4 +++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala index 226173236..07c185fc0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala @@ -322,7 +322,7 @@ abstract class BaseRelation extends AbstractInstance with Relation { */ override def describe(execution:Execution) : StructType = { val partitions = SetIgnoreCase(this.partitions.map(_.name)) - if (!fields.forall(f => partitions.contains(f.name))) { + val result = if (!fields.forall(f => partitions.contains(f.name))) { // Use given fields if relation contains valid list of fields in addition to the partition columns StructType(fields) } @@ -331,6 +331,8 @@ abstract class BaseRelation extends AbstractInstance with Relation { val df = read(execution) StructType.of(df.schema) } + + applyDocumentation(result) } /** @@ -509,6 +511,12 @@ abstract class BaseRelation extends AbstractInstance with Relation { } .getOrElse(df) } + + protected def applyDocumentation(schema:StructType) : StructType = { + documentation + .flatMap(_.schema.map(_.enrich(schema))) + .getOrElse(schema) + } } diff --git a/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala b/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala index b3a760990..7e29e6f34 100644 --- a/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala +++ b/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala @@ -122,7 +122,9 @@ case class KafkaRelation( * @return */ override def describe(execution: Execution): types.StructType = { - types.StructType(fields) + val result = types.StructType(fields) + + applyDocumentation(result) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 69f845b29..674d20912 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -148,7 +148,7 @@ case class JdbcRelation( * @return */ override def describe(execution:Execution) : FlowmanStructType = { - if (schema.nonEmpty) { + val result = if (schema.nonEmpty) { FlowmanStructType(fields) } else { @@ -156,6 +156,8 @@ case class JdbcRelation( JdbcUtils.getSchema(con, tableIdentifier, options) } } + + applyDocumentation(result) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala index a562c5e44..f5387043e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala @@ -226,7 +226,9 @@ case class MockRelation( * @return */ override def describe(execution: Execution): types.StructType = { - types.StructType(mocked.fields) + val result = types.StructType(mocked.fields) + + applyDocumentation(result) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala index b2385fc5b..9ef2acebe 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala @@ -167,7 +167,9 @@ case class NullRelation( * @return */ override def describe(execution:Execution) : types.StructType = { - types.StructType(fields) + val result = types.StructType(fields) + + applyDocumentation(result) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala index b3fb9c560..9ad8f5581 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala @@ -214,7 +214,9 @@ case class ValuesRelation( * @return */ override def describe(execution: Execution): types.StructType = { - types.StructType(effectiveSchema.fields) + val result = types.StructType(effectiveSchema.fields) + + applyDocumentation(result) } /** From 3f7a9e05ee388b5bbfe49f2517949611c5f4454e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 9 Feb 2022 13:16:55 +0100 Subject: [PATCH 18/95] More work on Flowman documentation subsystem --- docs/spec/target/blackhole.md | 3 + docs/spec/target/compare.md | 5 ++ docs/spec/target/console.md | 2 + docs/spec/target/copy-file.md | 3 + docs/spec/target/copy.md | 3 + docs/spec/target/count.md | 2 + docs/spec/target/delete-file.md | 3 + docs/spec/target/deltaVacuum.md | 3 + docs/spec/target/file.md | 3 + docs/spec/target/hive-database.md | 11 ++++ docs/spec/target/local.md | 2 + docs/spec/target/merge-files.md | 2 + docs/spec/target/merge.md | 3 + docs/spec/target/null.md | 9 +++ docs/spec/target/relation.md | 3 + docs/spec/target/sftp-upload.md | 3 +- docs/spec/target/stream.md | 3 + docs/spec/target/truncate.md | 3 + docs/spec/target/validate.md | 3 + docs/spec/target/verify.md | 3 + examples/weather/target/aggregates.yml | 1 + examples/weather/target/measurements.yml | 1 + examples/weather/target/stations.yml | 1 + .../flowman/documentation/ColumnDoc.scala | 10 +++- .../flowman/documentation/MappingDoc.scala | 18 +++++- .../flowman/documentation/ProjectDoc.scala | 1 + .../flowman/documentation/RelationDoc.scala | 9 ++- .../flowman/documentation/SchemaDoc.scala | 9 ++- .../documentation/TargetCollector.scala | 8 ++- .../flowman/documentation/TargetDoc.scala | 18 +++++- .../flowman/documentation/velocity.scala | 20 +++++++ .../com/dimajix/flowman/model/Target.scala | 16 ++++++ .../flowman/documentation/text/project.vtl | 55 ++++++++++++++----- .../flowman/spec/relation/RelationSpec.scala | 2 +- .../flowman/spec/target/TargetSpec.scala | 6 +- .../flowman/spec/target/TemplateTarget.scala | 8 +++ 36 files changed, 227 insertions(+), 28 deletions(-) diff --git a/docs/spec/target/blackhole.md b/docs/spec/target/blackhole.md index ac37a385a..d034a497d 100644 --- a/docs/spec/target/blackhole.md +++ b/docs/spec/target/blackhole.md @@ -17,6 +17,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `blackhole` +* `description` **(optional)** *(type: string)*: +Optional descriptive text of the build target + * `mapping` **(mandatory)** *(type: string)*: Specifies the name of the mapping output to be materialized diff --git a/docs/spec/target/compare.md b/docs/spec/target/compare.md index 3ec0cf5b2..6edd3ed74 100644 --- a/docs/spec/target/compare.md +++ b/docs/spec/target/compare.md @@ -24,9 +24,14 @@ targets: ## Fields * `kind` **(mandatory)** *(type: string)*: `relation` + +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `actual` **(mandatory)** *(type: dataset)*: Specifies the data set containing the actual data. Often you will either use a relation written to by Flowman or a mapping. + * `expected` **(mandatory)** *(type: dataset)*: Specifies the data set containing the expected data. In most cases you probably will use a file data set referencing some predefined results diff --git a/docs/spec/target/console.md b/docs/spec/target/console.md index bc7df3cb3..af7e883ef 100644 --- a/docs/spec/target/console.md +++ b/docs/spec/target/console.md @@ -16,6 +16,8 @@ targets: ## Fields * `kind` **(mandatory)** *(type: string)*: `console` +* `description` **(optional)** *(type: string)*: +Optional descriptive text of the build target * `input` **(mandatory)** *(type: dataset)*: Specified the [dataset](../dataset/index.md) containing the records to be dumped * `limit` **(optional)** *(type: integer)* *(default: 100)*: diff --git a/docs/spec/target/copy-file.md b/docs/spec/target/copy-file.md index a50d813fd..40c5e6823 100644 --- a/docs/spec/target/copy-file.md +++ b/docs/spec/target/copy-file.md @@ -4,6 +4,9 @@ * `kind` **(mandatory)** *(type: string)*: `copyFile` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `source` **(mandatory)** *(type: string)*: * `target` **(mandatory)** *(type: string)*: diff --git a/docs/spec/target/copy.md b/docs/spec/target/copy.md index 6f682e1a1..3d20f7291 100644 --- a/docs/spec/target/copy.md +++ b/docs/spec/target/copy.md @@ -27,6 +27,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `copy` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `source` **(mandatory)** *(type: dataset)*: Specifies the source data set to be copied from. diff --git a/docs/spec/target/count.md b/docs/spec/target/count.md index 9a5496404..d388f8a16 100644 --- a/docs/spec/target/count.md +++ b/docs/spec/target/count.md @@ -10,6 +10,8 @@ targets: ## Fields * `kind` **(mandatory)** *(string)*: `count` + * `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target * `mapping` **(mandatory)** *(string)*: Specifies the name of the input mapping to be counted diff --git a/docs/spec/target/delete-file.md b/docs/spec/target/delete-file.md index 75cc2215f..060d2c458 100644 --- a/docs/spec/target/delete-file.md +++ b/docs/spec/target/delete-file.md @@ -13,6 +13,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `deleteFile` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `location` **(mandatory)** *(type: string)*: diff --git a/docs/spec/target/deltaVacuum.md b/docs/spec/target/deltaVacuum.md index 96efd545b..d985b5bca 100644 --- a/docs/spec/target/deltaVacuum.md +++ b/docs/spec/target/deltaVacuum.md @@ -37,6 +37,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `deleteFile` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `relation` **(mandatory)** *(type: string or relation)*: Either the name of a `deltaTable` or `deltaFile` relation or alternatively an embedded delta relation diff --git a/docs/spec/target/file.md b/docs/spec/target/file.md index a68daf86d..7749a034c 100644 --- a/docs/spec/target/file.md +++ b/docs/spec/target/file.md @@ -26,6 +26,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `file` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `mapping` **(optional)** *(type: string)*: Specifies the name of the input mapping to be written diff --git a/docs/spec/target/hive-database.md b/docs/spec/target/hive-database.md index ccc0ecb15..88a9139a5 100644 --- a/docs/spec/target/hive-database.md +++ b/docs/spec/target/hive-database.md @@ -12,6 +12,17 @@ targets: database: "my_database" ``` +## Fields + +* `kind` **(mandatory)** *(type: string)*: `hiveDatabase` + +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + +* `database` **(mandatory)** *(type: string)*: + Name of the Hive database to be created + + ## Supported Phases * `CREATE` - Ensures that the specified Hive database exists and creates one if it is not found * `VERIFY` - Verifies that the specified Hive database exists diff --git a/docs/spec/target/local.md b/docs/spec/target/local.md index 1069be816..c81446811 100644 --- a/docs/spec/target/local.md +++ b/docs/spec/target/local.md @@ -18,6 +18,8 @@ targets: ## Fields * `kind` **(mandatory)** *(string)*: `local` + * `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target * `mapping` **(mandatory)** *(string)*: Specifies the name of the input mapping to be counted * `filename` **(mandatory)** *(string)*: diff --git a/docs/spec/target/merge-files.md b/docs/spec/target/merge-files.md index a512d96af..9b5af4e4b 100644 --- a/docs/spec/target/merge-files.md +++ b/docs/spec/target/merge-files.md @@ -16,6 +16,8 @@ targets: ## Fields * `kind` **(mandatory)** *(string)*: `mergeFiles` + * `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target * `source` **(mandatory)** *(string)*: Source directory containing all files to be concatenated * `target` **(optional)** *(string)*: Name of single target file * `overwrite` **(optional)** *(boolean)* *(default: true)*: diff --git a/docs/spec/target/merge.md b/docs/spec/target/merge.md index cada0694e..1b26a555a 100644 --- a/docs/spec/target/merge.md +++ b/docs/spec/target/merge.md @@ -69,6 +69,9 @@ relations: * `kind` **(mandatory)** *(type: string)*: `merge` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `mapping` **(optional)** *(type: string)*: Specifies the name of the input mapping to be written diff --git a/docs/spec/target/null.md b/docs/spec/target/null.md index a66a24c68..586c91a57 100644 --- a/docs/spec/target/null.md +++ b/docs/spec/target/null.md @@ -11,6 +11,15 @@ targets: kind: null ``` + +## Fields + +* `kind` **(mandatory)** *(type: string)*: `null` + +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + + ## Supported Phases * `CREATE` * `MIGRATE` diff --git a/docs/spec/target/relation.md b/docs/spec/target/relation.md index 1d81657c6..88d4b6736 100644 --- a/docs/spec/target/relation.md +++ b/docs/spec/target/relation.md @@ -55,6 +55,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `relation` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `mapping` **(optional)** *(type: string)*: Specifies the name of the input mapping to be written diff --git a/docs/spec/target/sftp-upload.md b/docs/spec/target/sftp-upload.md index df2d206a9..a0ff8ef24 100644 --- a/docs/spec/target/sftp-upload.md +++ b/docs/spec/target/sftp-upload.md @@ -47,8 +47,9 @@ jobs: ## Fields * `kind` **(mandatory)** *(type: string)*: `sftp-upload` + * `description` **(optional)** *(type: string)*: -A textual description of the task. +A textual description of the build target. * `source` **(mandatory)** *(type: string)*: Specifies the source location in the Hadoop compatible filesystem. This may be either a single diff --git a/docs/spec/target/stream.md b/docs/spec/target/stream.md index 0ee5e82df..379927cc1 100644 --- a/docs/spec/target/stream.md +++ b/docs/spec/target/stream.md @@ -41,6 +41,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `stream` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `mapping` **(optional)** *(type: string)*: Specifies the name of the input mapping to be read from diff --git a/docs/spec/target/truncate.md b/docs/spec/target/truncate.md index fe8ac2004..0badc57b5 100644 --- a/docs/spec/target/truncate.md +++ b/docs/spec/target/truncate.md @@ -21,6 +21,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `truncate` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `relation` **(mandatory)** *(type: string)*: Specifies the name of the relation to truncate. diff --git a/docs/spec/target/validate.md b/docs/spec/target/validate.md index 7f69b1c0a..31a563ab6 100644 --- a/docs/spec/target/validate.md +++ b/docs/spec/target/validate.md @@ -29,6 +29,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `validate` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `assertions` **(optional)** *(type: map:assertion)*: Map of [assertions](../assertion/index.md) to be executed. The validation is marked as *failed* if a single assertion fails. diff --git a/docs/spec/target/verify.md b/docs/spec/target/verify.md index 279591aac..db4681c54 100644 --- a/docs/spec/target/verify.md +++ b/docs/spec/target/verify.md @@ -31,6 +31,9 @@ targets: * `kind` **(mandatory)** *(type: string)*: `verify` +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + * `assertions` **(optional)** *(type: map:assertion)*: Map of [assertions](../assertion/index.md) to be executed. The verification is marked as *failed* if a single assertion fails. diff --git a/examples/weather/target/aggregates.yml b/examples/weather/target/aggregates.yml index a1109f2ab..0cf9a007c 100644 --- a/examples/weather/target/aggregates.yml +++ b/examples/weather/target/aggregates.yml @@ -1,6 +1,7 @@ targets: aggregates: kind: relation + description: "Write aggregated measurements per year" mapping: aggregates relation: aggregates partition: diff --git a/examples/weather/target/measurements.yml b/examples/weather/target/measurements.yml index 73e05bad0..cfdb09314 100644 --- a/examples/weather/target/measurements.yml +++ b/examples/weather/target/measurements.yml @@ -1,6 +1,7 @@ targets: measurements: kind: relation + description: "Write extracted measurements per year" mapping: measurements_extracted relation: measurements partition: diff --git a/examples/weather/target/stations.yml b/examples/weather/target/stations.yml index ad32188b5..48b689669 100644 --- a/examples/weather/target/stations.yml +++ b/examples/weather/target/stations.yml @@ -1,5 +1,6 @@ targets: stations: kind: relation + description: "Write stations" mapping: stations_raw relation: stations diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 7e9d5c867..050b07733 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -24,7 +24,15 @@ import com.dimajix.flowman.types.Field final case class ColumnReference( override val parent:Option[Reference], name:String -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(col:ColumnReference) => col.toString + "." + name + case Some(ref) => ref.toString + "/column=" + name + case None => name + } + } +} object ColumnDoc { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index 6164422d0..1224d8d84 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -23,7 +23,14 @@ import com.dimajix.flowman.model.MappingOutputIdentifier final case class MappingOutputReference( override val parent:Option[Reference], name:String -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/output=" + name + case None => name + } + } +} final case class MappingOutputDoc( @@ -57,7 +64,14 @@ final case class MappingOutputDoc( final case class MappingReference( override val parent:Option[Reference], name:String -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/mapping=" + name + case None => name + } + } +} final case class MappingDoc( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala index ce8c725f2..5195939cd 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -24,6 +24,7 @@ import com.dimajix.flowman.model.TargetIdentifier final case class ProjectReference( name:String ) extends Reference { + override def toString: String = "/project=" + name override def parent: Option[Reference] = None } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index 33f9bdea7..b0d7a1a1a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -24,7 +24,14 @@ import com.dimajix.flowman.types.FieldValue final case class RelationReference( parent:Option[Reference], name:String -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/relation=" + name + case None => name + } + } +} final case class RelationDoc( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index cf2c535a1..b2ac387c5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -26,7 +26,14 @@ import com.dimajix.flowman.types.StructType final case class SchemaReference( override val parent:Option[Reference] -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/schema" + case None => "schema" + } + } +} object SchemaDoc { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index 70a526fb0..ccf2d6961 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -18,13 +18,14 @@ package com.dimajix.flowman.documentation import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.TargetRef import com.dimajix.flowman.model.Target class TargetCollector extends Collector { override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference - val docs = graph.targets.map(t => t.target.identifier -> document(execution, parent, t.target)).toMap + val docs = graph.targets.map(t => t.target.identifier -> document(execution, parent, t)).toMap documentation.copy(targets = docs) } @@ -34,11 +35,12 @@ class TargetCollector extends Collector { * @param parent * @return */ - private def document(execution: Execution, parent:Reference, target:Target) : TargetDoc = { + private def document(execution: Execution, parent:Reference, node:TargetRef) : TargetDoc = { + val target = node.target val doc = TargetDoc( Some(parent), target.identifier, - None, + target.description, Seq(), Seq(), Seq() diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala index 8a7e794a0..b61b6fe3f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -24,7 +24,14 @@ import com.dimajix.flowman.model.TargetIdentifier final case class TargetPhaseReference( override val parent:Option[Reference], phase:Phase -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/phase=" + phase.upper + case None => phase.upper + } + } +} final case class TargetPhaseDoc( @@ -45,7 +52,14 @@ final case class TargetPhaseDoc( final case class TargetReference( override val parent:Option[Reference], name:String -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/target=" + name + case None => name + } + } +} final case class TargetDoc( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 607ffb385..b7a144f26 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -22,6 +22,7 @@ import scala.collection.JavaConverters._ final case class ColumnDocWrapper(column:ColumnDoc) { override def toString: String = column.name + def getReference() : String = column.reference.toString def getName() : String = column.name def getNullable() : Boolean = column.nullable def getType() : String = column.typeName @@ -34,6 +35,7 @@ final case class ColumnDocWrapper(column:ColumnDoc) { final case class SchemaDocWrapper(schema:SchemaDoc) { + def getReference() : String = schema.reference.toString def getDescription() : String = schema.description.getOrElse("") def getColumns() : java.util.List[ColumnDocWrapper] = schema.columns.map(ColumnDocWrapper).asJava } @@ -42,6 +44,7 @@ final case class SchemaDocWrapper(schema:SchemaDoc) { final case class MappingOutputDocWrapper(output:MappingOutputDoc) { override def toString: String = output.identifier.toString + def getReference() : String = output.reference.toString def getIdentifier() : String = output.identifier.toString def getProject() : String = output.identifier.project.getOrElse("") def getName() : String = output.identifier.output @@ -55,10 +58,12 @@ final case class MappingOutputDocWrapper(output:MappingOutputDoc) { final case class MappingDocWrapper(mapping:MappingDoc) { override def toString: String = mapping.identifier.toString + def getReference() : String = mapping.reference.toString def getIdentifier() : String = mapping.identifier.toString def getProject() : String = mapping.identifier.project.getOrElse("") def getName() : String = mapping.identifier.name def getDescription() : String = mapping.description.getOrElse("") + def getInputs() : java.util.List[String] = mapping.inputs.map(_.toString).asJava def getOutputs() : java.util.List[MappingOutputDocWrapper] = mapping.outputs.map(MappingOutputDocWrapper).asJava } @@ -66,6 +71,7 @@ final case class MappingDocWrapper(mapping:MappingDoc) { final case class RelationDocWrapper(relation:RelationDoc) { override def toString: String = relation.identifier.toString + def getReference() : String = relation.reference.toString def getIdentifier() : String = relation.identifier.toString def getProject() : String = relation.identifier.project.getOrElse("") def getName() : String = relation.identifier.name @@ -74,13 +80,27 @@ final case class RelationDocWrapper(relation:RelationDoc) { } +final case class TargetPhaseDocWrapper(phase:TargetPhaseDoc) { + override def toString: String = phase.phase.upper + + def getReference() : String = phase.reference.toString + def getName() : String = phase.phase.upper + def getDescription() : String = phase.description.getOrElse("") +} + + final case class TargetDocWrapper(target:TargetDoc) { override def toString: String = target.identifier.toString + def getReference() : String = target.reference.toString def getIdentifier() : String = target.identifier.toString def getProject() : String = target.identifier.project.getOrElse("") def getName() : String = target.identifier.name def getDescription() : String = target.description.getOrElse("") + def getPhases() : java.util.List[TargetPhaseDocWrapper] = target.phases.map(TargetPhaseDocWrapper).asJava + + def getOutputs() : java.util.List[String] = target.outputs.map(_.toString).asJava + def getInputs() : java.util.List[String] = target.inputs.map(_.toString).asJava } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala index 86f451ce5..89de0bf4f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala @@ -62,6 +62,7 @@ object Target { Metadata(context, name, Category.TARGET, kind), Seq(), Seq(), + None, None ) } @@ -71,6 +72,7 @@ object Target { metadata:Metadata, before: Seq[TargetIdentifier], after: Seq[TargetIdentifier], + description:Option[String], documentation: Option[TargetDoc] ) extends Instance.Properties[Properties] { override val namespace : Option[Namespace] = context.namespace @@ -97,6 +99,12 @@ trait Target extends Instance { */ def identifier : TargetIdentifier + /** + * Returns a description of the build target + * @return + */ + def description : Option[String] + /** * Returns a (static) documentation of this target * @return @@ -178,8 +186,16 @@ abstract class BaseTarget extends AbstractInstance with Target { */ override def identifier : TargetIdentifier = instanceProperties.identifier + /** + * Returns a description of the build target + * + * @return + */ + override def description: Option[String] = instanceProperties.description + /** * Returns a (static) documentation of this target + * * @return */ override def documentation : Option[TargetDoc] = instanceProperties.documentation diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl index 3117dd51a..cce8826ca 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -1,26 +1,53 @@ Project: ${project.name} version ${project.version} +Description: ${project.description} -Mappings: + +=========== Mappings: ================================================================================================= #foreach($mapping in ${project.mappings}) -Mapping '${mapping}' -${mapping.description} -#foreach($output in ${mapping.outputs}) - - Output '${output.name}' -#foreach($column in ${output.schema.columns}) +Mapping '${mapping}' (${mapping.reference}) + Description: ${mapping.description} + Inputs: + #foreach($input in ${mapping.inputs}) + - ${input} + #end + Outputs: + #foreach($output in ${mapping.outputs}) + - '${output.name}': + #foreach($column in ${output.schema.columns}) ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + #end + #end + #end -#end +=========== Relations: ================================================================================================ +#foreach($relation in ${project.relations}) +Relation '${relation}' (${relation.reference}) + Description: ${relation.description} + Schema: + #foreach($column in ${relation.schema.columns}) + ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + #end + #end -Relations: -#foreach($relation in ${project.relations}) -Relation '${relation}' - ${relation.description} - #foreach($column in ${relation.schema.columns}) - ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} - #end +=========== Targets: ================================================================================================== +#foreach($target in ${project.targets}) +Target '${target}' (${target.reference}) + Description: ${target.description} + Inputs: + #foreach($input in ${target.inputs}) + - ${input} + #end + Outputs: + #foreach($output in ${target.outputs}) + - ${output} + #end + Phases: + #foreach($phase in ${target.phases}) + - ${phase.name} ${phase.description} + #end #end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala index f3072730d..51d36c603 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala @@ -79,7 +79,7 @@ abstract class RelationSpec extends NamedSpec[Relation] { Relation.Properties( context, metadata.map(_.instantiate(context, name, Category.RELATION, kind)).getOrElse(Metadata(context, name, Category.RELATION, kind)), - description.map(context.evaluate), + context.evaluate(description), documentation.map(_.instantiate(context)) ) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala index 5e1030b87..cd3b5fb61 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala @@ -68,10 +68,11 @@ object TargetSpec extends TypeRegistry[TargetSpec] { new JsonSubTypes.Type(name = "verify", value = classOf[VerifyTargetSpec]) )) abstract class TargetSpec extends NamedSpec[Target] { - @JsonProperty(value = "kind", required = true) protected var kind: String = _ + @JsonProperty(value = "kind", required=true) protected var kind: String = _ @JsonProperty(value = "before", required=false) protected[spec] var before:Seq[String] = Seq() @JsonProperty(value = "after", required=false) protected[spec] var after:Seq[String] = Seq() - @JsonProperty(value="documentation", required = false) private var documentation: Option[TargetDocSpec] = None + @JsonProperty(value="description", required = false) private var description: Option[String] = None + @JsonProperty(value = "documentation", required=false) private var documentation: Option[TargetDocSpec] = None override def instantiate(context: Context): Target @@ -88,6 +89,7 @@ abstract class TargetSpec extends NamedSpec[Target] { metadata.map(_.instantiate(context, name, Category.TARGET, kind)).getOrElse(Metadata(context, name, Category.TARGET, kind)), before.map(context.evaluate).map(TargetIdentifier.parse), after.map(context.evaluate).map(TargetIdentifier.parse), + context.evaluate(description), documentation.map(_.instantiate(context)) ) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala index bd3f6ed14..2527d1235 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TemplateTarget.scala @@ -56,8 +56,16 @@ case class TemplateTarget( } } + /** + * Returns a description of the build target + * + * @return + */ + override def description: Option[String] = instanceProperties.description.orElse(targetInstance.description) + /** * Returns a (static) documentation of this target + * * @return */ override def documentation : Option[TargetDoc] = targetInstance.documentation.map(_.merge(instanceProperties.documentation)) From 7232c73c3da9591c2f8ae06b4f0db51e42ff1edc Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 9 Feb 2022 16:38:38 +0100 Subject: [PATCH 19/95] More work on Flowman documentation subsystem --- docs/documenting/index.md | 28 +++++++++++++++++++ docs/documenting/mappings.md | 1 + docs/documenting/relations.md | 1 + docs/documenting/targets.md | 1 + docs/documenting/tests.md | 5 ++++ docs/index.md | 1 + .../documentation/RelationCollector.scala | 28 +++++++++++++++---- .../flowman/documentation/RelationDoc.scala | 1 + .../documentation/TargetCollector.scala | 24 ++++++++++++++-- .../flowman/documentation/velocity.scala | 4 +++ .../com/dimajix/flowman/model/velocity.scala | 8 ++++++ .../flowman/documentation/text/project.vtl | 8 ++++++ .../spec/documentation/RelationDocSpec.scala | 1 + 13 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 docs/documenting/index.md create mode 100644 docs/documenting/mappings.md create mode 100644 docs/documenting/relations.md create mode 100644 docs/documenting/targets.md create mode 100644 docs/documenting/tests.md diff --git a/docs/documenting/index.md b/docs/documenting/index.md new file mode 100644 index 000000000..8ea4556e3 --- /dev/null +++ b/docs/documenting/index.md @@ -0,0 +1,28 @@ +# Documenting Projects and Models + +Flowman supports to automatically generate a documentation of your project. The documentation can either include all +major entities like mappings, relations and targets. Or you may want to focus only on some aspects like the relations +which is useful for providing a documentation of the data model. + +```eval_rst +.. toctree:: + :maxdepth: 1 + :glob: + + * +``` + +## Providing Descriptions + +Although Flowman will generate many valuable documentation bits by inspecting the project, the most important entities +(relations, mappings and targets) also provide the ability to manually and explicitly add documentation to them. This +documentation will override any automatically inferred information. + + +## Generating Documentation + +Generating the documentation is as easy as running [flowexec](../cli/flowexec.md) as follows: + +```shell +flowexec -f my_project_directory documentation generate +``` diff --git a/docs/documenting/mappings.md b/docs/documenting/mappings.md new file mode 100644 index 000000000..2f577ed08 --- /dev/null +++ b/docs/documenting/mappings.md @@ -0,0 +1 @@ +# Documenting Mappings diff --git a/docs/documenting/relations.md b/docs/documenting/relations.md new file mode 100644 index 000000000..b57bbf89a --- /dev/null +++ b/docs/documenting/relations.md @@ -0,0 +1 @@ +# Documenting Relations diff --git a/docs/documenting/targets.md b/docs/documenting/targets.md new file mode 100644 index 000000000..1208749af --- /dev/null +++ b/docs/documenting/targets.md @@ -0,0 +1 @@ +# Documenting Targets diff --git a/docs/documenting/tests.md b/docs/documenting/tests.md new file mode 100644 index 000000000..34ceda97e --- /dev/null +++ b/docs/documenting/tests.md @@ -0,0 +1,5 @@ +# Testing Model Properties + +In addition to provide pure descriptions of model entities, the documentation framework in Flowman also provides +the ability to specify model properties (like unqiue values in a column, not null etc). These properties will not only +be part of the documentation, they will also be verified as part of generating the documentation. diff --git a/docs/index.md b/docs/index.md index ba6b0cc7a..fefaf6c06 100644 --- a/docs/index.md +++ b/docs/index.md @@ -97,6 +97,7 @@ Flowman also provides optional plugins which extend functionality. You can find lifecycle spec/index testing/index + documenting/index cli/index history-server/index cookbook/index diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 46961b0f6..59574abfe 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -18,13 +18,17 @@ package com.dimajix.flowman.documentation import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.InputMapping +import com.dimajix.flowman.graph.MappingRef +import com.dimajix.flowman.graph.RelationRef +import com.dimajix.flowman.graph.WriteRelation import com.dimajix.flowman.model.Relation class RelationCollector extends Collector { override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference - val docs = graph.relations.map(t => t.relation.identifier -> document(execution, parent, t.relation)).toMap + val docs = graph.relations.map(t => t.relation.identifier -> document(execution, parent, t)).toMap documentation.copy(relations = docs) } @@ -35,19 +39,33 @@ class RelationCollector extends Collector { * @param parent * @return */ - private def document(execution:Execution, parent:Reference, relation:Relation) : RelationDoc = { + private def document(execution:Execution, parent:Reference, node:RelationRef) : RelationDoc = { + val inputs = node.incoming.flatMap { + case write:WriteRelation => + write.input.incoming.flatMap { + case map: InputMapping => + val mapref = MappingReference(Some(parent), map.input.name) + val outref = MappingOutputReference(Some(mapref), map.pin) + Some(outref) + case _ => None + } + case _ => Seq() + } + + val relation = node.relation val doc = RelationDoc( Some(parent), relation.identifier, relation.description, None, + inputs, relation.provides.toSeq, Map() ) val ref = doc.reference val desc = SchemaDoc.ofStruct(ref, relation.describe(execution)) - val schemaDoc = relation.schema.map { schema => + val schema = relation.schema.map { schema => val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) SchemaDoc( Some(ref), @@ -56,8 +74,8 @@ class RelationCollector extends Collector { Seq() ) } - val mergedDoc = desc.merge(schemaDoc) + val mergedSchema = desc.merge(schema) - doc.copy(schema = Some(mergedDoc)).merge(relation.documentation) + doc.copy(schema = Some(mergedSchema)).merge(relation.documentation) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index b0d7a1a1a..2adf65869 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -39,6 +39,7 @@ final case class RelationDoc( identifier:RelationIdentifier, description:Option[String], schema:Option[SchemaDoc], + inputs:Seq[Reference], provides:Seq[ResourceIdentifier], partitions:Map[String,FieldValue] = Map() ) extends EntityDoc { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index ccf2d6961..4da0c41ca 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -18,7 +18,10 @@ package com.dimajix.flowman.documentation import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.InputMapping +import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.graph.TargetRef +import com.dimajix.flowman.graph.WriteRelation import com.dimajix.flowman.model.Target @@ -36,14 +39,31 @@ class TargetCollector extends Collector { * @return */ private def document(execution: Execution, parent:Reference, node:TargetRef) : TargetDoc = { + val inputs = node.incoming.flatMap { + case map: InputMapping => + val mapref = MappingReference(Some(parent), map.input.name) + val outref = MappingOutputReference(Some(mapref), map.pin) + Some(outref) + case read: ReadRelation => + val relref = RelationReference(Some(parent), read.input.name) + Some(relref) + case _ => None + } + val outputs = node.outgoing.flatMap { + case write:WriteRelation => + val relref = RelationReference(Some(parent), write.output.name) + Some(relref) + case _ => None + } + val target = node.target val doc = TargetDoc( Some(parent), target.identifier, target.description, Seq(), - Seq(), - Seq() + inputs, + outputs ) val ref = doc.reference diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index b7a144f26..55d6a0fa9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -18,6 +18,8 @@ package com.dimajix.flowman.documentation import scala.collection.JavaConverters._ +import com.dimajix.flowman.model.ResourceIdentifierWrapper + final case class ColumnDocWrapper(column:ColumnDoc) { override def toString: String = column.name @@ -77,6 +79,8 @@ final case class RelationDocWrapper(relation:RelationDoc) { def getName() : String = relation.identifier.name def getDescription() : String = relation.description.getOrElse("") def getSchema() : SchemaDocWrapper = relation.schema.map(SchemaDocWrapper).orNull + def getInputs() : java.util.List[String] = relation.inputs.map(_.toString).asJava + def getResources() : java.util.List[ResourceIdentifierWrapper] = relation.provides.map(ResourceIdentifierWrapper).asJava } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala index f2c1d0a18..847d85e02 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/velocity.scala @@ -171,3 +171,11 @@ final case class MeasureResultWrapper(result:MeasureResult) extends ResultWrappe final case class AssertionTestResultWrapper(result:AssertionTestResult) extends ResultWrapper(result) { } + + +final case class ResourceIdentifierWrapper(resource:ResourceIdentifier) { + override def toString: String = resource.category + ":" + resource.name + + def getCategory() : String = resource.category + def getName() : String = resource.name +} diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl index cce8826ca..80ab3c528 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -25,6 +25,14 @@ Mapping '${mapping}' (${mapping.reference}) #foreach($relation in ${project.relations}) Relation '${relation}' (${relation.reference}) Description: ${relation.description} + Resources: + #foreach($resource in ${relation.resources}) + - ${resource.category} : ${resource.name} + #end + Inputs: + #foreach($input in ${relation.inputs}) + - ${input} + #end Schema: #foreach($column in ${relation.schema.columns}) ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala index b388f6c98..0ac1e007d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala @@ -37,6 +37,7 @@ class RelationDocSpec extends Spec[RelationDoc] { context.evaluate(description), None, Seq(), + Seq(), Map() ) val ref = doc.reference From 93d83472050fc96d166c16438716cbffa910f108 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 11:28:44 +0100 Subject: [PATCH 20/95] Implement unittests for ColumnTests --- CHANGELOG.md | 5 +- .../flowman/documentation/ColumnTest.scala | 54 ++++++++++++--- .../flowman/documentation/TestResult.scala | 13 +++- .../documentation/ColumnTestTest.scala | 66 +++++++++++++++++++ pom.xml | 8 +-- 5 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f2eccae..546e67237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # Version 0.22.0 * Add new 'sqlserver' relation - +* Work on new documentation subsystem +* Update to Spark 3.2.1 # Version 0.21.1 - 2022-01-28 -* flowexec now returns different exit codes depending on the processing result +* `flowexec` now returns different exit codes depending on the processing result # Version 0.21.0 - 2022-01-26 diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index cde1b7869..3208215bd 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -16,6 +16,9 @@ package com.dimajix.flowman.documentation +import scala.util.Success +import scala.util.Try + import org.apache.spark.sql.DataFrame import com.dimajix.flowman.execution.Execution @@ -24,7 +27,14 @@ import com.dimajix.flowman.spi.ColumnTestExecutor final case class ColumnTestReference( override val parent:Option[Reference] -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/test" + case None => "" + } + } +} abstract class ColumnTest extends Fragment with Product with Serializable { @@ -33,14 +43,14 @@ abstract class ColumnTest extends Fragment with Product with Serializable { override def reparent(parent: Reference): ColumnTest - override def parent: Some[Reference] + override def parent: Option[Reference] override def reference: ColumnTestReference = ColumnTestReference(parent) override def fragments: Seq[Fragment] = result.toSeq } case class NotNullColumnTest( - parent:Some[Reference], + parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { @@ -52,30 +62,39 @@ case class NotNullColumnTest( } case class UniqueColumnTest( - parent:Some[Reference], + parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) + override def reparent(parent: Reference): UniqueColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } } case class RangeColumnTest( - parent:Some[Reference], + parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) + override def reparent(parent: Reference): RangeColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } } case class ValuesColumnTest( - parent:Some[Reference], + parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ColumnTest = copy(parent=Some(parent)) + override def reparent(parent: Reference): ValuesColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } } //case class ForeignKeyColumnTest() extends ColumnTest @@ -83,5 +102,20 @@ case class ValuesColumnTest( class DefaultColumnTestExecutor extends ColumnTestExecutor { - override def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = ??? + override def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = { + test match { + case _: NotNullColumnTest => + val result = df.groupBy(df(column).isNotNull).count().collect() + val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) + val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) + val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + Some(TestResult(Some(test.reference), status, None, None)) + case _: UniqueColumnTest => + val agg = df.filter(df(column).isNotNull).groupBy(df(column)).count() + val result = agg.filter(agg(agg.columns(1)) > 1).orderBy(agg(agg.columns(1)).desc).limit(6).collect() + val status = if (result.isEmpty) TestStatus.SUCCESS else TestStatus.FAILED + Some(TestResult(Some(test.reference), status, None, None)) + case _ => None + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala index b859d6565..8057d959c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala @@ -27,14 +27,21 @@ object TestStatus { final case class TestResultReference( parent:Option[Reference] -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/result" + case None => "" + } + } +} final case class TestResult( parent:Some[Reference], status:TestStatus, - description:Option[String], - details:Option[Fragment] + description:Option[String] = None, + details:Option[Fragment] = None ) extends Fragment { override def reference: TestResultReference = TestResultReference(parent) override def fragments: Seq[Fragment] = details.toSeq diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala new file mode 100644 index 000000000..931487146 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Session +import com.dimajix.spark.testing.LocalSparkSession + + +class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { + "A NotNullColumnTest" should "be executable" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq((Some(1),2), (None,3))) + + val test = NotNullColumnTest(None) + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_3", test)) + } + + "A UniqueColumnTest" should "be executable" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,3), + (None,3,4), + (None,3,5), + )) + + val test = UniqueColumnTest(None) + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result3 = testExecutor.execute(execution, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + } +} diff --git a/pom.xml b/pom.xml index e6add6094..a14aea333 100644 --- a/pom.xml +++ b/pom.xml @@ -373,15 +373,15 @@ 2.12.15 2.12 - 3.2.5 + 3.2.9 3.2 1.2.0 1.1.2 - 3.2.0 + 3.2.1 3.2 1.1.8.4 4.1.68.Final - 4.8-1 + 4.8 1.27 2.12.3 2.12 @@ -396,7 +396,7 @@ 1.7.30 4.5.13 4.4.14 - 4.1.1 + 4.2.0 2.10.10 3.2.2 1.21 From 1a51d85d67aa9bec4442e60bbf122009aefbf165 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 13:37:58 +0100 Subject: [PATCH 21/95] Fix build for Scala 2.11 --- .../dimajix/flowman/documentation/ColumnTest.scala | 11 ++++------- .../scala/com/dimajix/flowman/model/Reference.scala | 5 +++-- .../flowman/documentation/ColumnTestTest.scala | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 3208215bd..0096025d8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -16,9 +16,6 @@ package com.dimajix.flowman.documentation -import scala.util.Success -import scala.util.Try - import org.apache.spark.sql.DataFrame import com.dimajix.flowman.execution.Execution @@ -49,7 +46,7 @@ abstract class ColumnTest extends Fragment with Product with Serializable { } -case class NotNullColumnTest( +final case class NotNullColumnTest( parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None @@ -61,7 +58,7 @@ case class NotNullColumnTest( } } -case class UniqueColumnTest( +final case class UniqueColumnTest( parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None @@ -73,7 +70,7 @@ case class UniqueColumnTest( } } -case class RangeColumnTest( +final case class RangeColumnTest( parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None @@ -85,7 +82,7 @@ case class RangeColumnTest( } } -case class ValuesColumnTest( +final case class ValuesColumnTest( parent:Option[Reference], description: Option[String] = None, result:Option[TestResult] = None diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Reference.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Reference.scala index 5954784ee..e2ab090fb 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Reference.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Reference.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,13 @@ package com.dimajix.flowman.model import com.dimajix.flowman.execution.Context -abstract class Reference[T] { +sealed abstract class Reference[T] { val value:T def name:String def identifier:Identifier[T] } + object RelationReference { def apply(context:Context, prototype:Prototype[Relation]) : ValueRelationReference = ValueRelationReference(context, prototype) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala index 931487146..d691cd811 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -51,7 +51,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val df = spark.createDataFrame(Seq( (Some(1),2,3), (None,3,4), - (None,3,5), + (None,3,5) )) val test = UniqueColumnTest(None) From 1e636c826d5ce97e12689e1d9b214ace20a41063 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 13:55:53 +0100 Subject: [PATCH 22/95] Fix support for Spark 3.2.1 --- .../dimajix/flowman/catalog/HiveCatalog.scala | 18 ++---------------- .../org/apache/spark/sql/SparkShim.scala | 9 +++++++++ .../org/apache/spark/sql/SparkShim.scala | 11 ++++++++++- .../org/apache/spark/sql/SparkShim.scala | 12 +++++++++++- .../org/apache/spark/sql/SparkShim.scala | 11 ++++++++++- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala index e14f20220..d0a814540 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala @@ -32,18 +32,15 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition import org.apache.spark.sql.catalyst.catalog.CatalogTableType -import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand import org.apache.spark.sql.execution.command.AlterTableAddPartitionCommand import org.apache.spark.sql.execution.command.AlterTableChangeColumnCommand import org.apache.spark.sql.execution.command.AlterTableDropPartitionCommand import org.apache.spark.sql.execution.command.AlterTableSetLocationCommand -import org.apache.spark.sql.execution.command.AlterViewAsCommand import org.apache.spark.sql.execution.command.AnalyzePartitionCommand import org.apache.spark.sql.execution.command.AnalyzeTableCommand import org.apache.spark.sql.execution.command.CreateDatabaseCommand import org.apache.spark.sql.execution.command.CreateTableCommand -import org.apache.spark.sql.execution.command.CreateViewCommand import org.apache.spark.sql.execution.command.DropDatabaseCommand import org.apache.spark.sql.execution.command.DropTableCommand import org.apache.spark.sql.hive.HiveClientShim @@ -635,12 +632,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Creating Hive view $table") val plan = spark.sql(select).queryExecution.analyzed - //@annotation.nowarn // Disable warning about unreachable code for Spark 3.2 - val cmd = CreateViewCommand(table, Nil, None, Map(), Some(select), plan, false, false, SparkShim.PersistedView) match { - // Workaround for providing compatibility with Spark 3.2 and older versions - case ac:AnalysisOnlyCommand => ac.markAsAnalyzed().asInstanceOf[CreateViewCommand] - case c:CreateViewCommand => c - } + val cmd = SparkShim.createView(table, select, plan, false, false) cmd.run(spark) // Publish view to external catalog @@ -656,13 +648,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Redefining Hive view $table") val plan = spark.sql(select).queryExecution.analyzed - //@annotation.nowarn // Disable warning about unreachable code for Spark 3.2 - val cmd = AlterViewAsCommand(table, select, plan) match { - // Workaround for providing compatibility with Spark 3.2 and older versions - case ac:AnalysisOnlyCommand => ac.markAsAnalyzed().asInstanceOf[AlterViewAsCommand] - case c:AlterViewAsCommand => c - } - + val cmd = SparkShim.alterView(table, select, plan) cmd.run(spark) // Publish view to external catalog diff --git a/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala index 7d99bf199..48a677a07 100644 --- a/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala @@ -34,6 +34,8 @@ import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.columnar.InMemoryRelation +import org.apache.spark.sql.execution.command.AlterViewAsCommand +import org.apache.spark.sql.execution.command.CreateViewCommand import org.apache.spark.sql.execution.command.ViewType import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.datasources.FileFormat @@ -97,6 +99,13 @@ object SparkShim { def functionRegistry(spark:SparkSession) : FunctionRegistry = spark.sessionState.functionRegistry + def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { + CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView) + } + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + AlterViewAsCommand(table, select, plan) + } + val LocalTempView : ViewType = org.apache.spark.sql.execution.command.LocalTempView val GlobalTempView : ViewType = org.apache.spark.sql.execution.command.GlobalTempView val PersistedView : ViewType = org.apache.spark.sql.execution.command.PersistedView diff --git a/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala index 9d0946771..570ea2972 100644 --- a/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,8 @@ import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.columnar.InMemoryRelation +import org.apache.spark.sql.execution.command.AlterViewAsCommand +import org.apache.spark.sql.execution.command.CreateViewCommand import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 @@ -101,6 +103,13 @@ object SparkShim { def functionRegistry(spark:SparkSession) : FunctionRegistry = spark.sessionState.functionRegistry + def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { + CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView) + } + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + AlterViewAsCommand(table, select, plan) + } + val LocalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.LocalTempView val GlobalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.GlobalTempView val PersistedView : ViewType = org.apache.spark.sql.catalyst.analysis.PersistedView diff --git a/flowman-spark-extensions/src/main/spark-3.1/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-3.1/org/apache/spark/sql/SparkShim.scala index 35307a2ff..b51715c4b 100644 --- a/flowman-spark-extensions/src/main/spark-3.1/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-3.1/org/apache/spark/sql/SparkShim.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import java.util.TimeZone import org.apache.spark.SparkConf import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.config.ConfigEntry +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.catalyst.analysis.ViewType import org.apache.spark.sql.catalyst.expressions.Expression @@ -32,6 +33,8 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.util.IntervalUtils import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.execution.SQLExecution +import org.apache.spark.sql.execution.command.AlterViewAsCommand +import org.apache.spark.sql.execution.command.CreateViewCommand import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 @@ -101,6 +104,13 @@ object SparkShim { def functionRegistry(spark:SparkSession) : FunctionRegistry = spark.sessionState.functionRegistry + def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { + CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView) + } + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan) : AlterViewAsCommand = { + AlterViewAsCommand(table, select, plan) + } + val LocalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.LocalTempView val GlobalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.GlobalTempView val PersistedView : ViewType = org.apache.spark.sql.catalyst.analysis.PersistedView diff --git a/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala index 5c877f668..c01098ec6 100644 --- a/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,8 @@ import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.columnar.InMemoryRelation +import org.apache.spark.sql.execution.command.AlterViewAsCommand +import org.apache.spark.sql.execution.command.CreateViewCommand import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 @@ -109,6 +111,13 @@ object SparkShim { def functionRegistry(spark:SparkSession) : FunctionRegistry = spark.sessionState.functionRegistry + def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { + CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView, isAnalyzed=true) + } + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + AlterViewAsCommand(table, select, plan, isAnalyzed=true) + } + val LocalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.LocalTempView val GlobalTempView : ViewType = org.apache.spark.sql.catalyst.analysis.GlobalTempView val PersistedView : ViewType = org.apache.spark.sql.catalyst.analysis.PersistedView From 4e4507274e919c294e701e2cbc27a349dce64608 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 16:55:01 +0100 Subject: [PATCH 23/95] Fix build for Spark != 3.1.x --- .../src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala | 2 +- .../src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala | 2 +- .../src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala index 48a677a07..cba359fab 100644 --- a/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-2.4/org/apache/spark/sql/SparkShim.scala @@ -102,7 +102,7 @@ object SparkShim { def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView) } - def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan) : AlterViewAsCommand = { AlterViewAsCommand(table, select, plan) } diff --git a/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala index 570ea2972..ef2978cd6 100644 --- a/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-3.0/org/apache/spark/sql/SparkShim.scala @@ -106,7 +106,7 @@ object SparkShim { def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView) } - def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan) : AlterViewAsCommand = { AlterViewAsCommand(table, select, plan) } diff --git a/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala b/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala index c01098ec6..539d1dc6e 100644 --- a/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala +++ b/flowman-spark-extensions/src/main/spark-3.2/org/apache/spark/sql/SparkShim.scala @@ -114,7 +114,7 @@ object SparkShim { def createView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : CreateViewCommand = { CreateViewCommand(table, Nil, None, Map(), Some(select), plan, allowExisting, replace, SparkShim.PersistedView, isAnalyzed=true) } - def alterView(table:TableIdentifier, select:String, plan:LogicalPlan, allowExisting:Boolean, replace:Boolean) : AlterViewAsCommand = { + def alterView(table:TableIdentifier, select:String, plan:LogicalPlan) : AlterViewAsCommand = { AlterViewAsCommand(table, select, plan, isAnalyzed=true) } From 846a2367a4807ae486b311da6f350342fe51c05b Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 20:06:09 +0100 Subject: [PATCH 24/95] Add unittests for documentation framework --- .../flowman/documentation/ColumnDoc.scala | 14 +-- .../flowman/documentation/ColumnTest.scala | 28 ++++-- .../flowman/documentation/SchemaDoc.scala | 8 +- .../com/dimajix/flowman/types/Field.scala | 3 +- .../flowman/documentation/ColumnDocTest.scala | 82 +++++++++++++++++ .../documentation/ColumnTestTest.scala | 88 +++++++++++++++++++ .../spec/documentation/ColumnDocSpec.scala | 4 +- .../spec/documentation/ColumnTestSpec.scala | 17 +++- .../spec/documentation/ColumnTestTest.scala | 32 +++++++ .../relation/HiveUnionTableRelationTest.scala | 4 +- 10 files changed, 258 insertions(+), 22 deletions(-) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 050b07733..7b73c0cca 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -18,6 +18,7 @@ package com.dimajix.flowman.documentation import com.dimajix.common.MapIgnoreCase import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.NullType @@ -50,11 +51,9 @@ object ColumnDoc { } final case class ColumnDoc( parent:Option[Reference], - name:String, field:Field, - description:Option[String], - children:Seq[ColumnDoc], - tests:Seq[ColumnTest] + children:Seq[ColumnDoc] = Seq(), + tests:Seq[ColumnTest] = Seq() ) extends EntityDoc { override def reference: ColumnReference = ColumnReference(parent, name) override def fragments: Seq[Fragment] = children @@ -67,6 +66,8 @@ final case class ColumnDoc( ) } + def name : String = field.name + def description : Option[String] = field.description def nullable : Boolean = field.nullable def typeName : String = field.typeName def sqlType : String = field.sqlType @@ -81,7 +82,10 @@ final case class ColumnDoc( this.children ++ other.children val desc = other.description.orElse(description) val tsts = tests ++ other.tests - copy(description=desc, children=childs, tests=tsts) + val ftyp = if (field.ftype == NullType) other.field.ftype else field.ftype + val nll = if (field.ftype == NullType) other.field.nullable else field.nullable + val fld = field.copy(ftype=ftyp, nullable=nll, description=desc) + copy(field=fld, children=childs, tests=tsts) } /** diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 0096025d8..a1ed5e1ac 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -16,7 +16,9 @@ package com.dimajix.flowman.documentation +import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.lit import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.spi.ColumnTestExecutor @@ -73,6 +75,8 @@ final case class UniqueColumnTest( final case class RangeColumnTest( parent:Option[Reference], description: Option[String] = None, + lower:Any, + upper:Any, result:Option[TestResult] = None ) extends ColumnTest { override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) @@ -85,6 +89,7 @@ final case class RangeColumnTest( final case class ValuesColumnTest( parent:Option[Reference], description: Option[String] = None, + values: Seq[Any] = Seq(), result:Option[TestResult] = None ) extends ColumnTest { override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) @@ -102,17 +107,30 @@ class DefaultColumnTestExecutor extends ColumnTestExecutor { override def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = { test match { case _: NotNullColumnTest => - val result = df.groupBy(df(column).isNotNull).count().collect() - val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) - val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) - val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS - Some(TestResult(Some(test.reference), status, None, None)) + executePredicateTest(df, column, test, df(column).isNotNull) case _: UniqueColumnTest => val agg = df.filter(df(column).isNotNull).groupBy(df(column)).count() val result = agg.filter(agg(agg.columns(1)) > 1).orderBy(agg(agg.columns(1)).desc).limit(6).collect() val status = if (result.isEmpty) TestStatus.SUCCESS else TestStatus.FAILED Some(TestResult(Some(test.reference), status, None, None)) + case v: ValuesColumnTest => + val dt = df.schema(column).dataType + val values = v.values.map(v => lit(v).cast(dt)) + executePredicateTest(df.filter(df(column).isNotNull), column, test, df(column).isin(values:_*)) + case v: RangeColumnTest => + val dt = df.schema(column).dataType + val lower = lit(v.lower).cast(dt) + val upper = lit(v.upper).cast(dt) + executePredicateTest(df.filter(df(column).isNotNull), column, test, df(column).between(lower, upper)) case _ => None } } + + private def executePredicateTest(df: DataFrame, column:String, test:ColumnTest, predicate:Column) : Option[TestResult] = { + val result = df.groupBy(predicate).count().collect() + val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) + val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) + val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + Some(TestResult(Some(test.reference), status, None, None)) + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index b2ac387c5..e158dcf4d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -58,7 +58,7 @@ object SchemaDoc { } def genColumn(parent:Reference, field:Field) : ColumnDoc = { - val doc = ColumnDoc(Some(parent), field.name, field, field.description, Seq(), Seq()) + val doc = ColumnDoc(Some(parent), field, Seq(), Seq()) val children = genChildren(doc.reference, field.ftype) doc.copy(children = children) } @@ -70,9 +70,9 @@ object SchemaDoc { final case class SchemaDoc( parent:Option[Reference], - description:Option[String], - columns:Seq[ColumnDoc], - tests:Seq[SchemaTest] + description:Option[String] = None, + columns:Seq[ColumnDoc] = Seq(), + tests:Seq[SchemaTest] = Seq() ) extends EntityDoc { override def reference: SchemaReference = SchemaReference(parent) override def fragments: Seq[Fragment] = columns ++ tests diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/types/Field.scala b/flowman-core/src/main/scala/com/dimajix/flowman/types/Field.scala index 3a37ff78a..249290615 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/types/Field.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/types/Field.scala @@ -165,7 +165,8 @@ class Field { val format = this.format.map(", format=" + _).getOrElse("") val default = this.default.map(", default=" + _).getOrElse("") val size = this.size.map(", size=" + _).getOrElse("") - s"Field($name, $ftype, $nullable$format$size$default})" + val desc = this.description.map(", description=\"" + _ + "\"").getOrElse("") + s"Field($name, $ftype, $nullable$format$size$default$desc))" } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala new file mode 100644 index 000000000..05e4299d9 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.types.DoubleType +import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.NullType +import com.dimajix.flowman.types.StringType + + +class ColumnDocTest extends AnyFlatSpec with Matchers { + "A ColumnDoc" should "support merge" in { + val doc1 = ColumnDoc( + None, + Field("col1", NullType, description = Some("Some desc 1")), + children = Seq( + ColumnDoc(None, Field("child1", StringType, description = Some("Some child desc 1"))), + ColumnDoc(None, Field("child2", StringType, description = Some("Some child desc 1"))) + ) + ) + val doc2 = ColumnDoc( + None, + Field("col2", DoubleType, description = Some("Some desc 2")), + children = Seq( + ColumnDoc(None, Field("child2", NullType, description = Some("Some override child desc 1"))), + ColumnDoc(None, Field("child3", NullType, description = Some("Some override child desc 1"))) + ) + ) + + val result = doc1.merge(doc2) + + result should be (ColumnDoc( + None, + Field("col1", DoubleType, description = Some("Some desc 2")), + children = Seq( + ColumnDoc(None, Field("child1", StringType, description = Some("Some child desc 1"))), + ColumnDoc(None, Field("child2", StringType, description = Some("Some override child desc 1"))), + ColumnDoc(None, Field("child3", NullType, description = Some("Some override child desc 1"))) + ) + )) + } + + it should "support reparent" in { + val doc1 = ColumnDoc( + None, + Field("col1", NullType, description = Some("Some desc 1")), + children = Seq( + ColumnDoc(None, Field("child1", StringType, description = Some("Some child desc 1"))), + ColumnDoc(None, Field("child2", StringType, description = Some("Some child desc 1"))) + ) + ) + val parent = SchemaDoc(None) + + val result = doc1.reparent(parent.reference) + + result should be (ColumnDoc( + Some(parent.reference), + Field("col1", NullType, description = Some("Some desc 1")), + children = Seq( + ColumnDoc(Some(ColumnReference(Some(parent.reference), "col1")), Field("child1", StringType, description = Some("Some child desc 1"))), + ColumnDoc(Some(ColumnReference(Some(parent.reference), "col1")), Field("child2", StringType, description = Some("Some child desc 1"))) + ) + )) + } +} diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala index d691cd811..ad4ea474c 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -63,4 +63,92 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) } + + "A ValuesColumnTest" should "be executable" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = ValuesColumnTest(None, values=Seq(1,2)) + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result3 = testExecutor.execute(execution, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + } + + it should "use correct data types" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = ValuesColumnTest(None, values=Seq(1,2)) + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result3 = testExecutor.execute(execution, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + } + + "A RangeColumnTest" should "be executable" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = RangeColumnTest(None, lower=1, upper=2) + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result3 = testExecutor.execute(execution, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + } + + it should "use correct data types" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = RangeColumnTest(None, lower="1.0", upper="2.2") + val result1 = testExecutor.execute(execution, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + val result2 = testExecutor.execute(execution, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + val result3 = testExecutor.execute(execution, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala index c13d0ebba..4a682527e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala @@ -34,9 +34,7 @@ class ColumnDocSpec { def instantiate(context: Context, parent:Reference): ColumnDoc = { val doc = ColumnDoc( Some(parent), - context.evaluate(name), - Field(name, NullType), - context.evaluate(description), + Field(context.evaluate(name), NullType, description=context.evaluate(description)), Seq(), Seq() ) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala index 1972456ac..9d5ef25ef 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.spec.documentation +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo @@ -62,8 +63,20 @@ class UniqueColumnTestSpec extends ColumnTestSpec { override def instantiate(context: Context, parent:ColumnReference): ColumnTest = UniqueColumnTest(Some(parent)) } class RangeColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = RangeColumnTest(Some(parent)) + @JsonProperty(value="lower", required=true) private var lower:String = "" + @JsonProperty(value="upper", required=true) private var upper:String = "" + + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = RangeColumnTest( + Some(parent), + lower=context.evaluate(lower), + upper=context.evaluate(upper) + ) } class ValuesColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = ValuesColumnTest(Some(parent)) + @JsonProperty(value="values", required=false) private var values:Seq[String] = Seq() + + override def instantiate(context: Context, parent:ColumnReference): ColumnTest = ValuesColumnTest( + Some(parent), + values=values.map(context.evaluate) + ) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala index fdb5e815c..604151b6b 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala @@ -41,4 +41,36 @@ class ColumnTestTest extends AnyFlatSpec with Matchers { Some(ColumnReference(None, "col0")) )) } + + "A RangeColumnTest" should "be deserializable" in { + val yaml = + """ + |kind: range + """.stripMargin + + val spec = ObjectMapper.parse[ColumnTestSpec](yaml) + spec shouldBe a[UniqueColumnTestSpec] + + val context = RootContext.builder().build() + val test = spec.instantiate(context, ColumnReference(None, "col0")) + test should be (UniqueColumnTest( + Some(ColumnReference(None, "col0")) + )) + } + + "A ValuesColumnTest" should "be deserializable" in { + val yaml = + """ + |kind: values + """.stripMargin + + val spec = ObjectMapper.parse[ColumnTestSpec](yaml) + spec shouldBe a[UniqueColumnTestSpec] + + val context = RootContext.builder().build() + val test = spec.instantiate(context, ColumnReference(None, "col0")) + test should be (UniqueColumnTest( + Some(ColumnReference(None, "col0")) + )) + } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala index f63854aa5..4ed8186c3 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala @@ -146,7 +146,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa table.identifier should be (TableIdentifier("lala_1", Some("default"))) table.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { - table.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(table.schema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", CharType(10)), @@ -154,7 +154,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa ))) } else { - table.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(table.schema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", StringType), From 559496c1766526c6717c39296cac786238676363 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 10 Feb 2022 21:11:06 +0100 Subject: [PATCH 25/95] Fix build issues in unittests --- docs/documenting/index.md | 2 +- docs/documenting/mappings.md | 51 ++++++++++++++ docs/documenting/relations.md | 40 +++++++++++ docs/spec/schema/embedded.md | 66 +++++++++++++++++++ .../flowman/documentation/MappingDoc.scala | 30 +++++++++ .../flowman/documentation/SchemaDoc.scala | 3 + .../spec/documentation/ColumnTestSpec.scala | 5 +- .../spec/documentation/MappingDocSpec.scala | 4 +- .../spec/documentation/ColumnTestTest.scala | 20 ++++-- .../spec/documentation/MappingDocTest.scala | 23 ++++++- .../spec/documentation/RelationDocTest.scala | 14 +++- 11 files changed, 245 insertions(+), 13 deletions(-) diff --git a/docs/documenting/index.md b/docs/documenting/index.md index 8ea4556e3..be38dc2b8 100644 --- a/docs/documenting/index.md +++ b/docs/documenting/index.md @@ -1,4 +1,4 @@ -# Documenting Projects and Models +# Documenting with Flowman Flowman supports to automatically generate a documentation of your project. The documentation can either include all major entities like mappings, relations and targets. Or you may want to focus only on some aspects like the relations diff --git a/docs/documenting/mappings.md b/docs/documenting/mappings.md index 2f577ed08..da75c31b5 100644 --- a/docs/documenting/mappings.md +++ b/docs/documenting/mappings.md @@ -1 +1,52 @@ # Documenting Mappings + +As with other entities, Flowman tries to automatically infer a meaningful documentation for mappings, especially +for the schema of all mappings outputs. But of course this is not always possible especially when mappings perform +complex transformations such that a single output column depends on multiple input columns. Probably the most +complex example is the [SQL](../spec/mapping/sql.md) mapping which allows to implement most complex transformations. + +In order to mitigate this issue, you can explicitly provide additional documentation for mappings via the +`documentation` tag, which is supported by all mappings. + +## Example + +```yaml +mappings: + # Extract multiple columns from the raw measurements data using SQL SUBSTR functions + measurements_extracted: + kind: select + input: measurements_raw + columns: + usaf: "SUBSTR(raw_data,5,6)" + wban: "SUBSTR(raw_data,11,5)" + date: "TO_DATE(SUBSTR(raw_data,16,8), 'yyyyMMdd')" + time: "SUBSTR(raw_data,24,4)" + air_temperature: "CAST(SUBSTR(raw_data,88,5) AS FLOAT)/10" + air_temperature_qual: "SUBSTR(raw_data,93,1)" + + documentation: + columns: + - name: usaf + description: "The USAF (US Air Force) id of the weather station" + - name: wban + description: "The WBAN id of the weather station" + - name: date + description: "The date when the measurement was made" + - name: time + description: "The time when the measurement was made" + - name: report_type + description: "The report type of the measurement" + description: "The quality indicator of the wind speed. 1 means trustworthy quality." + - name: air_temperature + description: "The air temperature in degree Celsius" + - name: air_temperature_qual + description: "The quality indicator of the air temperature. 1 means trustworthy quality." +``` + +## Fields + +* `description` **(optional)** *(type: string)*: A description of the mapping + +* `columns` **(optional)** *(type: schema)*: A documentation of the output schema. Note that Flowman will inspect +the schema of the mapping itself and only overlay the provided documentation. Only fields found in the original +output schema will be documented, so you cannot add fields to the documentation which actually do not exist. diff --git a/docs/documenting/relations.md b/docs/documenting/relations.md index b57bbf89a..a4a9aa0e8 100644 --- a/docs/documenting/relations.md +++ b/docs/documenting/relations.md @@ -1 +1,41 @@ # Documenting Relations + +As with other entities, Flowman tries to automatically infer a meaningful documentation for mappings, especially +for the schema of a relation. In order to do so, Flowman will query the original data source and look up any +metadata (for example Flowman will pick up column descriptions in the Hive Metastore). + +In order to provide additiona information, you can explicitly provide additional documentation for mappings via the +`documentation` tag, which is supported by all mappings. + +## Example + +```yaml +relations: + aggregates: + kind: file + format: parquet + location: "$basedir/aggregates/" + partitions: + - name: year + type: integer + granularity: 1 + documentation: + description: "The table contains all aggregated measurements" + columns: + - name: country + description: "Country of the weather station" + - name: min_temperature + description: "Minimum air temperature per year in degrees Celsius" + - name: max_temperature + description: "Maximum air temperature per year in degrees Celsius" + - name: avg_temperature + description: "Average air temperature per year in degrees Celsius" +``` + +## Fields + +* `description` **(optional)** *(type: string)*: A description of the mapping + +* `columns` **(optional)** *(type: schema)*: A documentation of the output schema. Note that Flowman will inspect + the schema of the mapping itself and only overlay the provided documentation. Only fields found in the original + output schema will be documented, so you cannot add fields to the documentation which actually do not exist. diff --git a/docs/spec/schema/embedded.md b/docs/spec/schema/embedded.md index ea9a5a23e..474e4821f 100644 --- a/docs/spec/schema/embedded.md +++ b/docs/spec/schema/embedded.md @@ -28,4 +28,70 @@ relations: ## Fields * `kind` **(mandatory)** *(type: string)*: `embedded` + * `fields` **(mandatory)** *(type: list:field)*: Contains all fields + + +## Field properties + +* `name` **(mandatory)** *(type: string)*: specifies the name of the column +* `type` **(mandatory)** *(type: data type)*: specifies the data type of the column +* `nullable` **(optional)** *(type: boolean)* *(default: true)* +* `description` **(optional)** *(type: string)* +* `default` **(optional)** *(type: string)* Specifies a default value +* `format` **(optional)** *(type: string)* Some relations or file formats may support different formats for example +for storing dates + + +## Data Types + +The following simple data types are supported by Flowman + +* `string`, `text` - text and strings of arbitrary length +* `binary` - binary data of arbitrary length +* `tinyint`, `byte` - 8 bit signed numbers +* `smallint`, `short` - 16 bit signed numbers +* `int`, `integer` - 32 bit signed numbers +* `bigint`, `long` - 64 bit signed numbers +* `boolean` - true or false +* `float` - 32 bit floating point number +* `double` - 64 bit floating point number +* `decimal(a,b)` +* `varchar(n)` - text with up to `n`characters. Note that this data type is only supported for specifying input or +output data types. Internally Spark and therefore Flowman convert these columns to a `string` column of arbitrary length. +* `char(n)` - text with exactly `n`characters. Note that this data type is only supported for specifying input or + output data types. Internally Spark and therefore Flowman convert these columns to a `string` column of arbitrary length. +* `date` - date type +* `timestamp` - timestamp type (date and time) +* `duration` - duration type + +In addition to those simple data types the following complex types are supported: + +* `struct` for creating nested data types +```yaml +name: some_struct +type: + kind: struct + fields: + - name: some_field + type: int + - name: some_other_field + type: string +``` + +* `map` +```yaml +name: keyValue +type: + kind: map + keyType: string + valueType: int +``` + +* `array` for storing arrays of sub elements + ```yaml +name: names +type: + kind: array + elementType: string +``` diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index 1224d8d84..609365230 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -49,6 +49,24 @@ final case class MappingOutputDoc( ) } + /** + * Returns the name of the project of the mapping of this output + * @return + */ + def project : Option[String] = identifier.project + + /** + * Returns the mapping identifier of this output + * @return + */ + def mapping : MappingIdentifier = identifier.mapping + + /** + * Returns the name of the output + * @return + */ + def name : String = identifier.output + def merge(other:MappingOutputDoc) : MappingOutputDoc = { val id = if (identifier.mapping.isEmpty) other.identifier else identifier val desc = other.description.orElse(this.description) @@ -91,6 +109,18 @@ final case class MappingDoc( ) } + /** + * Returns the name of the project of this mapping + * @return + */ + def project : Option[String] = identifier.project + + /** + * Returns the name of this mapping + * @return + */ + def name : String = identifier.name + def merge(other:Option[MappingDoc]) : MappingDoc = other.map(merge).getOrElse(this) def merge(other:MappingDoc) : MappingDoc = { val id = if (identifier.isEmpty) other.identifier else identifier diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index e158dcf4d..614078a10 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.documentation +import scala.annotation.tailrec + import com.dimajix.common.MapIgnoreCase import com.dimajix.flowman.types.ArrayType import com.dimajix.flowman.types.Field @@ -44,6 +46,7 @@ object SchemaDoc { def genColumns(parent:Reference, fields:Seq[Field]) : Seq[ColumnDoc] = { fields.map(f => genColumn(parent, f)) } + @tailrec def genChildren(parent:Reference, ftype:FieldType) : Seq[ColumnDoc] = { ftype match { case s:StructType => diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala index 9d5ef25ef..e1db805c2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala @@ -68,8 +68,9 @@ class RangeColumnTestSpec extends ColumnTestSpec { override def instantiate(context: Context, parent:ColumnReference): ColumnTest = RangeColumnTest( Some(parent), - lower=context.evaluate(lower), - upper=context.evaluate(upper) + None, + context.evaluate(lower), + context.evaluate(upper) ) } class ValuesColumnTestSpec extends ColumnTestSpec { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala index 868453461..092fc1715 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala @@ -103,12 +103,12 @@ class MappingDocSpec extends Spec[MappingDoc] { ) val ref3 = schema.reference val cols = columns.map(_.instantiate(context, ref3)) - val tests = this.tests.map(_.instantiate(context, ref3)) + val tsts = tests.map(_.instantiate(context, ref3)) Some( output.copy( schema = Some(schema.copy( columns=cols, - tests=tests + tests=tsts )) ) ) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala index 604151b6b..b10d30bf4 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala @@ -20,7 +20,9 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.documentation.RangeColumnTest import com.dimajix.flowman.documentation.UniqueColumnTest +import com.dimajix.flowman.documentation.ValuesColumnTest import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper @@ -46,15 +48,19 @@ class ColumnTestTest extends AnyFlatSpec with Matchers { val yaml = """ |kind: range + |lower: 7 + |upper: 23 """.stripMargin val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[UniqueColumnTestSpec] + spec shouldBe a[RangeColumnTestSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (UniqueColumnTest( - Some(ColumnReference(None, "col0")) + test should be (RangeColumnTest( + Some(ColumnReference(None, "col0")), + lower="7", + upper="23" )) } @@ -62,15 +68,17 @@ class ColumnTestTest extends AnyFlatSpec with Matchers { val yaml = """ |kind: values + |values: ['a', 12, null] """.stripMargin val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[UniqueColumnTestSpec] + spec shouldBe a[ValuesColumnTestSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (UniqueColumnTest( - Some(ColumnReference(None, "col0")) + test should be (ValuesColumnTest( + Some(ColumnReference(None, "col0")), + values = Seq("a", "12", null) )) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala index 10d134fc9..bcad1723a 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala @@ -20,6 +20,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.documentation.NotNullColumnTest import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper @@ -36,6 +37,7 @@ class MappingDocTest extends AnyFlatSpec with Matchers { | - kind: notNull |outputs: | other: + | description: "This is an additional output" | columns: | - name: col_x | description: "Column of other output" @@ -46,6 +48,25 @@ class MappingDocTest extends AnyFlatSpec with Matchers { val context = RootContext.builder().build() val mapping = spec.instantiate(context) - println(mapping.toString) + mapping.description should be (Some("This is a mapping")) + + val main = mapping.outputs.find(_.name == "main").get + main.description should be (None) + val mainSchema = main.schema.get + mainSchema.columns.size should be (1) + mainSchema.columns(0).name should be ("col_a") + mainSchema.columns(0).description should be (Some("This is column a")) + mainSchema.columns(0).tests.size should be (1) + mainSchema.columns(0).tests(0) shouldBe a[NotNullColumnTest] + mainSchema.tests.size should be (0) + + val other = mapping.outputs.find(_.name == "other").get + other.description should be (Some("This is an additional output")) + val otherSchema = other.schema.get + otherSchema.columns.size should be (1) + otherSchema.columns(0).name should be ("col_x") + otherSchema.columns(0).description should be (Some("Column of other output")) + otherSchema.columns(0).tests.size should be (0) + otherSchema.tests.size should be (0) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala index 7e7c2382d..5da396742 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala @@ -19,6 +19,7 @@ package com.dimajix.flowman.spec.documentation import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.documentation.NotNullColumnTest import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper @@ -44,6 +45,17 @@ class RelationDocTest extends AnyFlatSpec with Matchers { val context = RootContext.builder().build() val relation = spec.instantiate(context) - println(relation.toString) + relation.description should be (Some("This is a mapping")) + + val mainSchema = relation.schema.get + mainSchema.columns.size should be (2) + mainSchema.columns(0).name should be ("col_a") + mainSchema.columns(0).description should be (Some("This is column a")) + mainSchema.columns(0).tests.size should be (1) + mainSchema.columns(0).tests(0) shouldBe a[NotNullColumnTest] + mainSchema.columns(1).name should be ("col_x") + mainSchema.columns(1).description should be (Some("Column of other output")) + mainSchema.columns(1).tests.size should be (0) + mainSchema.tests.size should be (0) } } From bbed7b6b72a2d3a97636d1d4490cdcddb956110a Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 11 Feb 2022 10:05:23 +0100 Subject: [PATCH 26/95] Fix build issues for Spark 3.2 --- CHANGELOG.md | 2 +- docker/pom.xml | 18 +++++- flowman-client/pom.xml | 2 +- flowman-common/pom.xml | 2 +- flowman-core/pom.xml | 2 +- .../flowman/documentation/ColumnDoc.scala | 19 ++++-- .../flowman/documentation/MappingDoc.scala | 29 ++++++++- .../flowman/documentation/RelationDoc.scala | 13 ++++ .../flowman/documentation/SchemaDoc.scala | 17 +++++ .../flowman/documentation/TargetDoc.scala | 13 ++++ flowman-dist/pom.xml | 2 +- flowman-dsl/pom.xml | 2 +- flowman-hub/pom.xml | 2 +- flowman-parent/pom.xml | 2 +- flowman-plugins/aws/pom.xml | 2 +- flowman-plugins/azure/pom.xml | 2 +- flowman-plugins/delta/pom.xml | 4 +- flowman-plugins/impala/pom.xml | 2 +- flowman-plugins/json/pom.xml | 2 +- flowman-plugins/kafka/pom.xml | 2 +- flowman-plugins/mariadb/pom.xml | 2 +- flowman-plugins/mssqlserver/pom.xml | 2 +- flowman-plugins/mysql/pom.xml | 2 +- flowman-plugins/openapi/pom.xml | 2 +- flowman-plugins/swagger/pom.xml | 2 +- flowman-scalatest-compat/pom.xml | 2 +- flowman-server-ui/pom.xml | 2 +- flowman-server/pom.xml | 2 +- flowman-spark-extensions/pom.xml | 2 +- flowman-spark-testing/pom.xml | 2 +- flowman-spec/pom.xml | 2 +- .../spec/relation/HiveTableRelationTest.scala | 8 +-- .../relation/HiveUnionTableRelationTest.scala | 8 +-- flowman-studio-ui/pom.xml | 2 +- flowman-studio/pom.xml | 2 +- flowman-testing/pom.xml | 2 +- flowman-tools/pom.xml | 2 +- pom.xml | 62 +++++++++---------- 38 files changed, 171 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546e67237..6096984ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ * Add new 'sqlserver' relation * Work on new documentation subsystem -* Update to Spark 3.2.1 +* Change default build to Spark 3.2.1 and Hadoop 3.3.1 # Version 0.21.1 - 2022-01-28 diff --git a/docker/pom.xml b/docker/pom.xml index 058a705eb..46cd47e19 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -10,10 +10,14 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml + + ${hadoop-api.version} + + CDH-6.3 @@ -27,6 +31,16 @@ true + + spark-3.2 + + true + + + + 3.2 + + @@ -93,7 +107,7 @@ false ${spark.version} - ${hadoop-api.version} + ${spark-hadoop-archive.version} flowman-dist-${flowman.dist.label}-bin.tar.gz ${env.http_proxy} ${env.https_proxy} diff --git a/flowman-client/pom.xml b/flowman-client/pom.xml index cf3d9614e..bb9c286ab 100644 --- a/flowman-client/pom.xml +++ b/flowman-client/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-common/pom.xml b/flowman-common/pom.xml index cd1112f72..3df2b462f 100644 --- a/flowman-common/pom.xml +++ b/flowman-common/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-core/pom.xml b/flowman-core/pom.xml index 21f606b68..f2efb9630 100644 --- a/flowman-core/pom.xml +++ b/flowman-core/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 7b73c0cca..a2128b126 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -41,10 +41,7 @@ object ColumnDoc { val thisColsByName = MapIgnoreCase(thisCols.map(c => c.name -> c)) val otherColsByName = MapIgnoreCase(otherCols.map(c => c.name -> c)) val mergedColumns = thisCols.map { column => - otherColsByName.get(column.name) match { - case Some(other) => column.merge(other) - case None => column - } + column.merge(otherColsByName.get(column.name)) } mergedColumns ++ otherCols.filter(c => !thisColsByName.contains(c.name)) } @@ -74,6 +71,20 @@ final case class ColumnDoc( def sparkType : String = field.sparkType.sql def catalogType : String = field.catalogType.sql + /** + * Merge this schema documentation with another column documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ + def merge(other:Option[ColumnDoc]) : ColumnDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another column documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:ColumnDoc) : ColumnDoc = { val childs = if (this.children.nonEmpty && other.children.nonEmpty) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index 609365230..48b733965 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -67,6 +67,20 @@ final case class MappingOutputDoc( */ def name : String = identifier.output + /** + * Merge this schema documentation with another mapping documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ + def merge(other:Option[MappingOutputDoc]) : MappingOutputDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another mapping documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:MappingOutputDoc) : MappingOutputDoc = { val id = if (identifier.mapping.isEmpty) other.identifier else identifier val desc = other.description.orElse(this.description) @@ -121,13 +135,26 @@ final case class MappingDoc( */ def name : String = identifier.name + /** + * Merge this schema documentation with another mapping documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:Option[MappingDoc]) : MappingDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another mapping documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:MappingDoc) : MappingDoc = { val id = if (identifier.isEmpty) other.identifier else identifier val desc = other.description.orElse(this.description) val in = inputs.toSet ++ other.inputs.toSet val out = outputs.map { out => - other.outputs.find(_.identifier.output == out.identifier.output).map(out.merge).getOrElse(out) + out.merge(other.outputs.find(_.identifier.output == out.identifier.output)) } ++ other.outputs.filter(out => !outputs.exists(_.identifier.output == out.identifier.output)) val result = copy(identifier=id, description=desc, inputs=in.toSeq, outputs=out) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index 2adf65869..e2008c950 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -53,7 +53,20 @@ final case class RelationDoc( ) } + /** + * Merge this schema documentation with another relation documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:Option[RelationDoc]) : RelationDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another relation documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:RelationDoc) : RelationDoc = { val id = if (identifier.isEmpty) other.identifier else identifier val desc = other.description.orElse(this.description) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index 614078a10..5fbf5d2d5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -88,8 +88,25 @@ final case class SchemaDoc( ) } + /** + * Convert this schema documentation to a Flowman struct + */ def toStruct : StructType = StructType(columns.map(_.field)) + + /** + * Merge this schema documentation with another schema documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:Option[SchemaDoc]) : SchemaDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another schema documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:SchemaDoc) : SchemaDoc = { val desc = other.description.orElse(this.description) val tsts = tests ++ other.tests diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala index b61b6fe3f..ff986ff34 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -80,7 +80,20 @@ final case class TargetDoc( ) } + /** + * Merge this schema documentation with another target documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:Option[TargetDoc]) : TargetDoc = other.map(merge).getOrElse(this) + + /** + * Merge this schema documentation with another target documentation. Note that while documentation attributes + * of [[other]] have a higher priority than those of the instance itself, the parent of itself has higher priority + * than the one of [[other]]. This allows for a simply information overlay mechanism. + * @param other + */ def merge(other:TargetDoc) : TargetDoc = { val id = if (identifier.isEmpty) other.identifier else identifier val desc = other.description.orElse(this.description) diff --git a/flowman-dist/pom.xml b/flowman-dist/pom.xml index 358d4dae1..1acbc0c21 100644 --- a/flowman-dist/pom.xml +++ b/flowman-dist/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-dsl/pom.xml b/flowman-dsl/pom.xml index 78bd6f133..abf32ea0e 100644 --- a/flowman-dsl/pom.xml +++ b/flowman-dsl/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-hub/pom.xml b/flowman-hub/pom.xml index ffef1a3f3..103d9cb05 100644 --- a/flowman-hub/pom.xml +++ b/flowman-hub/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-parent/pom.xml b/flowman-parent/pom.xml index 0abd89f7d..0ffcc7a6f 100644 --- a/flowman-parent/pom.xml +++ b/flowman-parent/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-plugins/aws/pom.xml b/flowman-plugins/aws/pom.xml index 80306b290..f71183d03 100644 --- a/flowman-plugins/aws/pom.xml +++ b/flowman-plugins/aws/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/azure/pom.xml b/flowman-plugins/azure/pom.xml index a0dac5f24..a94327f35 100644 --- a/flowman-plugins/azure/pom.xml +++ b/flowman-plugins/azure/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/delta/pom.xml b/flowman-plugins/delta/pom.xml index 7cf72bfd8..9cac2d812 100644 --- a/flowman-plugins/delta/pom.xml +++ b/flowman-plugins/delta/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml @@ -17,7 +17,7 @@ flowman-delta ${project.version} ${project.build.finalName}.jar - 1.0.0 + 1.1.0 diff --git a/flowman-plugins/impala/pom.xml b/flowman-plugins/impala/pom.xml index 420ae0bfc..f14bc30ab 100644 --- a/flowman-plugins/impala/pom.xml +++ b/flowman-plugins/impala/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/json/pom.xml b/flowman-plugins/json/pom.xml index 874b9fb9d..2aa53666e 100644 --- a/flowman-plugins/json/pom.xml +++ b/flowman-plugins/json/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/kafka/pom.xml b/flowman-plugins/kafka/pom.xml index 3447e8e3b..a1eb26ef5 100644 --- a/flowman-plugins/kafka/pom.xml +++ b/flowman-plugins/kafka/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mariadb/pom.xml b/flowman-plugins/mariadb/pom.xml index 526b59d44..541972d99 100644 --- a/flowman-plugins/mariadb/pom.xml +++ b/flowman-plugins/mariadb/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index ec506414c..b57209144 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/mysql/pom.xml b/flowman-plugins/mysql/pom.xml index 791741adc..d790b4891 100644 --- a/flowman-plugins/mysql/pom.xml +++ b/flowman-plugins/mysql/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/openapi/pom.xml b/flowman-plugins/openapi/pom.xml index 2df8fdb62..8ced7bc7e 100644 --- a/flowman-plugins/openapi/pom.xml +++ b/flowman-plugins/openapi/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-plugins/swagger/pom.xml b/flowman-plugins/swagger/pom.xml index 7b4b2323c..4a8404bd8 100644 --- a/flowman-plugins/swagger/pom.xml +++ b/flowman-plugins/swagger/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../../pom.xml diff --git a/flowman-scalatest-compat/pom.xml b/flowman-scalatest-compat/pom.xml index a660600fe..549712291 100644 --- a/flowman-scalatest-compat/pom.xml +++ b/flowman-scalatest-compat/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-server-ui/pom.xml b/flowman-server-ui/pom.xml index e03cbd525..02a11f214 100644 --- a/flowman-server-ui/pom.xml +++ b/flowman-server-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-server/pom.xml b/flowman-server/pom.xml index 8bcaf18bb..a481296c4 100644 --- a/flowman-server/pom.xml +++ b/flowman-server/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-spark-extensions/pom.xml b/flowman-spark-extensions/pom.xml index 6d606958e..3ae14d0c7 100644 --- a/flowman-spark-extensions/pom.xml +++ b/flowman-spark-extensions/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-spark-testing/pom.xml b/flowman-spark-testing/pom.xml index 1a64bb373..592f6c8df 100644 --- a/flowman-spark-testing/pom.xml +++ b/flowman-spark-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-spec/pom.xml b/flowman-spec/pom.xml index 868bbe48a..dd7a83ff7 100644 --- a/flowman-spec/pom.xml +++ b/flowman-spec/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala index be2db60d6..3c57a3547 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala @@ -1547,12 +1547,12 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes table_1.identifier should be (TableIdentifier("some_table", Some("default"))) table_1.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { - table_1.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_1.schema) should be(StructType(Seq( StructField("f1", VarcharType(4)), StructField("f2", CharType(4)), StructField("f3", StringType) ))) - table_1.dataSchema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_1.dataSchema) should be(StructType(Seq( StructField("f1", VarcharType(4)), StructField("f2", CharType(4)), StructField("f3", StringType) @@ -1694,13 +1694,13 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes table_2.identifier should be (TableIdentifier("lala", Some("default"))) table_2.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { - table_2.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_2.schema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", VarcharType(10)), StructField("partition_col", StringType, nullable = false) ))) - table_2.dataSchema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_2.dataSchema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", VarcharType(10)) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala index 4ed8186c3..3a0934537 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala @@ -154,7 +154,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa ))) } else { - SchemaUtils.dropMetadata(table.schema) should be(StructType(Seq( + table.schema should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", StringType), @@ -757,7 +757,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa view_2.identifier should be (TableIdentifier("lala", Some("default"))) view_2.tableType should be (CatalogTableType.VIEW) if (hiveVarcharSupported) { - view_2.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(view_2.schema) should be(StructType(Seq( StructField("str_col", StringType), StructField("char_col", CharType(10)), StructField("int_col", IntegerType), @@ -780,13 +780,13 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa table_2.identifier should be (TableIdentifier("lala_1", Some("default"))) table_2.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { - table_2.schema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_2.schema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", CharType(10)), StructField("partition_col", StringType, nullable = false) ))) - table_2.dataSchema should be(StructType(Seq( + SchemaUtils.dropMetadata(table_2.dataSchema) should be(StructType(Seq( StructField("str_col", StringType), StructField("int_col", IntegerType), StructField("char_col", CharType(10)) diff --git a/flowman-studio-ui/pom.xml b/flowman-studio-ui/pom.xml index dad5d6711..77bd98025 100644 --- a/flowman-studio-ui/pom.xml +++ b/flowman-studio-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-studio/pom.xml b/flowman-studio/pom.xml index a79414a6b..ce7dfd0e2 100644 --- a/flowman-studio/pom.xml +++ b/flowman-studio/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-testing/pom.xml b/flowman-testing/pom.xml index 5e89263dc..7be01aff8 100644 --- a/flowman-testing/pom.xml +++ b/flowman-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/flowman-tools/pom.xml b/flowman-tools/pom.xml index 7e3bdeadf..59e34109e 100644 --- a/flowman-tools/pom.xml +++ b/flowman-tools/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index a14aea333..c5b3ac91c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.dimajix.flowman flowman-root - 0.21.2-SNAPSHOT + 0.22.0-SNAPSHOT pom Flowman root pom A Spark based ETL tool @@ -57,13 +57,12 @@ 2.4.0 1.9.13 4.0.0 - 10.12.1.1 + 2.1.1 2.1.210 1.2.17 1.1 5.1.0 2.5.0 - 2.10.5 2.2.4 3.5.2 4.0.4 @@ -83,44 +82,45 @@ 1.6 3.9.9.Final - - 3.2.0 - 3.2 - 2.3 + + 3.3.1 + 3.3 + 2.4.2 1.9.3 - 1.11 + 1.15 - - 2.12.10 + + 2.12.15 2.12 - 3.2.5 + 3.2.9 3.2 1.2.0 1.1.2 - 2.1.1 - 3.1.2 - 3.1 - 4.1.51.Final - 4.8-1 - 1.24 - 2.10.0 - 2.10 - 2.10.0 + 3.2.1 + 3.2 + 1.1.8.4 + 4.1.68.Final + 4.8 + 1.27 + 2.12.3 + 2.12 + 2.12.3 2.8 - 2.6.0 - 3.5.7 - 1.8.2 - 3.7.0-M5 + 2.8.0 + 10.14.2.0 + 3.6.2 + 1.10.2 + 3.7.0-M11 14.0.1 1.7.30 - 4.5.6 - 4.4.12 - 4.1.1 - 1.1.8.2 - 2.4 + 4.5.13 + 4.4.14 + 4.2.0 + 2.10.10 3.2.2 - 1.20 - 3.9 + 1.21 + 2.8.0 + 3.12.0 ${project.version} From 14420337e98f5f31eebc11508eb71511fadf1fc1 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 11 Feb 2022 16:08:52 +0100 Subject: [PATCH 27/95] Implement new 'drop' target --- CHANGELOG.md | 3 +- docs/spec/target/drop.md | 62 ++++++ docs/spec/target/relation.md | 4 +- docs/spec/target/truncate.md | 3 +- .../com/dimajix/flowman/graph/Linker.scala | 34 ++++ .../com/dimajix/flowman/graph/GraphTest.scala | 10 +- .../spec/target/DeltaVacuumTarget.scala | 2 +- .../spec/mapping/ReadStreamMapping.scala | 3 +- .../flowman/spec/target/DropTarget.scala | 181 ++++++++++++++++++ .../flowman/spec/target/MergeTarget.scala | 9 +- .../flowman/spec/target/RelationTarget.scala | 8 +- .../flowman/spec/target/StreamTarget.scala | 6 +- .../flowman/spec/target/TargetSpec.scala | 1 + .../flowman/spec/target/TruncateTarget.scala | 76 ++++++-- .../flowman/spec/target/DropTargetTest.scala | 115 +++++++++++ .../spec/target/TruncateTargetTest.scala | 59 ++++-- pom.xml | 2 +- 17 files changed, 519 insertions(+), 59 deletions(-) create mode 100644 docs/spec/target/drop.md create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DropTargetTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index 6096984ce..4a885212a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # Version 0.22.0 -* Add new 'sqlserver' relation +* Add new `sqlserver` relation * Work on new documentation subsystem * Change default build to Spark 3.2.1 and Hadoop 3.3.1 +* Add new `drop` target for removing tables # Version 0.21.1 - 2022-01-28 diff --git a/docs/spec/target/drop.md b/docs/spec/target/drop.md new file mode 100644 index 000000000..842e74f31 --- /dev/null +++ b/docs/spec/target/drop.md @@ -0,0 +1,62 @@ +# Drop Relation Target + +The `drop` target is used for dropping relations, i.e. dropping tables in relational database, +dropping tables in Hive or removing output directories. The target can be used for cleaning up tables which are not +used any more. + +## Example + +```yaml +targets: + drop_stations: + kind: drop + relation: stations + +relations: + stations: + kind: file + format: parquet + location: "$basedir/stations/" + schema: + kind: avro + file: "${project.basedir}/schema/stations.avsc" +``` + +Since Flowman 0.18.0, you can also directly specify the relation inside the target definition. This saves you +from having to create a separate relation definition in the `relations` section. This is only recommended, if you +do not access the target relation otherwise, such that a shared definition would not provide any benefit. +```yaml +targets: + drop_stations: + kind: drop + relation: + kind: file + name: stations-relation + format: parquet + location: "$basedir/stations/" + schema: + kind: avro + file: "${project.basedir}/schema/stations.avsc" +``` + +## Fields + +* `kind` **(mandatory)** *(type: string)*: `drop` + +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + +* `relation` **(mandatory)** *(type: string)*: +Specifies the name of the relation to write to + + +## Description + +The `drop` target will drop a relation and all its contents. It will be executed both during the `CREATE` phase and +during the `DESTROY` phase. + + +## Supported Phases +* `CREATE` - This will drop the target relation or migrate it to the newest schema (if possible). +* `VERIFY` - This will verify that the target relation does not exist any more +* `DESTROY` - This will also drop the relation itself and all its content. diff --git a/docs/spec/target/relation.md b/docs/spec/target/relation.md index 88d4b6736..99112c202 100644 --- a/docs/spec/target/relation.md +++ b/docs/spec/target/relation.md @@ -29,8 +29,8 @@ relations: ``` Since Flowman 0.18.0, you can also directly specify the relation inside the target definition. This saves you -from having to create a separate relation definition in the `relations` section. This is only recommeneded, if you -do not access the target relation otherwise, such that a shared definition would not provide any benefir. +from having to create a separate relation definition in the `relations` section. This is only recommended, if you +do not access the target relation otherwise, such that a shared definition would not provide any benefit. ```yaml targets: stations: diff --git a/docs/spec/target/truncate.md b/docs/spec/target/truncate.md index 0badc57b5..7faebb7f0 100644 --- a/docs/spec/target/truncate.md +++ b/docs/spec/target/truncate.md @@ -2,7 +2,7 @@ The `truncate` target is used to truncate a relation or individual partitions of a relation. Truncating means that the relation itself is not removed, but the contents are deleted (either all records or individual partitions). -Note that the `truncate` target is executed as part of the `BUILD`phase, which might be surprising. +Note that the `truncate` target is executed both as part of the `BUILD` and `TRUNCATE` phases, which might be surprising. ## Example @@ -34,3 +34,4 @@ targets: ## Supported Phases * `BUILD` - This will truncate the specified relation. * `VERIFY` - This will verify that the relation (and any specified partition) actually contains no data. +* `TRUNCATE` - This will truncate the specified relation. diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala index a12e8fe74..bc6fa7d93 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala @@ -17,25 +17,59 @@ package com.dimajix.flowman.graph import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.IdentifierRelationReference +import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.Reference +import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.ValueRelationReference import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.SingleValue final case class Linker private[graph](builder:GraphBuilder, context:Context, node:Node) { + def input(mapping: Mapping, output:String) : Linker = { + val in = builder.refMapping(mapping) + val edge = InputMapping(in, node, output) + link(edge) + } def input(mapping: MappingIdentifier, output:String) : Linker = { val instance = context.getMapping(mapping) val in = builder.refMapping(instance) val edge = InputMapping(in, node, output) link(edge) } + + def read(relation: Reference[Relation], partitions:Map[String,FieldValue]) : Linker = { + relation match { + case ref:ValueRelationReference => read(ref.value, partitions) + case ref:IdentifierRelationReference => read(ref.identifier, partitions) + } + } + def read(relation: Relation, partitions:Map[String,FieldValue]) : Linker = { + val in = builder.refRelation(relation) + val edge = ReadRelation(in, node, partitions) + link(edge) + } def read(relation: RelationIdentifier, partitions:Map[String,FieldValue]) : Linker = { val instance = context.getRelation(relation) val in = builder.refRelation(instance) val edge = ReadRelation(in, node, partitions) link(edge) } + + def write(relation: Reference[Relation], partitions:Map[String,SingleValue]) : Linker = { + relation match { + case ref:ValueRelationReference => write(ref.value, partitions) + case ref:IdentifierRelationReference => write(ref.identifier, partitions) + } + } + def write(relation: Relation, partition:Map[String,SingleValue]) : Linker = { + val out = builder.refRelation(relation) + val edge = WriteRelation(node, out, partition) + link(edge) + } def write(relation: RelationIdentifier, partition:Map[String,SingleValue]) : Linker = { val instance = context.getRelation(relation) val out = builder.refRelation(instance) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala index 272b60fd6..895d5ff70 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,11 +25,13 @@ import com.dimajix.flowman.execution.Session import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetIdentifier -import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.types.FieldValue +import com.dimajix.flowman.types.SingleValue class GraphTest extends AnyFlatSpec with Matchers with MockFactory { @@ -70,7 +72,7 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { (mappingTemplate2.instantiate _).expects(context).returns(mapping2) (mapping2.context _).expects().returns(context) (mapping2.name _).expects().atLeastOnce().returns("m2") - (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map()))) + (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map.empty[String,FieldValue]))) (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) (sourceRelation.context _).expects().returns(context) @@ -87,7 +89,7 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { (target.name _).expects().atLeastOnce().returns("t") (target.link _).expects(*,*).onCall((l:Linker, _:Phase) => Some(1).foreach { _ => l.input(MappingIdentifier("m1"), "main") - l.write(RelationIdentifier("tgt"), Map()) + l.write(RelationIdentifier("tgt"), Map.empty[String,SingleValue]) }) val graph = Graph.ofProject(session, project, Phase.BUILD) diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala index 8c8a587fc..5c26fdc30 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala @@ -120,7 +120,7 @@ case class DeltaVacuumTarget( */ override def link(linker: Linker, phase:Phase): Unit = { if (phase == Phase.BUILD) { - linker.write(relation.identifier, Map()) + linker.write(relation.identifier, Map.empty[String,SingleValue]) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala index 4db6b556f..75ed819a2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala @@ -34,6 +34,7 @@ import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.spec.relation.RelationReferenceSpec import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.FieldType +import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.StructType import com.dimajix.spark.sql.SchemaUtils @@ -116,7 +117,7 @@ case class ReadStreamMapping ( * Params: linker - The linker object to use for creating new edges */ override def link(linker: Linker): Unit = { - linker.read(relation.identifier, Map()) + linker.read(relation, Map.empty[String,FieldValue]) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala new file mode 100644 index 000000000..81b06de72 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala @@ -0,0 +1,181 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.target + +import com.fasterxml.jackson.annotation.JsonProperty +import org.slf4j.LoggerFactory + +import com.dimajix.common.No +import com.dimajix.common.Trilean +import com.dimajix.common.Yes +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.VerificationFailedException +import com.dimajix.flowman.graph.Linker +import com.dimajix.flowman.model.BaseTarget +import com.dimajix.flowman.model.Reference +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.RelationReference +import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.model.Target +import com.dimajix.flowman.model.TargetDigest +import com.dimajix.flowman.spec.relation.RelationReferenceSpec +import com.dimajix.flowman.types.SingleValue + + +object DropTarget { + def apply(context: Context, relation: RelationIdentifier) : DropTarget = { + new DropTarget( + Target.Properties(context, relation.name, "relation"), + RelationReference(context, relation) + ) + } +} +case class DropTarget( + instanceProperties: Target.Properties, + relation: Reference[Relation], +) extends BaseTarget { + private val logger = LoggerFactory.getLogger(classOf[RelationTarget]) + + /** + * Returns an instance representing this target with the context + * @return + */ + override def digest(phase:Phase) : TargetDigest = { + TargetDigest( + namespace.map(_.name).getOrElse(""), + project.map(_.name).getOrElse(""), + name, + phase, + Map() + ) + } + + /** + * Returns all phases which are implemented by this target in the execute method + * @return + */ + override def phases : Set[Phase] = { + Set(Phase.CREATE, Phase.VERIFY, Phase.DESTROY) + } + + /** + * Returns a list of physical resources produced by this target + * @return + */ + override def provides(phase: Phase) : Set[ResourceIdentifier] = { + Set() + } + + /** + * Returns a list of physical resources required by this target + * @return + */ + override def requires(phase: Phase) : Set[ResourceIdentifier] = { + phase match { + case Phase.CREATE|Phase.DESTROY => relation.value.provides ++ relation.value.requires + case _ => Set() + } + } + + /** + * Returns the state of the target, specifically of any artifacts produces. If this method return [[Yes]], + * then an [[execute]] should update the output, such that the target is not 'dirty' any more. + * + * @param execution + * @param phase + * @return + */ + override def dirty(execution: Execution, phase: Phase): Trilean = { + val rel = relation.value + + phase match { + case Phase.CREATE => + rel.exists(execution) != No + case Phase.VERIFY => Yes + case Phase.DESTROY => + rel.exists(execution) != No + case _ => No + } + } + + /** + * Creates all known links for building a descriptive graph of the whole data flow + * Params: linker - The linker object to use for creating new edges + */ + override def link(linker: Linker, phase:Phase): Unit = { + phase match { + case Phase.CREATE|Phase.DESTROY => + linker.write(relation, Map.empty[String,SingleValue]) + case _ => + } + } + + /** + * Drop the relation and all data contained + * + * @param executor + */ + override def create(execution: Execution) : Unit = { + require(execution != null) + + logger.info(s"Destroying relation '${relation.identifier}'") + val rel = relation.value + rel.destroy(execution, true) + } + + /** + * Verifies that the relation does not exist any more + * + * @param execution + */ + override def verify(execution: Execution) : Unit = { + require(execution != null) + + val rel = relation.value + if (rel.exists(execution) == Yes) { + logger.error(s"Verification of target '$identifier' failed - relation '${relation.identifier}' still exists") + throw new VerificationFailedException(identifier) + } + } + + /** + * Destroys both the logical relation and the physical data + * @param executor + */ + override def destroy(execution: Execution) : Unit = { + require(execution != null) + + logger.info(s"Destroying relation '${relation.identifier}'") + val rel = relation.value + rel.destroy(execution, true) + } +} + + +class DropTargetSpec extends TargetSpec { + @JsonProperty(value="relation", required=true) private var relation:RelationReferenceSpec = _ + + override def instantiate(context: Context): DropTarget = { + DropTarget( + instanceProperties(context), + relation.instantiate(context) + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MergeTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MergeTarget.scala index 7ed2d8d15..7f7fc59b0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MergeTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MergeTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,6 +54,7 @@ import com.dimajix.flowman.model.Target import com.dimajix.flowman.spec.relation.IdentifierRelationReferenceSpec import com.dimajix.flowman.spec.relation.RelationReferenceSpec import com.dimajix.flowman.spec.target.MergeTargetSpec.MergeClauseSpec +import com.dimajix.flowman.types.SingleValue object MergeTarget { @@ -155,12 +156,12 @@ case class MergeTarget( override def link(linker: Linker, phase:Phase): Unit = { phase match { case Phase.CREATE|Phase.DESTROY => - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case Phase.BUILD => linker.input(mapping.mapping, mapping.output) - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case Phase.TRUNCATE => - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case _ => } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala index bafdb424c..38be75e19 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -200,14 +200,14 @@ case class RelationTarget( override def link(linker: Linker, phase:Phase): Unit = { phase match { case Phase.CREATE|Phase.DESTROY => - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case Phase.BUILD if (mapping.nonEmpty) => val partition = this.partition.mapValues(v => SingleValue(v)) linker.input(mapping.mapping, mapping.output) - linker.write(relation.identifier, partition) + linker.write(relation, partition) case Phase.TRUNCATE => val partition = this.partition.mapValues(v => SingleValue(v)) - linker.write(relation.identifier, partition) + linker.write(relation, partition) case _ => } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/StreamTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/StreamTarget.scala index 914057cfd..d7f29a34e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/StreamTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/StreamTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -136,9 +136,9 @@ case class StreamTarget( phase match { case Phase.BUILD => linker.input(mapping.mapping, mapping.output) - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case Phase.TRUNCATE|Phase.DESTROY => - linker.write(relation.identifier, Map()) + linker.write(relation, Map.empty[String,SingleValue]) case _ => } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala index cd3b5fb61..793c90fec 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala @@ -49,6 +49,7 @@ object TargetSpec extends TypeRegistry[TargetSpec] { new JsonSubTypes.Type(name = "copyFile", value = classOf[CopyFileTargetSpec]), new JsonSubTypes.Type(name = "count", value = classOf[CountTargetSpec]), new JsonSubTypes.Type(name = "deleteFile", value = classOf[DeleteFileTargetSpec]), + new JsonSubTypes.Type(name = "drop", value = classOf[DropTargetSpec]), new JsonSubTypes.Type(name = "file", value = classOf[FileTargetSpec]), new JsonSubTypes.Type(name = "getFile", value = classOf[GetFileTargetSpec]), new JsonSubTypes.Type(name = "hiveDatabase", value = classOf[HiveDatabaseTargetSpec]), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala index addc76a51..3529559b1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,27 +22,51 @@ import org.slf4j.LoggerFactory import com.dimajix.common.No import com.dimajix.common.Trilean import com.dimajix.common.Yes +import com.dimajix.flowman.config.FlowmanConf.DEFAULT_TARGET_OUTPUT_MODE +import com.dimajix.flowman.config.FlowmanConf.DEFAULT_TARGET_PARALLELISM +import com.dimajix.flowman.config.FlowmanConf.DEFAULT_TARGET_REBALANCE import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.execution.OutputMode import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.execution.VerificationFailedException import com.dimajix.flowman.graph.Linker import com.dimajix.flowman.model.BaseTarget +import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.PartitionSchema +import com.dimajix.flowman.model.Reference import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.RelationReference import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetDigest +import com.dimajix.flowman.spec.relation.RelationReferenceSpec import com.dimajix.flowman.types.ArrayValue import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.RangeValue import com.dimajix.flowman.types.SingleValue +object TruncateTarget { + def apply(context: Context, relation: RelationIdentifier) : TruncateTarget = { + new TruncateTarget( + Target.Properties(context, relation.name, "relation"), + RelationReference(context, relation), + Map() + ) + } + def apply(context: Context, relation: RelationIdentifier, partitions:Map[String,FieldValue]) : TruncateTarget = { + new TruncateTarget( + Target.Properties(context, relation.name, "relation"), + RelationReference(context, relation), + partitions + ) + } +} case class TruncateTarget( instanceProperties: Target.Properties, - relation: RelationIdentifier, + relation: Reference[Relation], partitions:Map[String,FieldValue] = Map() ) extends BaseTarget { private val logger = LoggerFactory.getLogger(classOf[RelationTarget]) @@ -66,7 +90,7 @@ case class TruncateTarget( * @return */ override def phases : Set[Phase] = { - Set(Phase.BUILD, Phase.VERIFY) + Set(Phase.BUILD, Phase.VERIFY, Phase.TRUNCATE) } /** @@ -75,8 +99,8 @@ case class TruncateTarget( */ override def provides(phase: Phase) : Set[ResourceIdentifier] = { phase match { - case Phase.BUILD => - val rel = context.getRelation(relation) + case Phase.BUILD|Phase.TRUNCATE => + val rel = relation.value rel.provides ++ rel.resources(partitions) case _ => Set() } @@ -87,10 +111,10 @@ case class TruncateTarget( * @return */ override def requires(phase: Phase) : Set[ResourceIdentifier] = { - val rel = context.getRelation(relation) - phase match { - case Phase.BUILD => rel.provides ++ rel.requires + case Phase.BUILD|Phase.TRUNCATE => + val rel = relation.value + rel.provides ++ rel.requires case _ => Set() } } @@ -106,14 +130,11 @@ case class TruncateTarget( */ override def dirty(execution: Execution, phase: Phase): Trilean = { phase match { - case Phase.VALIDATE => No - case Phase.CREATE => No - case Phase.BUILD => - val rel = context.getRelation(relation) + case Phase.BUILD|Phase.TRUNCATE => + val rel = relation.value resolvedPartitions(rel).foldLeft(No:Trilean)((l,p) => l || rel.loaded(execution, p)) case Phase.VERIFY => Yes - case Phase.TRUNCATE => No - case Phase.DESTROY => No + case _ => No } } @@ -122,9 +143,10 @@ case class TruncateTarget( * Params: linker - The linker object to use for creating new edges */ override def link(linker: Linker, phase:Phase): Unit = { - if (phase == Phase.BUILD) { - val rel = context.getRelation(relation) - resolvedPartitions(rel).foreach(p => linker.write(relation, p)) + phase match { + case Phase.BUILD|Phase.TRUNCATE => + val rel = relation.value + resolvedPartitions(rel).foreach(p => linker.write(rel, p)) } } @@ -136,7 +158,7 @@ case class TruncateTarget( override def build(execution:Execution) : Unit = { require(execution != null) - val rel = context.getRelation(relation) + val rel = relation.value rel.truncate(execution, partitions) } @@ -148,7 +170,7 @@ case class TruncateTarget( override def verify(execution: Execution) : Unit = { require(execution != null) - val rel = context.getRelation(relation) + val rel = relation.value resolvedPartitions(rel) .find(p => rel.loaded(execution, p) == Yes) .foreach { partition => @@ -160,6 +182,18 @@ case class TruncateTarget( } } + /** + * Builds the target using the given input tables + * + * @param execution + */ + override def truncate(execution:Execution) : Unit = { + require(execution != null) + + val rel = relation.value + rel.truncate(execution, partitions) + } + private def resolvedPartitions(relation:Relation) : Iterable[Map[String,SingleValue]] = { if (this.partitions.isEmpty) { Seq(Map()) @@ -174,7 +208,7 @@ case class TruncateTarget( class TruncateTargetSpec extends TargetSpec { - @JsonProperty(value = "relation", required = true) private var relation:String = _ + @JsonProperty(value="relation", required=true) private var relation:RelationReferenceSpec = _ @JsonProperty(value = "partitions", required=false) private var partitions:Map[String,FieldValue] = Map() /** @@ -190,7 +224,7 @@ class TruncateTargetSpec extends TargetSpec { } TruncateTarget( instanceProperties(context), - RelationIdentifier(context.evaluate(relation)), + relation.instantiate(context), partitions ) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DropTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DropTargetTest.scala new file mode 100644 index 000000000..3ae1d2801 --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DropTargetTest.scala @@ -0,0 +1,115 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.target + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.common.No +import com.dimajix.common.Yes +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.ScopeContext +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.execution.VerificationFailedException +import com.dimajix.flowman.model.IdentifierRelationReference +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.spec.ObjectMapper +import com.dimajix.flowman.types.FieldValue +import com.dimajix.spark.testing.LocalSparkSession + + +class DropTargetTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { + "The DropTarget" should "be parseable" in { + val spec = + """ + |kind: drop + |relation: some_relation + |""".stripMargin + + val session = Session.builder().disableSpark().build() + val context = session.context + + val targetSpec = ObjectMapper.parse[TargetSpec](spec) + val target = targetSpec.instantiate(context).asInstanceOf[DropTarget] + + target.relation should be (IdentifierRelationReference(context, "some_relation")) + } + + it should "work" in { + val session = Session.builder().withSparkSession(spark).build() + val execution = session.execution + + val relationTemplate = mock[Prototype[Relation]] + val relation = mock[Relation] + val context = ScopeContext.builder(session.context) + .withRelations(Map("some_relation" -> relationTemplate)) + .build() + val target = DropTarget( + context, + RelationIdentifier("some_relation") + ) + + (relationTemplate.instantiate _).expects(*).returns(relation) + + target.phases should be (Set(Phase.CREATE, Phase.VERIFY, Phase.DESTROY)) + + target.provides(Phase.VALIDATE) should be (Set()) + target.provides(Phase.CREATE) should be (Set()) + target.provides(Phase.BUILD) should be (Set()) + target.provides(Phase.VERIFY) should be (Set()) + target.provides(Phase.DESTROY) should be (Set()) + + target.requires(Phase.VALIDATE) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.requires(Phase.CREATE) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) + target.requires(Phase.BUILD) should be (Set()) + target.requires(Phase.VERIFY) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.requires(Phase.DESTROY) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) + + (relation.exists _).expects(execution).returns(Yes) + target.dirty(execution, Phase.CREATE) should be (Yes) + target.dirty(execution, Phase.VERIFY) should be (Yes) + (relation.exists _).expects(execution).returns(Yes) + target.dirty(execution, Phase.DESTROY) should be (Yes) + + (relation.exists _).expects(execution).returns(Yes) + target.execute(execution, Phase.VERIFY).exception.get shouldBe a[VerificationFailedException] + + (relation.destroy _).expects(execution, true) + target.execute(execution, Phase.CREATE).status should be (Status.SUCCESS) + + (relation.exists _).expects(execution).returns(Yes) + target.execute(execution, Phase.VERIFY).status should be (Status.FAILED) + + (relation.exists _).expects(execution).returns(No) + target.execute(execution, Phase.VERIFY).status should be (Status.SUCCESS) + + (relation.exists _).expects(execution).returns(No) + target.dirty(execution, Phase.CREATE) should be (No) + + (relation.exists _).expects(execution).returns(No) + target.dirty(execution, Phase.DESTROY) should be (No) + } +} diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/TruncateTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/TruncateTargetTest.scala index 9471fc693..ca0515b28 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/TruncateTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/TruncateTargetTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import com.dimajix.flowman.execution.ScopeContext import com.dimajix.flowman.execution.Session import com.dimajix.flowman.execution.Status import com.dimajix.flowman.execution.VerificationFailedException +import com.dimajix.flowman.model.IdentifierRelationReference import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier @@ -61,7 +62,7 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with val targetSpec = ObjectMapper.parse[TargetSpec](spec) val target = targetSpec.instantiate(context).asInstanceOf[TruncateTarget] - target.relation should be (RelationIdentifier("some_relation")) + target.relation should be (IdentifierRelationReference(context, "some_relation")) target.partitions should be (Map( "p1" -> SingleValue("1234"), "p2" -> RangeValue("a", "x") @@ -78,7 +79,7 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with .withRelations(Map("some_relation" -> relationTemplate)) .build() val target = TruncateTarget( - Target.Properties(context), + context, RelationIdentifier("some_relation"), Map( "p1" -> SingleValue("1234"), @@ -88,30 +89,42 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with (relationTemplate.instantiate _).expects(*).returns(relation) - target.phases should be (Set(Phase.BUILD, Phase.VERIFY)) + target.phases should be (Set(Phase.BUILD, Phase.VERIFY, Phase.TRUNCATE)) + target.provides(Phase.VALIDATE) should be (Set()) + target.provides(Phase.CREATE) should be (Set()) (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) (relation.resources _).expects(Map("p1" -> SingleValue("1234"),"p2" -> RangeValue("1", "3"))).returns(Set( ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "1")), ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "2")) )) - - target.provides(Phase.VALIDATE) should be (Set()) - target.provides(Phase.CREATE) should be (Set()) target.provides(Phase.BUILD) should be (Set( ResourceIdentifier.ofHiveTable("some_table"), ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "1")), ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "2")) )) target.provides(Phase.VERIFY) should be (Set()) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + (relation.resources _).expects(Map("p1" -> SingleValue("1234"),"p2" -> RangeValue("1", "3"))).returns(Set( + ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "1")), + ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "2")) + )) + target.provides(Phase.TRUNCATE) should be (Set( + ResourceIdentifier.ofHiveTable("some_table"), + ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "1")), + ResourceIdentifier.ofHivePartition("some_table", Some("db"), Map("p1" -> "1234", "p2" -> "2")) + )) target.provides(Phase.DESTROY) should be (Set()) - (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) - (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.VALIDATE) should be (Set()) target.requires(Phase.CREATE) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.BUILD) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.VERIFY) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.requires(Phase.TRUNCATE) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.DESTROY) should be (Set()) (relation.partitions _).expects().returns(Seq(PartitionField("p1", StringType), PartitionField("p2", IntegerType))) @@ -119,6 +132,10 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with (relation.loaded _).expects(execution, Map("p1" -> SingleValue("1234"),"p2" -> SingleValue("2"))).returns(No) target.dirty(execution, Phase.BUILD) should be (Yes) target.dirty(execution, Phase.VERIFY) should be (Yes) + (relation.partitions _).expects().returns(Seq(PartitionField("p1", StringType), PartitionField("p2", IntegerType))) + (relation.loaded _).expects(execution, Map("p1" -> SingleValue("1234"),"p2" -> SingleValue("1"))).returns(Yes) + (relation.loaded _).expects(execution, Map("p1" -> SingleValue("1234"),"p2" -> SingleValue("2"))).returns(No) + target.dirty(execution, Phase.TRUNCATE) should be (Yes) (relation.partitions _).expects().returns(Seq(PartitionField("p1", StringType), PartitionField("p2", IntegerType))) (relation.loaded _).expects(execution, Map("p1" -> SingleValue("1234"),"p2" -> SingleValue("1"))).returns(No) @@ -149,34 +166,41 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with .withRelations(Map("some_relation" -> relationTemplate)) .build() val target = TruncateTarget( - Target.Properties(context), + context, RelationIdentifier("some_relation") ) (relationTemplate.instantiate _).expects(*).returns(relation) - target.phases should be (Set(Phase.BUILD, Phase.VERIFY)) - - (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) - (relation.resources _).expects(Map.empty[String,FieldValue]).returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.phases should be (Set(Phase.BUILD, Phase.VERIFY, Phase.TRUNCATE)) target.provides(Phase.VALIDATE) should be (Set()) target.provides(Phase.CREATE) should be (Set()) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + (relation.resources _).expects(Map.empty[String,FieldValue]).returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) target.provides(Phase.BUILD) should be (Set(ResourceIdentifier.ofHiveTable("some_table"))) target.provides(Phase.VERIFY) should be (Set()) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + (relation.resources _).expects(Map.empty[String,FieldValue]).returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.provides(Phase.TRUNCATE) should be (Set(ResourceIdentifier.ofHiveTable("some_table"))) target.provides(Phase.DESTROY) should be (Set()) - (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) - (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.VALIDATE) should be (Set()) target.requires(Phase.CREATE) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.BUILD) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.VERIFY) should be (Set()) + (relation.requires _).expects().returns(Set(ResourceIdentifier.ofHiveDatabase("db"))) + (relation.provides _).expects().returns(Set(ResourceIdentifier.ofHiveTable("some_table"))) + target.requires(Phase.TRUNCATE) should be (Set(ResourceIdentifier.ofHiveDatabase("db"), ResourceIdentifier.ofHiveTable("some_table"))) target.requires(Phase.DESTROY) should be (Set()) (relation.loaded _).expects(execution, Map.empty[String,SingleValue]).returns(Yes) target.dirty(execution, Phase.BUILD) should be (Yes) target.dirty(execution, Phase.VERIFY) should be (Yes) + (relation.loaded _).expects(execution, Map.empty[String,SingleValue]).returns(Yes) + target.dirty(execution, Phase.TRUNCATE) should be (Yes) (relation.loaded _).expects(execution, Map.empty[String,SingleValue]).returns(Yes) target.execute(execution, Phase.VERIFY).exception.get shouldBe a[VerificationFailedException] @@ -189,5 +213,8 @@ class TruncateTargetTest extends AnyFlatSpec with Matchers with MockFactory with (relation.loaded _).expects(execution, Map.empty[String,SingleValue]).returns(No) target.dirty(execution, Phase.BUILD) should be (No) + + (relation.loaded _).expects(execution, Map.empty[String,SingleValue]).returns(No) + target.dirty(execution, Phase.TRUNCATE) should be (No) } } diff --git a/pom.xml b/pom.xml index c5b3ac91c..8c8f81aac 100644 --- a/pom.xml +++ b/pom.xml @@ -903,7 +903,7 @@ net.alchim31.maven scala-maven-plugin - 4.5.3 + 4.5.4 -Xms64m From 85ef47af50f8cff8db1c6ddee5dc8a0d07e69d76 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 11 Feb 2022 17:02:57 +0100 Subject: [PATCH 28/95] Fix build for Scala 2.11 --- .../dimajix/flowman/spec/target/DropTarget.scala | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala index 81b06de72..f07c78388 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DropTarget.scala @@ -49,24 +49,10 @@ object DropTarget { } case class DropTarget( instanceProperties: Target.Properties, - relation: Reference[Relation], + relation: Reference[Relation] ) extends BaseTarget { private val logger = LoggerFactory.getLogger(classOf[RelationTarget]) - /** - * Returns an instance representing this target with the context - * @return - */ - override def digest(phase:Phase) : TargetDigest = { - TargetDigest( - namespace.map(_.name).getOrElse(""), - project.map(_.name).getOrElse(""), - name, - phase, - Map() - ) - } - /** * Returns all phases which are implemented by this target in the execute method * @return From 20cd8fbcdd437fe4c679d605748c9aa320e80546 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 11 Feb 2022 19:09:07 +0100 Subject: [PATCH 29/95] Improve documentation --- docs/documenting/relations.md | 23 ++++++++-------- docs/documenting/targets.md | 16 ++++++++++++ docs/spec/target/drop.md | 4 +-- docs/spec/target/relation.md | 14 +++++++--- docs/spec/target/truncate.md | 39 +++++++++++++++++++++++++++- examples/weather/target/stations.yml | 3 +++ 6 files changed, 82 insertions(+), 17 deletions(-) diff --git a/docs/documenting/relations.md b/docs/documenting/relations.md index a4a9aa0e8..98de36456 100644 --- a/docs/documenting/relations.md +++ b/docs/documenting/relations.md @@ -19,17 +19,18 @@ relations: - name: year type: integer granularity: 1 - documentation: - description: "The table contains all aggregated measurements" - columns: - - name: country - description: "Country of the weather station" - - name: min_temperature - description: "Minimum air temperature per year in degrees Celsius" - - name: max_temperature - description: "Maximum air temperature per year in degrees Celsius" - - name: avg_temperature - description: "Average air temperature per year in degrees Celsius" + + documentation: + description: "The table contains all aggregated measurements" + columns: + - name: country + description: "Country of the weather station" + - name: min_temperature + description: "Minimum air temperature per year in degrees Celsius" + - name: max_temperature + description: "Maximum air temperature per year in degrees Celsius" + - name: avg_temperature + description: "Average air temperature per year in degrees Celsius" ``` ## Fields diff --git a/docs/documenting/targets.md b/docs/documenting/targets.md index 1208749af..d65c6589e 100644 --- a/docs/documenting/targets.md +++ b/docs/documenting/targets.md @@ -1 +1,17 @@ # Documenting Targets + +Flowman also supports documenting build targets. + +## Example + +```yaml +targets: + stations: + kind: relation + description: "Write stations" + mapping: stations_raw + relation: stations + + documentation: + description: "This build target is used to write the weather stations" +``` diff --git a/docs/spec/target/drop.md b/docs/spec/target/drop.md index 842e74f31..3e8bf7497 100644 --- a/docs/spec/target/drop.md +++ b/docs/spec/target/drop.md @@ -22,7 +22,7 @@ relations: file: "${project.basedir}/schema/stations.avsc" ``` -Since Flowman 0.18.0, you can also directly specify the relation inside the target definition. This saves you +You can also directly specify the relation inside the target definition. This saves you from having to create a separate relation definition in the `relations` section. This is only recommended, if you do not access the target relation otherwise, such that a shared definition would not provide any benefit. ```yaml @@ -47,7 +47,7 @@ targets: Optional descriptive text of the build target * `relation` **(mandatory)** *(type: string)*: -Specifies the name of the relation to write to +Specifies the name of the relation to drop, or alternatively directly embeds the relation. ## Description diff --git a/docs/spec/target/relation.md b/docs/spec/target/relation.md index 99112c202..9665e826a 100644 --- a/docs/spec/target/relation.md +++ b/docs/spec/target/relation.md @@ -16,7 +16,7 @@ targets: parallelism: 32 rebalance: true partition: - processing_date: "${processing_date}" + year: "${processing_date}" relations: stations: @@ -26,6 +26,10 @@ relations: schema: kind: avro file: "${project.basedir}/schema/stations.avsc" + partitions: + - name: year + type: integer + granularity: 1 ``` Since Flowman 0.18.0, you can also directly specify the relation inside the target definition. This saves you @@ -44,11 +48,15 @@ targets: schema: kind: avro file: "${project.basedir}/schema/stations.avsc" + partitions: + - name: year + type: integer + granularity: 1 mode: overwrite parallelism: 32 rebalance: true partition: - processing_date: "${processing_date}" + year: "${processing_date}" ``` ## Fields @@ -62,7 +70,7 @@ targets: Specifies the name of the input mapping to be written * `relation` **(mandatory)** *(type: string)*: -Specifies the name of the relation to write to +Specifies the name of the relation to write to, or alternatively directly embeds the relation. * `mode` **(optional)** *(type: string)* *(default=overwrite)*: Specifies the behavior when data or table or partition already exists. Options include: diff --git a/docs/spec/target/truncate.md b/docs/spec/target/truncate.md index 7faebb7f0..cf213efe7 100644 --- a/docs/spec/target/truncate.md +++ b/docs/spec/target/truncate.md @@ -15,6 +15,43 @@ targets: year: start: $start_year end: $end_year + +relations: + stations: + kind: file + format: parquet + location: "$basedir/stations/" + schema: + kind: avro + file: "${project.basedir}/schema/stations.avsc" + partitions: + - name: year + type: integer + granularity: 1 +``` + +Since Flowman 0.22.0, you can also directly specify the relation inside the target definition. This saves you +from having to create a separate relation definition in the `relations` section. This is only recommended, if you +do not access the target relation otherwise, such that a shared definition would not provide any benefit. +```yaml +targets: + truncate_stations: + kind: truncate + partitions: + year: + start: $start_year + end: $end_year + relation: stations-relation + kind: file + format: parquet + location: "$basedir/stations/" + schema: + kind: avro + file: "${project.basedir}/schema/stations.avsc" + partitions: + - name: year + type: integer + granularity: 1 ``` ## Fields @@ -25,7 +62,7 @@ targets: Optional descriptive text of the build target * `relation` **(mandatory)** *(type: string)*: - Specifies the name of the relation to truncate. + Specifies the name of the relation to truncate, or alternatively directly embeds the relation. * `partitions` **(optional)** *(type: map:partition)*: Specifies the partition (or multiple partitions) to truncate. diff --git a/examples/weather/target/stations.yml b/examples/weather/target/stations.yml index 48b689669..294889278 100644 --- a/examples/weather/target/stations.yml +++ b/examples/weather/target/stations.yml @@ -4,3 +4,6 @@ targets: description: "Write stations" mapping: stations_raw relation: stations + + documentation: + description: "This build target is used to write the weather stations" From 043467bd9c564a5ac43582ce5dc2edbbd8aa1d2e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 11 Feb 2022 21:15:35 +0100 Subject: [PATCH 30/95] Improve importing mechanism --- docs/cookbook/sharing.md | 97 +++++++++++ docs/spec/project.md | 39 +++-- .../flowman/execution/RootContext.scala | 152 ++++++++++++------ .../com/dimajix/flowman/model/Project.scala | 9 ++ .../flowman/execution/RootContextTest.scala | 50 ++++++ .../spec/{Module.scala => ModuleSpec.scala} | 2 +- .../{Namespace.scala => NamespaceSpec.scala} | 0 .../spec/{Profile.scala => ProfileSpec.scala} | 2 +- .../spec/{Project.scala => ProjectSpec.scala} | 27 +++- .../dimajix/flowman/spec/ProjectTest.scala | 11 +- .../com/dimajix/flowman/tools/Tool.scala | 3 +- 11 files changed, 322 insertions(+), 70 deletions(-) create mode 100644 docs/cookbook/sharing.md rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/{Module.scala => ModuleSpec.scala} (98%) rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/{Namespace.scala => NamespaceSpec.scala} (100%) rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/{Profile.scala => ProfileSpec.scala} (97%) rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/{Project.scala => ProjectSpec.scala} (56%) diff --git a/docs/cookbook/sharing.md b/docs/cookbook/sharing.md new file mode 100644 index 000000000..c0d68f94c --- /dev/null +++ b/docs/cookbook/sharing.md @@ -0,0 +1,97 @@ +# Sharing Entities between Projects + +In bigger projects, it makes sense to organize data transformations like Flowman projects into separate subprojects, +so they can be maintained independently by possibly different teams. A classical example would be to have a different +Flowman project per source system (let it be your CRM system, your financial transaction processing system etc). +In a data lake environment, you probably want to implement independent Flowman projects to perform the first +technical transformations for each of these source systems. Then in the next layer, you want to create a more +complex and integrated data model built on top of these independent models. + +In such scenarios, you want to share some common entity definitions between these projects, for example the Flowman +project for building the integrated data model may want to reuse the relations from the other projects. + +Flowman well supports these scenarios by the concept of imports. + +## Example + +First you define a project which exports entities. Actually you might need to do nothing, since importing a project +will make all entities available to the importing side. But maybe your project also requires some variables to be +set, like the processing date. Typically you would include such variables as job parameters: +```yaml +# Project A, which contains shared resources +jobs: + # Define a base job with common environment variables + base: + parameters: + - name: processing_datetime + type: timestamp + description: "Specifies the datetime in yyyy-MM-ddTHH:mm:ss.Z for which the result will be generated" + - name: processing_duration + type: duration + description: "Specifies the processing duration (either P1D or PT1H)" + environment: + - start_ts=$processing_datetime + - end_ts=${Timestamp.add(${processing_datetime}, ${processing_duration})} + - start_unixtime=${Timestamp.parse($start_ts).toEpochSeconds()} + - end_unixtime=${Timestamp.parse($end_ts).toEpochSeconds()} + + # Define a specific job for daily processing + daily: + extends: base + parameters: + - name: processing_datetime + type: timestamp + environment: + - processing_duration=P1D + + # Define a specific job for hourly processing + hourly: + extends: base + parameters: + - name: processing_datetime + environment: + - processing_duration=PT1H +``` + +The another project may want to access resources from project A, but within the context of the `export` job. This +can be achieved by declaring the dependency in an `imports` section within the project manifest: + +```yaml +# project.yml of another project +name: raw-exporter + +imports: + # Import with no job and no (or default) parameters + - poject: project_b + + # Import project with specified job context + - project: project_a + # The job may even be a variable, so different job context can be imported + job: $period + arguments: + processing_datetime: $processing_datetime +``` +Then you can easily access entities from `project_a` and `project_b` as follows: + +```yaml +mappings: + # You can access all entities from different projects by using the project name followed by a slash ("/") + sap_transactions: + kind: filter + input: project_b/transactions + condition: "transaction_code = 8100" + +relations: + ad_impressions: + kind: alias + input: project_a/ad_impressions + +jobs: + main: + parameters: + - name: processing_datetime + type: timestamp + environment: + # Set the variable $period, so it will be used to import the correct job + - period=daily +``` diff --git a/docs/spec/project.md b/docs/spec/project.md index 9cea1c64a..68c839b13 100644 --- a/docs/spec/project.md +++ b/docs/spec/project.md @@ -1,12 +1,15 @@ # Projects +The specification of all relations, data transformations and build targets is done within Flowman projects. Each +project has a top level project descriptor which mainly contains some meta information like project name and +version and a list of subdirectories, which contain the entity definitions. + ## Project Specification -Flowman always requires a *Project* top level file containing general information (like a -projects name and version) and directories where to look for specifications. The project -file should be named `project.yml`, this way `flowexec` will directly pick it up when only -the directory is given on the command line. +Flowman always requires a *Project* top level file containing general information (like a projects name and version) +and directories where to look for specifications. The project file should be named `project.yml`, this way `flowexec` +and `flowshell` will directly pick it up when only the directory is given on the command line. A typical `project.yml` file looks as follows: @@ -21,6 +24,13 @@ modules: - mapping - target - job + +imports: + - project: other_project + + - project: commons + arguments: + processing_date: $processing_date ``` ## Fields @@ -28,21 +38,24 @@ modules: Each project supports the following fields: * `name` **(mandatory)** *(string)* -The name of the overall project. This field will be used in a later Flowman version for -sharing mappings and relations between different projects. +The name of the overall project. This field is used by Flowman for sharing mappings and relations between different +projects. * `version` **(optional)** *(string)* -The version currently is not used by Flowman, but can be used for the end-user to help keeping -track of which version of a project is currently being used. +The version currently is not used by Flowman, but can be used for the end-user to help keeping track of which version +of a project is currently being used. * `description` **(optional)** *(string)* A description of the overall project. Can be any text, is not used by Flowman otherwise -* `modules` **(mandatory)** *(list)* -The `modules` secion contains a list of *subdirectories* or *filenames* where Flowman should -search for more YAML specification files. This helps to organize complex projects into -different modules and/or aspects. The directory and file names are relative to the project -file itself. +* `modules` **(mandatory)** *(list:string)* +The `modules` secion contains a list of *subdirectories* or *filenames* where Flowman should search for more YAML +specification files. This helps to organize complex projects into different modules and/or aspects. The directory and +file names are relative to the project file itself. + +* `imports` **(optional)** *(list:import)* +Within the `imports` section you can specify different projects to be imported and made available for referencing +its entities. ## Proposed Directory Layout diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala index 9a3f9e9c7..7f4b5c435 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala @@ -16,15 +16,15 @@ package com.dimajix.flowman.execution +import scala.collection.concurrent.TrieMap import scala.collection.mutable +import scala.util.control.NonFatal import org.apache.hadoop.conf.{Configuration => HadoopConf} import org.apache.spark.SparkConf import org.slf4j.LoggerFactory -import com.dimajix.flowman.config.Configuration import com.dimajix.flowman.config.FlowmanConf -import com.dimajix.flowman.execution.ProjectContext.Builder import com.dimajix.flowman.hadoop.FileSystem import com.dimajix.flowman.model.Connection import com.dimajix.flowman.model.ConnectionIdentifier @@ -36,11 +36,11 @@ import com.dimajix.flowman.model.Namespace import com.dimajix.flowman.model.NamespaceWrapper import com.dimajix.flowman.model.Profile import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetIdentifier -import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Template import com.dimajix.flowman.model.TemplateIdentifier import com.dimajix.flowman.model.Test @@ -125,7 +125,8 @@ final class RootContext private[execution]( _env + ("namespace" -> (NamespaceWrapper(_namespace) -> SettingLevel.SCOPE_OVERRIDE.level)), _config ) { - private val _children: mutable.Map[String, Context] = mutable.Map() + private val _children: TrieMap[String, Context] = TrieMap() + private val _imports:TrieMap[String,(Context,Project.Import)] = TrieMap() private lazy val _fs = FileSystem(hadoopConf) private lazy val _exec = _execution match { case Some(execution) => execution @@ -169,10 +170,12 @@ final class RootContext private[execution]( override def getMapping(identifier: MappingIdentifier, allowOverrides:Boolean=true): Mapping = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchMappingException(identifier) - val child = getProjectContext(identifier.project.get) - child.getMapping(identifier, allowOverrides) + identifier.project match { + case None => throw new NoSuchMappingException(identifier) + case Some(project) => + val child = getProjectContext(project) + child.getMapping(identifier, allowOverrides) + } } /** @@ -184,10 +187,12 @@ final class RootContext private[execution]( override def getRelation(identifier: RelationIdentifier, allowOverrides:Boolean=true): Relation = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchRelationException(identifier) - val child = getProjectContext(identifier.project.get) - child.getRelation(identifier, allowOverrides) + identifier.project match { + case None => throw new NoSuchRelationException (identifier) + case Some(project) => + val child = getProjectContext (project) + child.getRelation (identifier, allowOverrides) + } } /** @@ -199,10 +204,12 @@ final class RootContext private[execution]( override def getTarget(identifier: TargetIdentifier): Target = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchTargetException(identifier) - val child = getProjectContext(identifier.project.get) - child.getTarget(identifier) + identifier.project match { + case None => throw new NoSuchTargetException(identifier) + case Some(project) => + val child = getProjectContext(project) + child.getTarget(identifier) + } } /** @@ -214,20 +221,20 @@ final class RootContext private[execution]( override def getConnection(identifier:ConnectionIdentifier) : Connection = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) { - connections.getOrElseUpdate(identifier.name, - extraConnections.get(identifier.name) - .orElse( - namespace - .flatMap(_.connections.get(identifier.name)) - ) - .map(_.instantiate(this)) - .getOrElse(throw new NoSuchConnectionException(identifier)) - ) - } - else { - val child = getProjectContext(identifier.project.get) - child.getConnection(identifier) + identifier.project match { + case None => + connections.getOrElseUpdate(identifier.name, + extraConnections.get(identifier.name) + .orElse( + namespace + .flatMap(_.connections.get(identifier.name)) + ) + .map(_.instantiate(this)) + .getOrElse(throw new NoSuchConnectionException(identifier)) + ) + case Some(project) => + val child = getProjectContext(project) + child.getConnection(identifier) } } @@ -240,10 +247,12 @@ final class RootContext private[execution]( override def getJob(identifier: JobIdentifier): Job = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchJobException(identifier) - val child = getProjectContext(identifier.project.get) - child.getJob(identifier) + identifier.project match { + case None => throw new NoSuchJobException (identifier) + case Some(project) => + val child = getProjectContext (project) + child.getJob (identifier) + } } /** @@ -255,10 +264,12 @@ final class RootContext private[execution]( override def getTest(identifier: TestIdentifier): Test = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchTestException(identifier) - val child = getProjectContext(identifier.project.get) - child.getTest(identifier) + identifier.project match { + case None => throw new NoSuchTestException(identifier) + case Some(project) => + val child = getProjectContext(project) + child.getTest(identifier) + } } /** @@ -270,10 +281,24 @@ final class RootContext private[execution]( override def getTemplate(identifier: TemplateIdentifier): Template[_] = { require(identifier != null && identifier.nonEmpty) - if (identifier.project.isEmpty) - throw new NoSuchTemplateException(identifier) - val child = getProjectContext(identifier.project.get) - child.getTemplate(identifier) + identifier.project match { + case None => throw new NoSuchTemplateException(identifier) + case Some(project) => + val child = getProjectContext(project) + child.getTemplate(identifier) + } + } + + /** + * Returns the context for a specific project. This will either return an existing context or create a new + * one if it does not exist yet. + * + * @param project + * @return + */ + def getProjectContext(project:Project) : Context = { + require(project != null) + _children.getOrElseUpdate(project.name, createProjectContext(project)) } /** @@ -286,10 +311,6 @@ final class RootContext private[execution]( require(projectName != null && projectName.nonEmpty) _children.getOrElseUpdate(projectName, createProjectContext(loadProject(projectName))) } - def getProjectContext(project:Project) : Context = { - require(project != null) - _children.getOrElseUpdate(project.name, createProjectContext(project)) - } private def createProjectContext(project: Project) : Context = { val builder = ProjectContext.builder(this, project) @@ -299,15 +320,50 @@ final class RootContext private[execution]( } } + // We need to instantiate the projects job within its context, so we create a very temporary context + def getJob(name:String) : Job = { + try { + val projectContext = ProjectContext.builder(this, project) + .withEnvironment(project.environment, SettingLevel.PROJECT_SETTING) + .build() + projectContext.getJob(JobIdentifier(name)) + } catch { + case NonFatal(ex) => + throw new IllegalArgumentException(s"Cannot instantiate job '$name' to apply import settings for project ${project.name}", ex) + } + } + + // Apply any import setting + _imports.get(project.name).foreach { case(context,imprt) => + val job = context.evaluate(imprt.job) match { + case Some(name) => + Some(getJob(name)) + case None => + if (project.jobs.contains("main")) + Some(getJob("main")) + else None + } + job.foreach { job => + val args = job.arguments(context.evaluate(imprt.arguments)) + builder.withEnvironment(args, SettingLevel.SCOPE_OVERRIDE) + builder.withEnvironment(job.environment, SettingLevel.JOB_OVERRIDE) + } + } + // Apply overrides builder.overrideMappings(overrideMappings.filter(_._1.project.contains(project.name)).map(kv => (kv._1.name, kv._2))) builder.overrideRelations(overrideRelations.filter(_._1.project.contains(project.name)).map(kv => (kv._1.name, kv._2))) - val context = builder.withEnvironment(project.environment) + val context = builder + .withEnvironment(project.environment, SettingLevel.PROJECT_SETTING) .withConfig(project.config) .build() - _children.update(project.name, context) + // Store imports, together with context + project.imports.foreach { im => + _imports.update(im.project, (context, im)) + } + context } private def loadProject(name: String): Project = { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala index 21eff1e85..95080826f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala @@ -26,9 +26,16 @@ import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.spi.ProjectReader + object Project { private lazy val loader = ServiceLoader.load(classOf[ProjectReader]).iterator().asScala.toSeq + case class Import( + project:String, + job:Option[String] = None, + arguments:Map[String,String] = Map() + ) + class Reader { private val logger = LoggerFactory.getLogger(classOf[Reader]) private var format = "yaml" @@ -138,7 +145,9 @@ final case class Project( config : Map[String,String] = Map(), environment : Map[String,String] = Map(), + imports: Seq[Project.Import] = Seq(), profiles : Map[String,Profile] = Map(), + relations : Map[String,Prototype[Relation]] = Map(), connections : Map[String,Prototype[Connection]] = Map(), mappings : Map[String,Prototype[Mapping]] = Map(), diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootContextTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootContextTest.scala index 4c49dd655..0c3aa98f9 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootContextTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootContextTest.scala @@ -22,6 +22,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.model.Connection import com.dimajix.flowman.model.ConnectionIdentifier +import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.Namespace @@ -30,6 +31,7 @@ import com.dimajix.flowman.model.Project import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.types.StringType class RootContextTest extends AnyFlatSpec with Matchers with MockFactory { @@ -238,4 +240,52 @@ class RootContextTest extends AnyFlatSpec with Matchers with MockFactory { rootContext.getRelation(RelationIdentifier("my_project/m2")) should be (overrideRelation) rootContext.getRelation(RelationIdentifier("my_project/m2"), false) should be (projectRelation2) } + + it should "support importing projects" in { + val session = Session.builder() + .disableSpark() + .build() + val rootContext = RootContext.builder(session.context) + .build() + + val project1 = Project( + name = "project1", + imports = Seq( + Project.Import(project="project2"), + Project.Import(project="project3"), + Project.Import(project="project4", job=Some("job"), arguments=Map("arg1" -> "val1")) + ) + ) + val project1Ctx = rootContext.getProjectContext(project1) + project1Ctx.evaluate("$project") should be ("project1") + + val project2 = Project( + name = "project2", + environment = Map("env1" -> "val1") + ) + val project2Ctx = rootContext.getProjectContext(project2) + project2Ctx.evaluate("$project") should be ("project2") + project2Ctx.evaluate("$env1") should be ("val1") + + val project3JobGen = mock[Prototype[Job]] + (project3JobGen.instantiate _).expects(*).onCall((ctx:Context) => Job.builder(ctx).setName("main").addEnvironment("jobenv", "jobval").build()) + val project3 = Project( + name = "project3", + jobs = Map("main" -> project3JobGen) + ) + val project3Ctx = rootContext.getProjectContext(project3) + project3Ctx.evaluate("$project") should be ("project3") + project3Ctx.evaluate("$jobenv") should be ("jobval") + + val project4JobGen = mock[Prototype[Job]] + (project4JobGen.instantiate _).expects(*).onCall((ctx:Context) => Job.builder(ctx).setName("job").addParameter("arg1", StringType).addParameter("arg2", StringType, value=Some("default")).build()) + val project4 = Project( + name = "project4", + jobs = Map("job" -> project4JobGen) + ) + val project4Ctx = rootContext.getProjectContext(project4) + project4Ctx.evaluate("$project") should be ("project4") + project4Ctx.evaluate("$arg1") should be ("val1") + project4Ctx.evaluate("$arg2") should be ("default") + } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Module.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ModuleSpec.scala similarity index 98% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/Module.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/ModuleSpec.scala index 9b44a8ee2..64cb4192a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Module.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ModuleSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Namespace.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/NamespaceSpec.scala similarity index 100% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/Namespace.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/NamespaceSpec.scala diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Profile.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProfileSpec.scala similarity index 97% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/Profile.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProfileSpec.scala index d55bccd21..633999c76 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Profile.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProfileSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Project.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProjectSpec.scala similarity index 56% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/Project.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProjectSpec.scala index 52d53c0b3..8c11f20fd 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/Project.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ProjectSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,21 +19,38 @@ package com.dimajix.flowman.spec import com.fasterxml.jackson.annotation.JsonProperty import com.dimajix.flowman.model.Project - - +import com.dimajix.flowman.spec.ProjectSpec.ImportSpec + + +object ProjectSpec { + final class ImportSpec { + @JsonProperty(value = "project", required = true) private var project: String = "" + @JsonProperty(value = "job", required = false) private var job: Option[String] = None + @JsonProperty(value = "arguments", required = false) private var arguments: Map[String,String] = Map() + def instantiate(): Project.Import = { + Project.Import( + project, + job, + arguments + ) + } + } +} final class ProjectSpec { @JsonProperty(value="name", required = true) private var name: String = "" @JsonProperty(value="description", required = false) private var description: Option[String] = None @JsonProperty(value="version", required = false) private var version: Option[String] = None - @JsonProperty(value="modules", required = true) private[spec] var modules: Seq[String] = Seq() + @JsonProperty(value="modules", required = true) private var modules: Seq[String] = Seq() + @JsonProperty(value="imports", required = true) private var imports: Seq[ImportSpec] = Seq() def instantiate(): Project = { Project( name=name, description=description, version=version, - modules=modules + modules=modules, + imports=imports.map(_.instantiate()) ) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/ProjectTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/ProjectTest.scala index 4ee37e925..d7306ce89 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/ProjectTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/ProjectTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,12 +32,21 @@ class ProjectTest extends AnyFlatSpec with Matchers { """ |name: test |version: 1.0 + | + |imports: + | - project: common + | job: some_job + | arguments: + | some_arg: $lala """.stripMargin val project = Project.read.string(spec) project.name should be ("test") project.version should be (Some("1.0")) project.filename should be (None) project.basedir should be (None) + project.imports should be (Seq( + Project.Import("common", job=Some("some_job"), arguments=Map("some_arg" -> "$lala")) + )) } it should "be readable from a file" in { diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/Tool.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/Tool.scala index ad18b365f..2f4beb84d 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/Tool.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/Tool.scala @@ -101,12 +101,13 @@ class Tool { // Create Flowman Session, which also includes a Spark Session val builder = Session.builder() .withNamespace(namespace) - .withProject(project.orNull) .withConfig(allConfigs) .withEnvironment(additionalEnvironment) .withProfiles(profiles) .withJars(plugins.jars.map(_.toString)) + project.foreach(builder.withProject) + if (sparkName.nonEmpty) builder.withSparkName(sparkName) if (sparkMaster.nonEmpty) From 462edb81bcd9af10755adc49c228304e4baa3652 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 12 Feb 2022 10:17:10 +0100 Subject: [PATCH 31/95] Implement test execution as part of generating documentation --- docs/documenting/index.md | 4 +- examples/weather/mapping/aggregates.yml | 12 ++ examples/weather/model/measurements.yml | 25 +++++ .../flowman/documentation/ColumnTest.scala | 5 + .../documentation/ExecuteTestCollector.scala | 26 ----- .../documentation/MappingCollector.scala | 20 +++- .../documentation/RelationCollector.scala | 25 ++++- .../documentation/TargetCollector.scala | 8 +- .../flowman/documentation/TestExecutor.scala | 104 ++++++++++++++++++ .../flowman/documentation/velocity.scala | 21 ++++ flowman-parent/pom.xml | 22 +++- .../flowman/documentation/text/project.vtl | 8 +- .../spec/mapping/AggregateMapping.scala | 2 +- .../flowman/spec/mapping/SelectMapping.scala | 4 +- .../flowman/spec/mapping/StackMapping.scala | 2 +- .../flowman/spec/target/TruncateTarget.scala | 1 + pom.xml | 18 +-- 17 files changed, 253 insertions(+), 54 deletions(-) delete mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala diff --git a/docs/documenting/index.md b/docs/documenting/index.md index be38dc2b8..be7c527b2 100644 --- a/docs/documenting/index.md +++ b/docs/documenting/index.md @@ -12,14 +12,14 @@ which is useful for providing a documentation of the data model. * ``` -## Providing Descriptions +### Providing Descriptions Although Flowman will generate many valuable documentation bits by inspecting the project, the most important entities (relations, mappings and targets) also provide the ability to manually and explicitly add documentation to them. This documentation will override any automatically inferred information. -## Generating Documentation +### Generating Documentation Generating the documentation is as easy as running [flowexec](../cli/flowexec.md) as follows: diff --git a/examples/weather/mapping/aggregates.yml b/examples/weather/mapping/aggregates.yml index a6a63b88c..f4091e1c6 100644 --- a/examples/weather/mapping/aggregates.yml +++ b/examples/weather/mapping/aggregates.yml @@ -12,3 +12,15 @@ mappings: min_temperature: "MIN(air_temperature)" max_temperature: "MAX(air_temperature)" avg_temperature: "AVG(air_temperature)" + + documentation: + description: "This mapping calculates the aggregated metrics per year and per country" + columns: + - name: country + tests: + - kind: notNull + - kind: unique + - name: min_wind_speed + description: Minimum wind speed + - name: max_wind_speed + description: Maximum wind speed diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index 348e046e4..409b36574 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -16,3 +16,28 @@ relations: schema: kind: mapping mapping: measurements_extracted + + documentation: + description: "This model contains all individual measurements" + columns: + - name: year + description: "The year of the measurement, used for partitioning the data" + tests: + - kind: notNull + - name: usaf + tests: + - kind: notNull + - name: wban + tests: + - kind: notNull + - name: date + tests: + - kind: notNull + - name: time + tests: + - kind: notNull + - name: air_temperature_qual + tests: + - kind: notNull + - kind: values + values: [0,1,2,3,4,5,6,7,8,9] diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index a1ed5e1ac..69eb53501 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -37,6 +37,7 @@ final case class ColumnTestReference( abstract class ColumnTest extends Fragment with Product with Serializable { + def name : String def result : Option[TestResult] def withResult(result:TestResult) : ColumnTest @@ -53,6 +54,7 @@ final case class NotNullColumnTest( description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { + override def name : String = "IS NOT NULL" override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): ColumnTest = { val ref = ColumnTestReference(Some(parent)) @@ -65,6 +67,7 @@ final case class UniqueColumnTest( description: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { + override def name : String = "HAS UNIQUE VALUES" override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): UniqueColumnTest = { val ref = ColumnTestReference(Some(parent)) @@ -79,6 +82,7 @@ final case class RangeColumnTest( upper:Any, result:Option[TestResult] = None ) extends ColumnTest { + override def name : String = s"IS BETWEEN $lower AND $upper" override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): RangeColumnTest = { val ref = ColumnTestReference(Some(parent)) @@ -92,6 +96,7 @@ final case class ValuesColumnTest( values: Seq[Any] = Seq(), result:Option[TestResult] = None ) extends ColumnTest { + override def name : String = s"IS IN (${values.mkString(",")})" override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): ValuesColumnTest = { val ref = ColumnTestReference(Some(parent)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala deleted file mode 100644 index 434fbfd23..000000000 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ExecuteTestCollector.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2022 Kaya Kupferschmidt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dimajix.flowman.documentation - -import com.dimajix.flowman.execution.Execution -import com.dimajix.flowman.graph.Graph -import com.dimajix.flowman.model.Project - - -class ExecuteTestCollector extends Collector { - override def collect(execution: Execution, graph:Graph, documentation:ProjectDoc) : ProjectDoc = ??? -} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index aaae4a008..d83ace35c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -18,6 +18,8 @@ package com.dimajix.flowman.documentation import scala.collection.mutable +import org.slf4j.LoggerFactory + import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.model.Mapping @@ -26,7 +28,11 @@ import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.types.StructType -class MappingCollector extends Collector { +class MappingCollector( + executeTests:Boolean = true +) extends Collector { + private val logger = LoggerFactory.getLogger(getClass) + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val mappings = mutable.Map[MappingIdentifier, MappingDoc]() val parent = documentation.reference @@ -41,6 +47,7 @@ class MappingCollector extends Collector { doc.outputs.find(_.identifier.output == output) } def genDoc(mapping:Mapping) : MappingDoc = { + logger.info(s"Collecting documentation for mapping '${mapping.identifier}'") val inputs = mapping.inputs.flatMap(in => getOutputDoc(in).map(in -> _)).toMap document(execution, parent, mapping, inputs) } @@ -82,6 +89,15 @@ class MappingCollector extends Collector { doc.copy(schema = Some(schemaDoc)) } - doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) + val result = doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) + if (executeTests) + runTests(execution, mapping, result) + else + result + } + + private def runTests(execution: Execution, mapping:Mapping, doc:MappingDoc) : MappingDoc = { + val executor = new TestExecutor(execution) + executor.executeTests(mapping, doc) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 59574abfe..dc0e430b2 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -16,23 +16,29 @@ package com.dimajix.flowman.documentation +import org.slf4j.LoggerFactory + import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.InputMapping import com.dimajix.flowman.graph.MappingRef import com.dimajix.flowman.graph.RelationRef import com.dimajix.flowman.graph.WriteRelation +import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation -class RelationCollector extends Collector { +class RelationCollector( + executeTests:Boolean = true +) extends Collector { + private val logger = LoggerFactory.getLogger(getClass) + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference val docs = graph.relations.map(t => t.relation.identifier -> document(execution, parent, t)).toMap documentation.copy(relations = docs) } - /** * Create a documentation for the relation. * @param execution @@ -40,6 +46,9 @@ class RelationCollector extends Collector { * @return */ private def document(execution:Execution, parent:Reference, node:RelationRef) : RelationDoc = { + val relation = node.relation + logger.info(s"Collecting documentation for relation '${relation.identifier}'") + val inputs = node.incoming.flatMap { case write:WriteRelation => write.input.incoming.flatMap { @@ -52,7 +61,6 @@ class RelationCollector extends Collector { case _ => Seq() } - val relation = node.relation val doc = RelationDoc( Some(parent), relation.identifier, @@ -76,6 +84,15 @@ class RelationCollector extends Collector { } val mergedSchema = desc.merge(schema) - doc.copy(schema = Some(mergedSchema)).merge(relation.documentation) + val result = doc.copy(schema = Some(mergedSchema)).merge(relation.documentation) + if (executeTests) + runTests(execution, relation, result) + else + result + } + + private def runTests(execution: Execution, relation:Relation, doc:RelationDoc) : RelationDoc = { + val executor = new TestExecutor(execution) + executor.executeTests(relation, doc) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index 4da0c41ca..f6e7be8e9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.documentation +import org.slf4j.LoggerFactory + import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.InputMapping @@ -26,6 +28,8 @@ import com.dimajix.flowman.model.Target class TargetCollector extends Collector { + private val logger = LoggerFactory.getLogger(getClass) + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference val docs = graph.targets.map(t => t.target.identifier -> document(execution, parent, t)).toMap @@ -39,6 +43,9 @@ class TargetCollector extends Collector { * @return */ private def document(execution: Execution, parent:Reference, node:TargetRef) : TargetDoc = { + val target = node.target + logger.info(s"Collecting documentation for target '${target.identifier}'") + val inputs = node.incoming.flatMap { case map: InputMapping => val mapref = MappingReference(Some(parent), map.input.name) @@ -56,7 +63,6 @@ class TargetCollector extends Collector { case _ => None } - val target = node.target val doc = TargetDoc( Some(parent), target.identifier, diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala new file mode 100644 index 000000000..bb3bf1c04 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala @@ -0,0 +1,104 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.apache.spark.sql.DataFrame +import org.slf4j.LoggerFactory + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.spi.ColumnTestExecutor + + +class TestExecutor(execution: Execution) { + private val logger = LoggerFactory.getLogger(getClass) + private val columnTestExecutors = ColumnTestExecutor.executors + + /** + * Executes all tests for a relation as defined within the documentation + * @param relation + * @param doc + * @return + */ + def executeTests(relation:Relation, doc:RelationDoc) : RelationDoc = { + val schemaDoc = doc.schema.map { schema => + if (containsTests(schema)) { + logger.info(s"Conducting tests on relation '${relation.identifier}'") + val df = relation.read(execution, doc.partitions) + runSchemaTests(df,schema) + } + else { + schema + } + } + doc.copy(schema=schemaDoc) + } + + /** + * Executes all tests for a mapping as defined within the documentation + * @param relation + * @param doc + * @return + */ + def executeTests(mapping:Mapping, doc:MappingDoc) : MappingDoc = { + val outputs = doc.outputs.map { output => + val schema = output.schema.map { schema => + if (containsTests(schema)) { + logger.info(s"Conducting tests on mapping '${mapping.identifier}'") + val df = execution.instantiate(mapping, output.name) + runSchemaTests(df,schema) + } + else { + schema + } + } + output.copy(schema=schema) + } + doc.copy(outputs=outputs) + } + + private def containsTests(doc:SchemaDoc) : Boolean = { + doc.tests.nonEmpty || containsTests(doc.columns) + } + private def containsTests(docs:Seq[ColumnDoc]) : Boolean = { + docs.exists(col => col.tests.nonEmpty || containsTests(col.children)) + } + + private def runSchemaTests(df:DataFrame, schema:SchemaDoc) : SchemaDoc = { + val columns = runColumnTests(df, schema.columns) + schema.copy(columns=columns) + } + private def runColumnTests(df:DataFrame, columns:Seq[ColumnDoc], path:String = "") : Seq[ColumnDoc] = { + columns.map(col => runColumnTests(df, col, path)) + } + private def runColumnTests(df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { + val columnPath = path + column.name + val tests = column.tests.map { test => + val result = columnTestExecutors.flatMap(_.execute(execution, df, columnPath,test)).headOption + result match { + case None => + logger.warn(s"Could not find appropriate test executor for testing column $columnPath") + test + case Some(result) => + test.withResult(result.reparent(test.reference)) + } + } + val children = runColumnTests(df, column.children, path + column.name + ".") + column.copy(children=children, tests=tests) + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 55d6a0fa9..d711068a9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -21,6 +21,26 @@ import scala.collection.JavaConverters._ import com.dimajix.flowman.model.ResourceIdentifierWrapper +final case class TestResultWrapper(result:TestResult) { + override def toString: String = result.status.toString + + def getReference() : String = result.reference.toString + def getDescription() : String = result.description.getOrElse("") + def getStatus() : String = result.status.toString +} + + +final case class ColumnTestWrapper(test:ColumnTest) { + override def toString: String = test.name + + def getReference() : String = test.reference.toString + def getName() : String = test.name + def getDescription() : String = test.description.getOrElse("") + def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull + def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") +} + + final case class ColumnDocWrapper(column:ColumnDoc) { override def toString: String = column.name @@ -33,6 +53,7 @@ final case class ColumnDocWrapper(column:ColumnDoc) { def getCatalogType() : String = column.catalogType def getDescription() : String = column.description.getOrElse("") def getColumns() : java.util.List[ColumnDocWrapper] = column.children.map(ColumnDocWrapper).asJava + def getTests() : java.util.List[ColumnTestWrapper] = column.tests.map(ColumnTestWrapper).asJava } diff --git a/flowman-parent/pom.xml b/flowman-parent/pom.xml index 0ffcc7a6f..21fd7cfc5 100644 --- a/flowman-parent/pom.xml +++ b/flowman-parent/pom.xml @@ -63,7 +63,7 @@ true org.codehaus.mojo build-helper-maven-plugin - 3.2.0 + 3.3.0 true @@ -90,7 +90,7 @@ true org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.9.0 ${maven.compiler.source} ${maven.compiler.target} @@ -100,7 +100,7 @@ true net.alchim31.maven scala-maven-plugin - 4.5.3 + 4.5.6 ${scala.version} ${scala.api_version} @@ -193,7 +193,7 @@ true org.codehaus.mojo versions-maven-plugin - 2.8.1 + 2.9.0 true @@ -240,7 +240,19 @@ true org.apache.maven.plugins maven-site-plugin - 3.9.1 + 3.10.0 + + + true + org.apache.maven.plugins + maven-project-info-reports-plugin + 3.2.1 + + + true + org.apache.maven.plugins + maven-help-plugin + 3.2.0 diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl index 80ab3c528..fb839c223 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -14,7 +14,10 @@ Mapping '${mapping}' (${mapping.reference}) #foreach($output in ${mapping.outputs}) - '${output.name}': #foreach($column in ${output.schema.columns}) - ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + #foreach($test in $column.tests) + Test: '${test.name}' => ${test.result.status} + #end #end #end @@ -36,6 +39,9 @@ Relation '${relation}' (${relation.reference}) Schema: #foreach($column in ${relation.schema.columns}) ${column.name} ${column.catalogType} #if(!$column.nullable)NOT NULL #end- ${column.description} + #foreach($test in $column.tests) + Test: '${test.name}' => ${test.result.status} + #end #end #end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala index 3fc572aef..eb3d47047 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala @@ -92,7 +92,7 @@ class AggregateMappingSpec extends MappingSpec { instanceProperties(context), MappingOutputIdentifier.parse(context.evaluate(input)), dimensions.map(context.evaluate), - context.evaluate(aggregations), + ListMap(aggregations.toSeq.map { case(k,v) => k -> context.evaluate(v) }:_*), context.evaluate(filter), if (partitions.isEmpty) 0 else context.evaluate(partitions).toInt ) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala index 005f0ddd6..915acd069 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,7 +87,7 @@ class SelectMappingSpec extends MappingSpec { SelectMapping( instanceProperties(context), MappingOutputIdentifier(context.evaluate(input)), - context.evaluate(columns).toSeq, + columns.toSeq.map { case(k,v) => k -> context.evaluate(v) }, context.evaluate(filter) ) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala index 02cef5365..fb64b58ad 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala @@ -140,7 +140,7 @@ class StackMappingSpec extends MappingSpec { MappingOutputIdentifier(context.evaluate(input)), context.evaluate(nameColumn), context.evaluate(valueColumn), - ListMap(context.evaluate(stackColumns).toSeq:_*), + ListMap(stackColumns.toSeq.map {case(k,v) => k -> context.evaluate(v) }:_*), context.evaluate(dropNulls).toBoolean, keepColumns.map(context.evaluate), dropColumns.map(context.evaluate), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala index 3529559b1..9f1feba97 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala @@ -147,6 +147,7 @@ case class TruncateTarget( case Phase.BUILD|Phase.TRUNCATE => val rel = relation.value resolvedPartitions(rel).foreach(p => linker.write(rel, p)) + case _ => } } diff --git a/pom.xml b/pom.xml index 8c8f81aac..3d8265d5f 100644 --- a/pom.xml +++ b/pom.xml @@ -600,12 +600,12 @@ org.apache.maven.plugins maven-site-plugin - 3.9.1 + 3.10.0 org.apache.maven.plugins maven-project-info-reports-plugin - 3.1.1 + 3.2.1 org.apache.maven.plugins @@ -629,7 +629,7 @@ true org.codehaus.mojo build-helper-maven-plugin - 3.2.0 + 3.3.0 true @@ -657,7 +657,7 @@ true org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.9.0 ${maven.compiler.source} ${maven.compiler.target} @@ -677,7 +677,7 @@ true net.alchim31.maven scala-maven-plugin - 4.5.4 + 4.5.6 ${scala.version} ${scala.api_version} @@ -858,7 +858,7 @@ true org.codehaus.mojo versions-maven-plugin - 2.8.1 + 2.9.0 true @@ -898,12 +898,12 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.1.1 + 3.2.1 net.alchim31.maven scala-maven-plugin - 4.5.4 + 4.5.6 -Xms64m @@ -914,7 +914,7 @@ org.scoverage scoverage-maven-plugin - 1.4.1 + 1.4.11 false From 945f6fdb1b08c9eba2f96bf79f62e4c91fd71f33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Feb 2022 09:18:16 +0000 Subject: [PATCH 32/95] Bump follow-redirects from 1.14.4 to 1.14.8 in /flowman-server-ui Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.14.4 to 1.14.8. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.14.4...v1.14.8) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- flowman-server-ui/package-lock.json | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/flowman-server-ui/package-lock.json b/flowman-server-ui/package-lock.json index a7435d279..a4cbbb424 100644 --- a/flowman-server-ui/package-lock.json +++ b/flowman-server-ui/package-lock.json @@ -7287,11 +7287,22 @@ } }, "node_modules/follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==", + "version": "1.14.8", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", + "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], "engines": { "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } } }, "node_modules/for-in": { @@ -22677,9 +22688,9 @@ } }, "follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==" + "version": "1.14.8", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", + "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==" }, "for-in": { "version": "1.0.2", From e25722ab437e96c2f7e1f80564e81f46c3d21599 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Feb 2022 09:19:52 +0000 Subject: [PATCH 33/95] Bump follow-redirects from 1.14.4 to 1.14.8 in /flowman-studio-ui Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.14.4 to 1.14.8. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.14.4...v1.14.8) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- flowman-studio-ui/package-lock.json | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/flowman-studio-ui/package-lock.json b/flowman-studio-ui/package-lock.json index b555fbcc6..da6bb130f 100644 --- a/flowman-studio-ui/package-lock.json +++ b/flowman-studio-ui/package-lock.json @@ -2375,7 +2375,6 @@ "thread-loader": "^2.1.3", "url-loader": "^2.2.0", "vue-loader": "^15.9.2", - "vue-loader-v16": "npm:vue-loader@^16.1.0", "vue-style-loader": "^4.1.2", "webpack": "^4.0.0", "webpack-bundle-analyzer": "^3.8.0", @@ -2476,7 +2475,6 @@ "merge-source-map": "^1.1.0", "postcss": "^7.0.36", "postcss-selector-parser": "^6.0.2", - "prettier": "^1.18.2", "source-map": "~0.6.1", "vue-template-es2015-compiler": "^1.9.0" }, @@ -4032,7 +4030,6 @@ "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", - "fsevents": "~2.3.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", @@ -7475,9 +7472,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==", + "version": "1.14.8", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", + "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==", "funding": [ { "type": "individual", @@ -9634,9 +9631,6 @@ "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", "dev": true, - "dependencies": { - "graceful-fs": "^4.1.6" - }, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -16162,10 +16156,8 @@ "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", "dev": true, "dependencies": { - "chokidar": "^3.4.1", "graceful-fs": "^4.1.2", - "neo-async": "^2.5.0", - "watchpack-chokidar2": "^2.0.1" + "neo-async": "^2.5.0" }, "optionalDependencies": { "chokidar": "^3.4.1", @@ -16227,7 +16219,6 @@ "anymatch": "^2.0.0", "async-each": "^1.0.1", "braces": "^2.3.2", - "fsevents": "^1.2.7", "glob-parent": "^3.1.0", "inherits": "^2.0.3", "is-binary-path": "^1.0.0", @@ -16564,7 +16555,6 @@ "anymatch": "^2.0.0", "async-each": "^1.0.1", "braces": "^2.3.2", - "fsevents": "^1.2.7", "glob-parent": "^3.1.0", "inherits": "^2.0.3", "is-binary-path": "^1.0.0", @@ -23113,9 +23103,9 @@ } }, "follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==" + "version": "1.14.8", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", + "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==" }, "for-in": { "version": "1.0.2", From f2dfcaa1937f14e25c07fd696d0e303cf6cad0e6 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 12 Feb 2022 11:38:22 +0100 Subject: [PATCH 34/95] Speed up loading by caching Jackson ObjectMapper --- .../com.dimajix.flowman.spi.PluginListener | 1 + .../dimajix/flowman/spec/ObjectMapper.scala | 91 +++++++++++-------- .../dimajix/flowman/tools/exec/Driver.scala | 4 + 3 files changed, 59 insertions(+), 37 deletions(-) create mode 100644 flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.PluginListener diff --git a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.PluginListener b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.PluginListener new file mode 100644 index 000000000..c4ccdcf7d --- /dev/null +++ b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.PluginListener @@ -0,0 +1 @@ +com.dimajix.flowman.spec.ObjectMapperPluginListener diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala index b907709fb..c2bcd71b1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala @@ -19,6 +19,7 @@ package com.dimajix.flowman.spec import com.fasterxml.jackson.databind.jsontype.NamedType import com.fasterxml.jackson.databind.{ObjectMapper => JacksonMapper} +import com.dimajix.flowman.plugin.Plugin import com.dimajix.flowman.spec.assertion.AssertionSpec import com.dimajix.flowman.spec.catalog.CatalogSpec import com.dimajix.flowman.spec.connection.ConnectionSpec @@ -35,6 +36,7 @@ import com.dimajix.flowman.spec.schema.SchemaSpec import com.dimajix.flowman.spec.storage.ParcelSpec import com.dimajix.flowman.spec.target.TargetSpec import com.dimajix.flowman.spi.ClassAnnotationScanner +import com.dimajix.flowman.spi.PluginListener import com.dimajix.flowman.util.{ObjectMapper => CoreObjectMapper} @@ -43,47 +45,62 @@ import com.dimajix.flowman.util.{ObjectMapper => CoreObjectMapper} * extensions and can directly be used for reading flowman specification files */ object ObjectMapper extends CoreObjectMapper { + private var _mapper:JacksonMapper = null + /** * Create a new Jackson ObjectMapper * @return */ - override def mapper : JacksonMapper = { - // Ensure that all extensions are loaded - ClassAnnotationScanner.load() + override def mapper : JacksonMapper = synchronized { + // Implement a stupidly simple cache + if (_mapper == null) { + // Ensure that all extensions are loaded + ClassAnnotationScanner.load() + + val stateStoreTypes = HistorySpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val catalogTypes = CatalogSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val monitorTypes = HistorySpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val relationTypes = RelationSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val mappingTypes = MappingSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val targetTypes = TargetSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val schemaTypes = SchemaSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val connectionTypes = ConnectionSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val assertionTypes = AssertionSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val measureTypes = MeasureSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val datasetTypes = DatasetSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val metricSinkTypes = MetricSinkSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val parcelTypes = ParcelSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val generatorTypes = GeneratorSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val columnTestTypes = ColumnTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val schemaTestTypes = SchemaTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val mapper = super.mapper + mapper.registerSubtypes(stateStoreTypes: _*) + mapper.registerSubtypes(catalogTypes: _*) + mapper.registerSubtypes(monitorTypes: _*) + mapper.registerSubtypes(relationTypes: _*) + mapper.registerSubtypes(mappingTypes: _*) + mapper.registerSubtypes(targetTypes: _*) + mapper.registerSubtypes(schemaTypes: _*) + mapper.registerSubtypes(connectionTypes: _*) + mapper.registerSubtypes(assertionTypes: _*) + mapper.registerSubtypes(measureTypes: _*) + mapper.registerSubtypes(datasetTypes: _*) + mapper.registerSubtypes(metricSinkTypes: _*) + mapper.registerSubtypes(parcelTypes: _*) + mapper.registerSubtypes(generatorTypes: _*) + mapper.registerSubtypes(columnTestTypes: _*) + mapper.registerSubtypes(schemaTestTypes: _*) + _mapper = mapper + } + _mapper + } - val stateStoreTypes = HistorySpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val catalogTypes = CatalogSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val monitorTypes = HistorySpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val relationTypes = RelationSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val mappingTypes = MappingSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val targetTypes = TargetSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val schemaTypes = SchemaSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val connectionTypes = ConnectionSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val assertionTypes = AssertionSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val measureTypes = MeasureSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val datasetTypes = DatasetSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val metricSinkTypes = MetricSinkSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val parcelTypes = ParcelSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val generatorTypes = GeneratorSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val columnTestTypes = ColumnTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val schemaTestTypes = SchemaTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val mapper = super.mapper - mapper.registerSubtypes(stateStoreTypes:_*) - mapper.registerSubtypes(catalogTypes:_*) - mapper.registerSubtypes(monitorTypes:_*) - mapper.registerSubtypes(relationTypes:_*) - mapper.registerSubtypes(mappingTypes:_*) - mapper.registerSubtypes(targetTypes:_*) - mapper.registerSubtypes(schemaTypes:_*) - mapper.registerSubtypes(connectionTypes:_*) - mapper.registerSubtypes(assertionTypes:_*) - mapper.registerSubtypes(measureTypes:_*) - mapper.registerSubtypes(datasetTypes:_*) - mapper.registerSubtypes(metricSinkTypes:_*) - mapper.registerSubtypes(parcelTypes:_*) - mapper.registerSubtypes(generatorTypes:_*) - mapper.registerSubtypes(columnTestTypes:_*) - mapper.registerSubtypes(schemaTestTypes:_*) - mapper + def invalidate(): Unit = synchronized { + _mapper = null } } + + +class ObjectMapperPluginListener extends PluginListener { + override def pluginLoaded(plugin: Plugin, classLoader: ClassLoader): Unit = ObjectMapper.invalidate() +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Driver.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Driver.scala index 46b8138a5..a357306a4 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Driver.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/Driver.scala @@ -16,6 +16,9 @@ package com.dimajix.flowman.tools.exec +import java.time.Duration +import java.time.Instant + import scala.util.Failure import scala.util.Success import scala.util.Try @@ -112,6 +115,7 @@ class Driver(options:Arguments) extends Tool { else { // Create Flowman Session, which also includes a Spark Session val project = loadProject(new Path(options.projectFile)) + val config = splitSettings(options.config) val environment = splitSettings(options.environment) val session = createSession( From 0833ce3a817cbffa3d5dce06a5dbf3232dc8f60e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 12 Feb 2022 17:09:15 +0100 Subject: [PATCH 35/95] Improve generating documentation --- CHANGELOG.md | 1 + docs/cookbook/data-qualioty.md | 52 ++++++++++++- docs/documenting/config.md | 23 ++++++ docs/documenting/tests.md | 76 +++++++++++++++++++ docs/testing/index.md | 8 ++ examples/weather/.gitignore | 1 + examples/weather/documentation.yml | 12 +++ examples/weather/mapping/measurements.yml | 8 +- .../flowman/documentation/ColumnDoc.scala | 1 + .../flowman/documentation/ColumnTest.scala | 1 + .../flowman/documentation/Documenter.scala | 70 ++++++++++++++++- .../documentation/MappingCollector.scala | 24 ++++-- .../flowman/documentation/MappingDoc.scala | 2 + .../flowman/documentation/ProjectDoc.scala | 1 + .../flowman/documentation/Reference.scala | 1 + .../flowman/documentation/RelationDoc.scala | 1 + .../flowman/documentation/SchemaDoc.scala | 1 + .../flowman/documentation/SchemaTest.scala | 10 ++- .../flowman/documentation/TargetDoc.scala | 2 + .../flowman/documentation/TestResult.scala | 1 + .../flowman/documentation/velocity.scala | 62 +++++++-------- .../dimajix/flowman/hadoop/FileSystem.scala | 6 +- .../com/dimajix/flowman/model/Module.scala | 4 +- .../com/dimajix/flowman/model/Project.scala | 32 ++------ .../flowman/spi/DocumenterReader.scala | 61 +++++++++++++++ .../dimajix/flowman/types/SchemaWriter.scala | 10 ++- flowman-plugins/mssqlserver/pom.xml | 2 - .../mssqlserver/src/main/resources/plugin.yml | 1 + .../com.dimajix.flowman.spi.DocumenterReader | 1 + .../flowman/documentation/text/project.vtl | 2 +- .../flowman/spec/YamlDocumenterReader.scala | 67 ++++++++++++++++ .../flowman/spec/YamlModuleReader.scala | 9 +++ .../flowman/spec/YamlNamespaceReader.scala | 9 +++ .../flowman/spec/YamlProjectReader.scala | 16 +++- .../spec/documentation/CollectorSpec.scala | 56 ++++++++++++++ .../spec/documentation/DocumenterSpec.scala | 36 +++++++++ .../spec/documentation/FileGenerator.scala | 40 ++++++++++ .../documentation/TemplateGenerator.scala | 27 ++++++- .../exec/documentation/DocumenterLoader.scala | 49 ++++++++++++ .../exec/documentation/GenerateCommand.scala | 19 +---- 40 files changed, 697 insertions(+), 108 deletions(-) create mode 100644 docs/documenting/config.md create mode 100644 examples/weather/.gitignore create mode 100644 examples/weather/documentation.yml create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/spi/DocumenterReader.scala create mode 100644 flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.DocumenterReader create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlDocumenterReader.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterSpec.scala create mode 100644 flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a885212a..e416a291c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Work on new documentation subsystem * Change default build to Spark 3.2.1 and Hadoop 3.3.1 * Add new `drop` target for removing tables +* Speed up project loading by reusing Jackson mapper # Version 0.21.1 - 2022-01-28 diff --git a/docs/cookbook/data-qualioty.md b/docs/cookbook/data-qualioty.md index 24a73182d..bf4559f9e 100644 --- a/docs/cookbook/data-qualioty.md +++ b/docs/cookbook/data-qualioty.md @@ -1,11 +1,55 @@ # Data Quality -Data quality is an important topic, which is also addressed in Flowman. The special [measure target](../spec/target/measure.md) -provides some means to collect some important metrics from data and provide the results as metrics. These in turn -can be [published to Prometheus](metrics.md) or other metric collectors. +Data quality is an important topic, which is also addressed in Flowman in multiple, complementary ways. -## Example +## Verification and Validation + +First you might want to add some [validate](../spec/target/validate.md) and [verify](../spec/target/verify.md) targets +to your job. The `validate` the target will be executed before the `CREATE` phase and is well suited for performing some tests +on the source data. If these tests fail, you may either emit a simple warning or stop the build altogether in failed +state (which is the default behaviour). + +The `verify` target will be executed in the `VERIFY` phase after the `BUILD` phase and is well suited for conducting +data quality tests after the build itself has finished. Again a failing `verify` target may either only generate a +warning, or may fail the build. + +### Example + +```yaml +targets: + validate_input: + kind: validate + mode: failFast + assertions: + assert_primary_key: + kind: sql + tests: + - query: "SELECT id,count(*) FROM source GROUP BY id HAVING count(*) > 0" + expected: [] + + assert_measurement_count: + kind: sql + tests: + - query: "SELECT COUNT(*) FROM measurements_extracted" + expected: 2 +``` + + +## Data Quality Tests as Documentation + +With the new [documentation framework](../documenting/index.md), Flowman adds the possibility not only to document +mappings and relations, but also to add test cases. These will be executed as part of the documentation (which is +generated with an independent command with [`flowexec`](../cli/flowexec.md)). + + +## Data Quality Metrics +In addition to the `validate` and `verify` targets, Flowman also offers a special [measure target](../spec/target/measure.md). +This target provides some means to collect some important metrics from data and provide the results as metrics. These +in turn can be [published to Prometheus](metrics.md) or other metric collectors. + + +### Example ```yaml targets: diff --git a/docs/documenting/config.md b/docs/documenting/config.md new file mode 100644 index 000000000..3e2482e51 --- /dev/null +++ b/docs/documenting/config.md @@ -0,0 +1,23 @@ +# Configuring the Documentation + +Flowman has a sound default for generating documentation for relations, mappings and targets. But you might want +to explicitly influence the way for what and how documentation is generated. This can be easily done by supplying +a `documentation.yml` file at the root level of your project (so it would be a sibling of the `project.yml` file). + + +## Example + +```yaml +collectors: + # Collect documentation of relations + - kind: relations + # Collect documentation of mappings + - kind: mappings + # Collect documentation of build targets + - kind: targets + +generators: + # Create an output file in the project directory + - kind: file + location: ${project.basedir}/doc +``` diff --git a/docs/documenting/tests.md b/docs/documenting/tests.md index 34ceda97e..e756c889d 100644 --- a/docs/documenting/tests.md +++ b/docs/documenting/tests.md @@ -3,3 +3,79 @@ In addition to provide pure descriptions of model entities, the documentation framework in Flowman also provides the ability to specify model properties (like unqiue values in a column, not null etc). These properties will not only be part of the documentation, they will also be verified as part of generating the documentation. + + +## Example + +```yaml +relations: + measurements: + kind: file + format: parquet + location: "$basedir/measurements/" + partitions: + - name: year + type: integer + granularity: 1 + # We prefer to use the inferred schema of the mapping that is written into the relation + schema: + kind: mapping + mapping: measurements_extracted + + documentation: + description: "This model contains all individual measurements" + columns: + - name: year + description: "The year of the measurement, used for partitioning the data" + tests: + - kind: notNull + - name: usaf + tests: + - kind: notNull + - name: wban + tests: + - kind: notNull + - name: air_temperature_qual + tests: + - kind: notNull + - kind: values + values: [0,1,2,3,4,5,6,7,8,9] +``` + +## Available Column Tests + +Flowman implements a couple of different test cases on a per column basis. + +### Not NULL + +One simple but yet important test is to check if a column does not contain any `NULL` values + +* `kind` **(mandatory)** *(string)*: `notNull` + + +### Unique Values + +Another important test is to check for unique values in a column. Note that this test will exclude `NULL` values, +so in many cases you might want to specify both `notNUll` and `unique`. + +* `kind` **(mandatory)** *(string)*: `unique` + + +### Specific Values + +In order to test if a column only contains specific values, you can use the `values` test. Note that this test will +exclude records with `NULL` values in the column, so in many cases you might want to specify both `notNUll` and `values`. + +* `kind` **(mandatory)** *(string)*: `values` +* `values` **(mandatory)** *(list:string)*: List of admissible values + + +### Range of Values + +Especially when working with numerical data, you might also want to check their range. This can be implemented by using +the `range` test. Note that this test will exclude records with `NULL` values in the column, so in many cases you might +want to specify both `notNUll` and `range`. + +* `kind` **(mandatory)** *(string)*: `range` +* `lower` **(mandatory)** *(string)*: Lower value (inclusive) +* `upper` **(mandatory)** *(string)*: Upper value (inclusive) diff --git a/docs/testing/index.md b/docs/testing/index.md index e8d542b51..45d5a4b2f 100644 --- a/docs/testing/index.md +++ b/docs/testing/index.md @@ -77,3 +77,11 @@ The easiest way to execute tests is to use the [Flowman Shell](../cli/flowshell. Flowman now also includes a `flowman-testing` library which allows one to write lightweight unittests using either Scala or Java. The library provides some simple test runner for executing tests and jobs specified as usual in YAML files. + + +## Data Quality Tests + +The testing framework above is meant for implementing unittests (i.e. self-contained tests without any dependency to +external systems like databases for additional files). If you want to assess the data quality of either the source tables +or the generated tables, you may want to have a look at [documenting with Flowman](../documenting/index.md) and +the [validation](../spec/target/validate.md) and [verify](../spec/target/verify.md) targets. diff --git a/examples/weather/.gitignore b/examples/weather/.gitignore new file mode 100644 index 000000000..0cdba66e7 --- /dev/null +++ b/examples/weather/.gitignore @@ -0,0 +1 @@ +generated-documentation diff --git a/examples/weather/documentation.yml b/examples/weather/documentation.yml new file mode 100644 index 000000000..6c6639cf8 --- /dev/null +++ b/examples/weather/documentation.yml @@ -0,0 +1,12 @@ +collectors: + # Collect documentation of relations + - kind: relations + # Collect documentation of mappings + - kind: mappings + # Collect documentation of build targets + - kind: targets + +generators: + # Create an output file in the project directory + - kind: file + location: ${project.basedir}/generated-documentation diff --git a/examples/weather/mapping/measurements.yml b/examples/weather/mapping/measurements.yml index 9d12fc69b..ca96ea6bc 100644 --- a/examples/weather/mapping/measurements.yml +++ b/examples/weather/mapping/measurements.yml @@ -11,17 +11,17 @@ mappings: kind: select input: measurements_raw columns: - usaf: "SUBSTR(raw_data,5,6)" - wban: "SUBSTR(raw_data,11,5)" + usaf: "CAST(SUBSTR(raw_data,5,6) AS INT)" + wban: "CAST(SUBSTR(raw_data,11,5) AS INT)" date: "TO_DATE(SUBSTR(raw_data,16,8), 'yyyyMMdd')" time: "SUBSTR(raw_data,24,4)" report_type: "SUBSTR(raw_data,42,5)" wind_direction: "SUBSTR(raw_data,61,3)" wind_direction_qual: "SUBSTR(raw_data,64,1)" wind_observation: "SUBSTR(raw_data,65,1)" - wind_speed: "CAST(SUBSTR(raw_data,66,4) AS FLOAT)/10" + wind_speed: "CAST(CAST(SUBSTR(raw_data,66,4) AS FLOAT)/10 AS FLOAT)" wind_speed_qual: "SUBSTR(raw_data,70,1)" - air_temperature: "CAST(SUBSTR(raw_data,88,5) AS FLOAT)/10" + air_temperature: "CAST(CAST(SUBSTR(raw_data,88,5) AS FLOAT)/10 AS FLOAT)" air_temperature_qual: "SUBSTR(raw_data,93,1)" documentation: diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index a2128b126..7b6f06e7e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -33,6 +33,7 @@ final case class ColumnReference( case None => name } } + override def kind : String = "column" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 69eb53501..540104e8e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -33,6 +33,7 @@ final case class ColumnTestReference( case None => "" } } + override def kind : String = "column_test" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala index 80615ed03..8c2c4ebda 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala @@ -16,18 +16,82 @@ package com.dimajix.flowman.documentation +import java.util.ServiceLoader + +import scala.collection.JavaConverters._ + +import org.apache.hadoop.fs.Path +import org.slf4j.LoggerFactory + import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.execution.Session import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.spi.DocumenterReader + + +object Documenter { + private lazy val loader = ServiceLoader.load(classOf[DocumenterReader]).iterator().asScala.toSeq + private lazy val defaultDocumenter = { + val collectors = Seq( + new RelationCollector(), + new MappingCollector(), + new TargetCollector() + ) + Documenter( + collectors=collectors + ) + } + + class Reader { + private val logger = LoggerFactory.getLogger(classOf[Documenter]) + private var format = "yaml" + + def default() : Documenter = defaultDocumenter + + def format(fmt:String) : Reader = { + format = fmt + this + } + + /** + * Loads a single file or a whole directory (non recursibely) + * + * @param file + * @return + */ + def file(file:File) : Prototype[Documenter] = { + if (!file.isAbsolute()) { + this.file(file.absolute) + } + else { + logger.info(s"Reading documenter from ${file.toString}") + reader.file(file) + } + } + + def string(text:String) : Prototype[Documenter] = { + reader.string(text) + } + + private def reader : DocumenterReader = { + loader.find(_.supports(format)) + .getOrElse(throw new IllegalArgumentException(s"Module format '$format' not supported'")) + } + } + + def read = new Reader +} -case class Documenter( - collectors:Seq[Collector], - generators:Seq[Generator] +final case class Documenter( + collectors:Seq[Collector] = Seq(), + generators:Seq[Generator] = Seq() ) { def execute(session:Session, job:Job, args:Map[String,Any]) : Unit = { val runner = session.runner diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index d83ace35c..25dc41763 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -22,6 +22,8 @@ import org.slf4j.LoggerFactory import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.MappingRef +import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.MappingOutputIdentifier @@ -37,23 +39,33 @@ class MappingCollector( val mappings = mutable.Map[MappingIdentifier, MappingDoc]() val parent = documentation.reference - def getMappingDoc(mapping:Mapping) : MappingDoc = { - mappings.getOrElseUpdate(mapping.identifier, genDoc(mapping)) + def getMappingDoc(node:MappingRef) : MappingDoc = { + val mapping = node.mapping + mappings.getOrElseUpdate(mapping.identifier, genDoc(node)) } def getOutputDoc(mappingOutput:MappingOutputIdentifier) : Option[MappingOutputDoc] = { val mapping = mappingOutput.mapping - val doc = mappings.getOrElseUpdate(mapping, genDoc(graph.mapping(mapping).mapping)) + val doc = mappings.getOrElseUpdate(mapping, genDoc(graph.mapping(mapping))) val output = mappingOutput.output doc.outputs.find(_.identifier.output == output) } - def genDoc(mapping:Mapping) : MappingDoc = { + def genDoc(node:MappingRef) : MappingDoc = { + val mapping = node.mapping logger.info(s"Collecting documentation for mapping '${mapping.identifier}'") + + // Collect fundamental basis information val inputs = mapping.inputs.flatMap(in => getOutputDoc(in).map(in -> _)).toMap - document(execution, parent, mapping, inputs) + val doc = document(execution, parent, mapping, inputs) + + // Add additional inputs from non-mapping entities + val incoming = node.incoming.collect { + case ReadRelation(input, _, _) => documentation.relations.get(input.relation.identifier).map(_.reference) + }.flatten + doc.copy(inputs=doc.inputs ++ incoming) } val docs = graph.mappings.map { mapping => - mapping.mapping.identifier -> getMappingDoc(mapping.mapping) + mapping.mapping.identifier -> getMappingDoc(mapping) }.toMap documentation.copy(mappings=docs) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index 48b733965..c3699de73 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -30,6 +30,7 @@ final case class MappingOutputReference( case None => name } } + override def kind : String = "mapping_output" } @@ -103,6 +104,7 @@ final case class MappingReference( case None => name } } + override def kind: String = "mapping" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala index 5195939cd..c297eeed5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -26,6 +26,7 @@ final case class ProjectReference( ) extends Reference { override def toString: String = "/project=" + name override def parent: Option[Reference] = None + override def kind : String = "reference" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala index e27e86503..c2089174e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Reference.scala @@ -18,6 +18,7 @@ package com.dimajix.flowman.documentation abstract class Reference extends Product with Serializable { + def kind : String def parent : Option[Reference] def path : Seq[Reference] = parent.toSeq.flatMap(_.path) :+ this } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index e2008c950..01dc49d1e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -31,6 +31,7 @@ final case class RelationReference( case None => name } } + override def kind : String = "relation" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index 5fbf5d2d5..11c30fbaf 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -35,6 +35,7 @@ final case class SchemaReference( case None => "schema" } } + override def kind : String = "schema" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala index 71e2ae596..dd65cde5e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -24,7 +24,15 @@ import com.dimajix.flowman.spi.SchemaTestExecutor final case class SchemaTestReference( override val parent:Option[SchemaReference] -) extends Reference +) extends Reference { + override def toString: String = { + parent match { + case Some(ref) => ref.toString + "/test" + case None => "" + } + } + override def kind : String = "schema_test" +} sealed abstract class SchemaTest extends Fragment with Product with Serializable { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala index ff986ff34..7cbfb3c85 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -31,6 +31,7 @@ final case class TargetPhaseReference( case None => phase.upper } } + override def kind : String = "target_phase" } @@ -59,6 +60,7 @@ final case class TargetReference( case None => name } } + override def kind: String = "target" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala index 8057d959c..70ba31ac8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala @@ -34,6 +34,7 @@ final case class TestResultReference( case None => "" } } + override def kind : String = "test_result" } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index d711068a9..ef6d1f41b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -21,120 +21,116 @@ import scala.collection.JavaConverters._ import com.dimajix.flowman.model.ResourceIdentifierWrapper -final case class TestResultWrapper(result:TestResult) { +final case class ReferenceWrapper(reference:Reference) { + override def toString: String = reference.toString + + def getParent() : ReferenceWrapper = reference.parent.map(ReferenceWrapper).orNull + def getKind() : String = reference.kind +} + + +class FragmentWrapper(fragment:Fragment) { + def getReference() : ReferenceWrapper = ReferenceWrapper(fragment.reference) + def getParent() : ReferenceWrapper = fragment.parent.map(ReferenceWrapper).orNull + def getDescription() : String = fragment.description.getOrElse("") +} + + +final case class TestResultWrapper(result:TestResult) extends FragmentWrapper(result) { override def toString: String = result.status.toString - def getReference() : String = result.reference.toString - def getDescription() : String = result.description.getOrElse("") def getStatus() : String = result.status.toString } -final case class ColumnTestWrapper(test:ColumnTest) { +final case class ColumnTestWrapper(test:ColumnTest) extends FragmentWrapper(test) { override def toString: String = test.name - def getReference() : String = test.reference.toString def getName() : String = test.name - def getDescription() : String = test.description.getOrElse("") def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") } -final case class ColumnDocWrapper(column:ColumnDoc) { +final case class ColumnDocWrapper(column:ColumnDoc) extends FragmentWrapper(column) { override def toString: String = column.name - def getReference() : String = column.reference.toString def getName() : String = column.name def getNullable() : Boolean = column.nullable def getType() : String = column.typeName def getSqlType() : String = column.sqlType def getSparkType() : String = column.sparkType def getCatalogType() : String = column.catalogType - def getDescription() : String = column.description.getOrElse("") def getColumns() : java.util.List[ColumnDocWrapper] = column.children.map(ColumnDocWrapper).asJava def getTests() : java.util.List[ColumnTestWrapper] = column.tests.map(ColumnTestWrapper).asJava } -final case class SchemaDocWrapper(schema:SchemaDoc) { - def getReference() : String = schema.reference.toString - def getDescription() : String = schema.description.getOrElse("") +final case class SchemaDocWrapper(schema:SchemaDoc) extends FragmentWrapper(schema) { def getColumns() : java.util.List[ColumnDocWrapper] = schema.columns.map(ColumnDocWrapper).asJava } -final case class MappingOutputDocWrapper(output:MappingOutputDoc) { +final case class MappingOutputDocWrapper(output:MappingOutputDoc) extends FragmentWrapper(output) { override def toString: String = output.identifier.toString - def getReference() : String = output.reference.toString def getIdentifier() : String = output.identifier.toString def getProject() : String = output.identifier.project.getOrElse("") def getName() : String = output.identifier.output def getMapping() : String = output.identifier.name def getOutput() : String = output.identifier.output - def getDescription() : String = output.description.getOrElse("") def getSchema() : SchemaDocWrapper = output.schema.map(SchemaDocWrapper).orNull } -final case class MappingDocWrapper(mapping:MappingDoc) { +final case class MappingDocWrapper(mapping:MappingDoc) extends FragmentWrapper(mapping) { override def toString: String = mapping.identifier.toString - def getReference() : String = mapping.reference.toString def getIdentifier() : String = mapping.identifier.toString def getProject() : String = mapping.identifier.project.getOrElse("") def getName() : String = mapping.identifier.name - def getDescription() : String = mapping.description.getOrElse("") - def getInputs() : java.util.List[String] = mapping.inputs.map(_.toString).asJava + def getInputs() : java.util.List[ReferenceWrapper] = mapping.inputs.map(ReferenceWrapper).asJava def getOutputs() : java.util.List[MappingOutputDocWrapper] = mapping.outputs.map(MappingOutputDocWrapper).asJava } -final case class RelationDocWrapper(relation:RelationDoc) { +final case class RelationDocWrapper(relation:RelationDoc) extends FragmentWrapper(relation) { override def toString: String = relation.identifier.toString - def getReference() : String = relation.reference.toString def getIdentifier() : String = relation.identifier.toString def getProject() : String = relation.identifier.project.getOrElse("") def getName() : String = relation.identifier.name - def getDescription() : String = relation.description.getOrElse("") def getSchema() : SchemaDocWrapper = relation.schema.map(SchemaDocWrapper).orNull - def getInputs() : java.util.List[String] = relation.inputs.map(_.toString).asJava + def getInputs() : java.util.List[ReferenceWrapper] = relation.inputs.map(ReferenceWrapper).asJava def getResources() : java.util.List[ResourceIdentifierWrapper] = relation.provides.map(ResourceIdentifierWrapper).asJava } -final case class TargetPhaseDocWrapper(phase:TargetPhaseDoc) { +final case class TargetPhaseDocWrapper(phase:TargetPhaseDoc) extends FragmentWrapper(phase) { override def toString: String = phase.phase.upper - def getReference() : String = phase.reference.toString def getName() : String = phase.phase.upper - def getDescription() : String = phase.description.getOrElse("") } -final case class TargetDocWrapper(target:TargetDoc) { +final case class TargetDocWrapper(target:TargetDoc) extends FragmentWrapper(target) { override def toString: String = target.identifier.toString - def getReference() : String = target.reference.toString def getIdentifier() : String = target.identifier.toString def getProject() : String = target.identifier.project.getOrElse("") def getName() : String = target.identifier.name - def getDescription() : String = target.description.getOrElse("") def getPhases() : java.util.List[TargetPhaseDocWrapper] = target.phases.map(TargetPhaseDocWrapper).asJava - def getOutputs() : java.util.List[String] = target.outputs.map(_.toString).asJava - def getInputs() : java.util.List[String] = target.inputs.map(_.toString).asJava + def getOutputs() : java.util.List[ReferenceWrapper] = target.outputs.map(ReferenceWrapper).asJava + def getInputs() : java.util.List[ReferenceWrapper] = target.inputs.map(ReferenceWrapper).asJava } -final case class ProjectDocWrapper(project:ProjectDoc) { +final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(project) { override def toString: String = project.name def getName() : String = project.name def getVersion() : String = project.version.getOrElse("") - def getDescription() : String = project.description.getOrElse("") def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.values.map(MappingDocWrapper).toSeq.asJava def getRelations() : java.util.List[RelationDocWrapper] = project.relations.values.map(RelationDocWrapper).toSeq.asJava diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/hadoop/FileSystem.scala b/flowman-core/src/main/scala/com/dimajix/flowman/hadoop/FileSystem.scala index 845c219b0..5eb818fe7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/hadoop/FileSystem.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/hadoop/FileSystem.scala @@ -19,6 +19,7 @@ package com.dimajix.flowman.hadoop import java.net.URI import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.LocalFileSystem import org.apache.hadoop.fs.Path @@ -29,7 +30,10 @@ import org.apache.hadoop.fs.Path case class FileSystem(conf:Configuration) { private val localFs = org.apache.hadoop.fs.FileSystem.getLocal(conf) - def file(path:Path) : File = File(path.getFileSystem(conf), path) + def file(path:Path) : File = { + val fs = path.getFileSystem(conf) + File(fs, path) + } def file(path:String) : File = file(new Path(path)) def file(path:URI) : File = file(new Path(path)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Module.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Module.scala index f65298559..b4ace46b7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Module.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Module.scala @@ -70,10 +70,10 @@ object Module { } private def readFile(file:File) : Module = { - if (file.isDirectory) { + if (file.isDirectory()) { logger.info(s"Reading all module files in directory ${file.toString}") file.list() - .filter(_.isFile) + .filter(_.isFile()) .map(f => loadFile(f)) .foldLeft(Module())((l,r) => l.merge(r)) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala index 95080826f..bb921a54c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Project.scala @@ -53,10 +53,12 @@ object Project { */ def file(file: File): Project = { if (!file.isAbsolute()) { - readFile(file.absolute) + this.file(file.absolute) } else { - readFile(file) + logger.info(s"Reading project from $file") + val spec = reader.file(file) + loadModules(spec, spec.basedir.getOrElse(file)) } } @@ -70,17 +72,9 @@ object Project { if (!file.isAbsolute()) { manifest(file.absolute) } - else if (file.isDirectory) { - logger.info(s"Reading project manifest in directory $file") - manifest(file / "project.yml") - } else { logger.info(s"Reading project manifest from $file") - val project = reader.file(file) - project.copy( - filename = Some(file.absolute), - basedir = Some(file.absolute.parent) - ) + reader.file(file) } } @@ -88,22 +82,6 @@ object Project { reader.string(text) } - private def readFile(file: File): Project = { - if (file.isDirectory) { - logger.info(s"Reading project in directory $file") - this.file(file / "project.yml") - } - else { - logger.info(s"Reading project from $file") - val spec = reader.file(file) - val project = loadModules(spec, file.parent) - project.copy( - filename = Some(file.absolute), - basedir = Some(file.absolute.parent) - ) - } - } - private def loadModules(project: Project, directory: File): Project = { val module = project.modules .map(f => Module.read.file(directory / f)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/DocumenterReader.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/DocumenterReader.scala new file mode 100644 index 000000000..94ff08658 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/DocumenterReader.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spi + +import java.io.IOException + +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.hadoop.File +import com.dimajix.flowman.model.Prototype + + +abstract class DocumenterReader { + /** + * Returns the human readable name of the documenter file format + * @return + */ + def name: String + + /** + * Returns the internally used short name of the documenter file format + * @return + */ + def format: String + + /** + * Returns true if a given format is supported by this reader + * @param format + * @return + */ + def supports(format: String): Boolean = this.format == format + + /** + * Loads a [[Documenter]] from the given file + * @param file + * @return + */ + @throws[IOException] + def file(file: File): Prototype[Documenter] + + /** + * Loads a [[Documenter]] from the given String + * @param file + * @return + */ + @throws[IOException] + def string(text: String): Prototype[Documenter] +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaWriter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaWriter.scala index 3f4fa72d0..f21a98a63 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaWriter.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaWriter.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,8 +59,12 @@ class SchemaWriter(fields:Seq[Field]) { // Manually convert string to UTF-8 and use write, since writeUTF apparently would write a BOM val bytes = Charset.forName("UTF-8").encode(schema) val output = file.create(true) - output.write(bytes.array(), bytes.arrayOffset(), bytes.limit()) - output.close() + try { + output.write(bytes.array(), bytes.arrayOffset(), bytes.limit()) + } + finally { + output.close() + } } private var format: String = "" diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index b57209144..a82b5e09b 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -140,5 +140,3 @@ - - diff --git a/flowman-plugins/mssqlserver/src/main/resources/plugin.yml b/flowman-plugins/mssqlserver/src/main/resources/plugin.yml index 02c34fb9e..da571a151 100644 --- a/flowman-plugins/mssqlserver/src/main/resources/plugin.yml +++ b/flowman-plugins/mssqlserver/src/main/resources/plugin.yml @@ -5,3 +5,4 @@ isolation: false jars: - ${plugin.jar} - mssql-jdbc-${mssqlserver-java-client.version}.jar + - spark-mssql-connector${spark-mssql-connector.suffix}-${spark-mssql-connector.version}.jar diff --git a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.DocumenterReader b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.DocumenterReader new file mode 100644 index 000000000..efbfcd79b --- /dev/null +++ b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.DocumenterReader @@ -0,0 +1 @@ +com.dimajix.flowman.spec.YamlDocumenterReader diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl index fb839c223..edcbd7b40 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/project.vtl @@ -8,7 +8,7 @@ Mapping '${mapping}' (${mapping.reference}) Description: ${mapping.description} Inputs: #foreach($input in ${mapping.inputs}) - - ${input} + - ${input.kind}: ${input} #end Outputs: #foreach($output in ${mapping.outputs}) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlDocumenterReader.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlDocumenterReader.scala new file mode 100644 index 000000000..a74e1e108 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlDocumenterReader.scala @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec + +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.hadoop.File +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.spec.documentation.DocumenterSpec +import com.dimajix.flowman.spi.DocumenterReader + + +class YamlDocumenterReader extends DocumenterReader { + /** + * Returns the human readable name of the documenter file format + * + * @return + */ + override def name: String = "yaml documenter settings reader" + + /** + * Returns the internally used short name of the documenter file format + * + * @return + */ + override def format: String = "yaml" + + override def supports(format: String): Boolean = format == "yaml" || format == "yml" + + /** + * Loads a [[Documenter]] from the given file + * + * @param file + * @return + */ + override def file(file: File): Prototype[Documenter] = { + if (file.isDirectory()) { + this.file(file / "documentation.yml") + } + else { + ObjectMapper.read[DocumenterSpec](file) + } + } + + /** + * Loads a [[Documenter]] from the given String + * + * @param file + * @return + */ + override def string(text: String): Prototype[Documenter] = { + ObjectMapper.parse[DocumenterSpec](text) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlModuleReader.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlModuleReader.scala index 2c1c26824..31ab6fd16 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlModuleReader.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlModuleReader.scala @@ -19,6 +19,9 @@ package com.dimajix.flowman.spec import java.io.IOException import java.io.InputStream +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException + import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.model.Module import com.dimajix.flowman.spi.ModuleReader @@ -39,16 +42,22 @@ class YamlModuleReader extends ModuleReader { * @return */ @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def file(file:File) : Module = { ObjectMapper.read[ModuleSpec](file).instantiate() } @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def stream(stream:InputStream) : Module = { ObjectMapper.read[ModuleSpec](stream).instantiate() } @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def string(text:String) : Module = { ObjectMapper.parse[ModuleSpec](text).instantiate() } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlNamespaceReader.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlNamespaceReader.scala index ea11fbe95..3b9092706 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlNamespaceReader.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlNamespaceReader.scala @@ -20,6 +20,9 @@ import java.io.File import java.io.IOException import java.io.InputStream +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException + import com.dimajix.flowman.model.Namespace import com.dimajix.flowman.spi.NamespaceReader @@ -39,16 +42,22 @@ class YamlNamespaceReader extends NamespaceReader { * @return */ @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def file(file:File) : Namespace = { ObjectMapper.read[NamespaceSpec](file).instantiate() } @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def stream(stream:InputStream) : Namespace = { ObjectMapper.read[NamespaceSpec](stream).instantiate() } @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def string(text:String) : Namespace = { ObjectMapper.parse[NamespaceSpec](text).instantiate() } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlProjectReader.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlProjectReader.scala index 9ccab626e..3e729f9cc 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlProjectReader.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/YamlProjectReader.scala @@ -18,13 +18,15 @@ package com.dimajix.flowman.spec import java.io.IOException +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException + import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.model.Project import com.dimajix.flowman.spi.ProjectReader class YamlProjectReader extends ProjectReader { - override def name: String = "yaml project reader" override def format: String = "yaml" @@ -38,11 +40,21 @@ class YamlProjectReader extends ProjectReader { * @return */ @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def file(file:File) : Project = { - ObjectMapper.read[ProjectSpec](file).instantiate() + if (file.isDirectory()) { + this.file(file / "project.yml") + } + else { + val prj = ObjectMapper.read[ProjectSpec](file).instantiate() + prj.copy(basedir = Some(file.parent.absolute), filename = Some(file)) + } } @throws[IOException] + @throws[JsonProcessingException] + @throws[JsonMappingException] override def string(text:String) : Project = { ObjectMapper.parse[ProjectSpec](text).instantiate() } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala new file mode 100644 index 000000000..195fe9ba2 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +import com.dimajix.flowman.documentation.Collector +import com.dimajix.flowman.documentation.MappingCollector +import com.dimajix.flowman.documentation.RelationCollector +import com.dimajix.flowman.documentation.TargetCollector +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.spec.Spec + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") +@JsonSubTypes(value = Array( + new JsonSubTypes.Type(name = "mappings", value = classOf[MappingCollectorSpec]), + new JsonSubTypes.Type(name = "relations", value = classOf[RelationCollectorSpec]), + new JsonSubTypes.Type(name = "targets", value = classOf[TargetCollectorSpec]) +)) +abstract class CollectorSpec extends Spec[Collector] { + override def instantiate(context: Context): Collector +} + +final class MappingCollectorSpec extends CollectorSpec { + override def instantiate(context: Context): MappingCollector = { + new MappingCollector() + } +} + +final class RelationCollectorSpec extends CollectorSpec { + override def instantiate(context: Context): RelationCollector = { + new RelationCollector() + } +} + +final class TargetCollectorSpec extends CollectorSpec { + override def instantiate(context: Context): TargetCollector = { + new TargetCollector() + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterSpec.scala new file mode 100644 index 000000000..391e1c026 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterSpec.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.spec.Spec + + +final class DocumenterSpec extends Spec[Documenter] { + @JsonProperty(value="collectors") private var collectors: Seq[CollectorSpec] = Seq() + @JsonProperty(value="generators") private var generators: Seq[GeneratorSpec] = Seq() + + def instantiate(context:Context): Documenter = { + Documenter( + collectors.map(_.instantiate(context)), + generators.map(_.instantiate(context)) + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala index 76e5aeb92..a086fe0fa 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -15,6 +15,7 @@ */ package com.dimajix.flowman.spec.documentation + import java.net.URL import java.nio.charset.Charset @@ -24,7 +25,10 @@ import org.apache.hadoop.fs.Path import org.slf4j.LoggerFactory import com.dimajix.flowman.documentation.Generator +import com.dimajix.flowman.documentation.ProjectDoc import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.hadoop.File object FileGenerator { @@ -38,6 +42,42 @@ case class FileGenerator( template:URL = FileGenerator.defaultTemplate ) extends TemplateGenerator(template) { private val logger = LoggerFactory.getLogger(classOf[FileGenerator]) + + override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + val fs = execution.fs + + val uri = location.toUri + val outputDir = if (uri.getAuthority == null && uri.getScheme == null) + fs.local(location) + else + fs.file(location) + + // Cleanup any existing output directory + if (outputDir.isDirectory()) { + outputDir.list().foreach(_.delete(true)) + } + else if (outputDir.isFile()) { + outputDir.isFile() + } + outputDir.mkdirs() + + val projectDoc = renderProject(context, documentation) + writeFile(outputDir / "project.txt", projectDoc) + } + + private def writeFile(file:File, content:String) : Unit = { + logger.info(s"Writing documentation file '${file.toString}'") + val out = file.create(true) + try { + // Manually convert string to UTF-8 and use write, since writeUTF apparently would write a BOM + val bytes = Charset.forName("UTF-8").encode(content) + val output = file.create(true) + out.write(bytes.array(), bytes.arrayOffset(), bytes.limit()) + } + finally { + out.close() + } + } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala index 679157a9b..a2eb9b181 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala @@ -22,8 +22,14 @@ import java.nio.charset.Charset import com.google.common.io.Resources import com.dimajix.flowman.documentation.BaseGenerator +import com.dimajix.flowman.documentation.MappingDoc +import com.dimajix.flowman.documentation.MappingDocWrapper import com.dimajix.flowman.documentation.ProjectDoc import com.dimajix.flowman.documentation.ProjectDocWrapper +import com.dimajix.flowman.documentation.RelationDoc +import com.dimajix.flowman.documentation.RelationDocWrapper +import com.dimajix.flowman.documentation.TargetDoc +import com.dimajix.flowman.documentation.TargetDocWrapper import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution @@ -31,13 +37,26 @@ import com.dimajix.flowman.execution.Execution abstract class TemplateGenerator( template:URL ) extends BaseGenerator { - override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit + + protected def renderProject(context:Context, documentation: ProjectDoc) : String = { val temp = loadResource("project.vtl") - val result = context.evaluate(temp, Map("project" -> ProjectDocWrapper(documentation))) - println(result) + context.evaluate(temp, Map("project" -> ProjectDocWrapper(documentation))) + } + protected def renderRelation(context:Context, documentation: RelationDoc) : String = { + val temp = loadResource("relation.vtl") + context.evaluate(temp, Map("relation" -> RelationDocWrapper(documentation))) + } + protected def renderMapping(context:Context, documentation: MappingDoc) : String = { + val temp = loadResource("mapping.vtl") + context.evaluate(temp, Map("mapping" -> MappingDocWrapper(documentation))) + } + protected def renderTarget(context:Context, documentation: TargetDoc) : String = { + val temp = loadResource("target.vtl") + context.evaluate(temp, Map("target" -> TargetDocWrapper(documentation))) } - private def loadResource(name: String): String = { + protected def loadResource(name: String): String = { val path = template.getPath val url = if (path.endsWith("/")) diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala new file mode 100644 index 000000000..027d0fab2 --- /dev/null +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.tools.exec.documentation + +import org.apache.hadoop.fs.Path + +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.spec.documentation.FileGenerator + + +object DocumenterLoader { + def load(project:Project, context:Context) : Documenter = { + project.basedir.flatMap { basedir => + val docpath = basedir / "documentation.yml" + if (docpath.isFile()) { + val file = Documenter.read.file(docpath) + Some(file.instantiate(context)) + } + else { + Some(defaultDocumenter((basedir / "doc").path)) + } + }.getOrElse { + defaultDocumenter(new Path("/tmp/flowman/generated-documentation")) + } + } + + private def defaultDocumenter(outputDir:Path) : Documenter = { + val generators = Seq( + new FileGenerator(outputDir) + ) + Documenter.read.default().copy(generators=generators) + } +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala index f50618168..85fece935 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala @@ -34,6 +34,7 @@ import com.dimajix.flowman.documentation.TargetCollector import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Session import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.JobIdentifier import com.dimajix.flowman.model.Project @@ -60,24 +61,12 @@ class GenerateCommand extends Command { logger.error(s"Error instantiating job '$job': ${reasons(e)}") Status.FAILED case Success(job) => - generateDoc(session, job, job.arguments(args)) + generateDoc(session, project, job, job.arguments(args)) } } - private def generateDoc(session: Session, job:Job, args:Map[String,Any]) : Status = { - val collectors = Seq( - new RelationCollector(), - new MappingCollector(), - new TargetCollector() - ) - val generators = Seq( - new FileGenerator(new Path("/tmp/flowman/doc")) - ) - val documenter = Documenter( - collectors, - generators - ) - + private def generateDoc(session: Session, project:Project, job:Job, args:Map[String,Any]) : Status = { + val documenter = DocumenterLoader.load(project, job.context) try { documenter.execute(session, job, args) Status.SUCCESS From 1cd4e2effdff1d06601e2962714481052a4e08ce Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 12 Feb 2022 19:23:42 +0100 Subject: [PATCH 36/95] Improve describe for partitioned relations --- .../documentation/RelationCollector.scala | 20 +++++++++++++++++-- .../com/dimajix/flowman/model/Relation.scala | 10 +++++----- .../flowman/spec/relation/KafkaRelation.scala | 2 +- .../spec/dataset/RelationDataset.scala | 2 +- .../spec/documentation/FileGenerator.scala | 1 - .../spec/mapping/ReadRelationMapping.scala | 2 +- .../flowman/spec/relation/JdbcRelation.scala | 2 +- .../flowman/spec/relation/MockRelation.scala | 2 +- .../flowman/spec/relation/NullRelation.scala | 2 +- .../spec/relation/ValuesRelation.scala | 2 +- .../spec/dataset/RelationDatasetTest.scala | 2 +- .../exec/documentation/DocumenterLoader.scala | 2 +- 12 files changed, 32 insertions(+), 17 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index dc0e430b2..448f4fe01 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -22,10 +22,12 @@ import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.InputMapping import com.dimajix.flowman.graph.MappingRef +import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.graph.RelationRef import com.dimajix.flowman.graph.WriteRelation import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.types.FieldValue class RelationCollector( @@ -60,6 +62,20 @@ class RelationCollector( } case _ => Seq() } + val inputPartitions = node.outgoing.flatMap { + case read:ReadRelation => + logger.debug(s"read partition ${relation.identifier}: ${read.input.relation.identifier} ${read.partitions}") + Some(read.partitions) + case _ => None + } + val outputPartitions = node.incoming.flatMap { + case write:WriteRelation => + logger.debug(s"write partition ${relation.identifier}: ${write.output.relation.identifier} ${write.partition}") + Some(write.partition) + case _ => None + } + + val partitions = (inputPartitions ++ outputPartitions).foldLeft(Map.empty[String,FieldValue])((a,b) => a ++ b) val doc = RelationDoc( Some(parent), @@ -68,11 +84,11 @@ class RelationCollector( None, inputs, relation.provides.toSeq, - Map() + partitions ) val ref = doc.reference - val desc = SchemaDoc.ofStruct(ref, relation.describe(execution)) + val desc = SchemaDoc.ofStruct(ref, relation.describe(execution, partitions)) val schema = relation.schema.map { schema => val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) SchemaDoc( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala index 07c185fc0..f7a8bedb8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Relation.scala @@ -158,7 +158,7 @@ trait Relation extends Instance { * @param execution * @return */ - def describe(execution:Execution) : StructType + def describe(execution:Execution, partitions:Map[String,FieldValue] = Map()) : StructType /** * Reads data from the relation, possibly from specific partitions @@ -320,15 +320,15 @@ abstract class BaseRelation extends AbstractInstance with Relation { * @param execution * @return */ - override def describe(execution:Execution) : StructType = { - val partitions = SetIgnoreCase(this.partitions.map(_.name)) - val result = if (!fields.forall(f => partitions.contains(f.name))) { + override def describe(execution:Execution, partitions:Map[String,FieldValue] = Map()) : StructType = { + val partitionNames = SetIgnoreCase(this.partitions.map(_.name)) + val result = if (!fields.forall(f => partitionNames.contains(f.name))) { // Use given fields if relation contains valid list of fields in addition to the partition columns StructType(fields) } else { // Otherwise let Spark infer the schema - val df = read(execution) + val df = read(execution, partitions) StructType.of(df.schema) } diff --git a/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala b/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala index 7e29e6f34..ce443d7f7 100644 --- a/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala +++ b/flowman-plugins/kafka/src/main/scala/com/dimajix/flowman/spec/relation/KafkaRelation.scala @@ -121,7 +121,7 @@ case class KafkaRelation( * @param execution * @return */ - override def describe(execution: Execution): types.StructType = { + override def describe(execution: Execution, partitions:Map[String,FieldValue] = Map()): types.StructType = { val result = types.StructType(fields) applyDocumentation(result) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala index 783461031..786ffc11f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala @@ -120,7 +120,7 @@ case class RelationDataset( */ override def describe(execution:Execution) : Option[StructType] = { val instance = relation.value - Some(instance.describe(execution)) + Some(instance.describe(execution, partition)) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala index a086fe0fa..316d5ee19 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -71,7 +71,6 @@ case class FileGenerator( try { // Manually convert string to UTF-8 and use write, since writeUTF apparently would write a BOM val bytes = Charset.forName("UTF-8").encode(content) - val output = file.create(true) out.write(bytes.array(), bytes.arrayOffset(), bytes.limit()) } finally { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala index 23b43d6b3..471a94e6f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala @@ -111,7 +111,7 @@ case class ReadRelationMapping( } else { val relation = this.relation.value - relation.describe(execution) + relation.describe(execution, partitions) } // Apply documentation diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 674d20912..10d43c6a5 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -147,7 +147,7 @@ case class JdbcRelation( * @param execution * @return */ - override def describe(execution:Execution) : FlowmanStructType = { + override def describe(execution:Execution, partitions:Map[String,FieldValue] = Map()) : FlowmanStructType = { val result = if (schema.nonEmpty) { FlowmanStructType(fields) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala index f5387043e..82caf628e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/MockRelation.scala @@ -225,7 +225,7 @@ case class MockRelation( * @param execution * @return */ - override def describe(execution: Execution): types.StructType = { + override def describe(execution: Execution, partitions:Map[String,FieldValue] = Map()): types.StructType = { val result = types.StructType(mocked.fields) applyDocumentation(result) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala index 9ef2acebe..80ab0a4e8 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/NullRelation.scala @@ -166,7 +166,7 @@ case class NullRelation( * @param execution * @return */ - override def describe(execution:Execution) : types.StructType = { + override def describe(execution:Execution, partitions:Map[String,FieldValue] = Map()) : types.StructType = { val result = types.StructType(fields) applyDocumentation(result) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala index 9ad8f5581..e4dd90758 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/ValuesRelation.scala @@ -213,7 +213,7 @@ case class ValuesRelation( * @param execution * @return */ - override def describe(execution: Execution): types.StructType = { + override def describe(execution: Execution, partitions:Map[String,FieldValue] = Map()): types.StructType = { val result = types.StructType(effectiveSchema.fields) applyDocumentation(result) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala index fa6f9f89f..757129f16 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala @@ -108,7 +108,7 @@ class RelationDatasetTest extends AnyFlatSpec with Matchers with MockFactory wit (relation.write _).expects(executor,spark.emptyDataFrame,*,OutputMode.APPEND).returns(Unit) dataset.write(executor, spark.emptyDataFrame, OutputMode.APPEND) - (relation.describe _).expects(executor).returns(new StructType()) + (relation.describe _).expects(executor, *).returns(new StructType()) dataset.describe(executor) should be (Some(new StructType())) } } diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala index 027d0fab2..2796b1cac 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala @@ -33,7 +33,7 @@ object DocumenterLoader { Some(file.instantiate(context)) } else { - Some(defaultDocumenter((basedir / "doc").path)) + Some(defaultDocumenter((basedir / "generated-documentation").path)) } }.getOrElse { defaultDocumenter(new Path("/tmp/flowman/generated-documentation")) From b0b0026b1b2cdc55e88dcaf7135a8588ac63a629 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 14 Feb 2022 07:42:37 +0100 Subject: [PATCH 37/95] Add partition parameter to describe command --- .../flowman/tools/exec/model/DescribeCommand.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala index 0d08cbb2f..ac5119892 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala @@ -22,6 +22,7 @@ import org.kohsuke.args4j.Argument import org.kohsuke.args4j.Option import org.slf4j.LoggerFactory +import com.dimajix.flowman.common.ParserUtils import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.NoSuchRelationException import com.dimajix.flowman.execution.Session @@ -29,6 +30,7 @@ import com.dimajix.flowman.execution.Status import com.dimajix.flowman.model.Project import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.tools.exec.Command +import com.dimajix.flowman.types.SingleValue class DescribeCommand extends Command { @@ -38,19 +40,22 @@ class DescribeCommand extends Command { var useSpark: Boolean = false @Argument(usage = "specifies the relation to describe", metaVar = "", required = true) var relation: String = "" + @Option(name="-p", aliases=Array("--partition"), usage = "specify partition to work on, as partition1=value1,partition2=value2") + var partition: String = "" override def execute(session: Session, project: Project, context:Context) : Status = { try { val identifier = RelationIdentifier(this.relation) val relation = context.getRelation(identifier) + val partition = ParserUtils.parseDelimitedKeyValues(this.partition).map { case(k,v) => (k,SingleValue(v)) } if (useSpark) { - val df = relation.read(session.execution, Map()) + val df = relation.read(session.execution, partition) df.printSchema() } else { val execution = session.execution - val schema = relation.describe(execution) + val schema = relation.describe(execution, partition) schema.printTree() } Status.SUCCESS From 9d99340d5f27a3ccdfac348e2c2408b723364266 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 14 Feb 2022 12:23:55 +0100 Subject: [PATCH 38/95] Add unittests for documentation framework --- .../documentation/MappingCollector.scala | 35 +++-- .../flowman/documentation/MappingDoc.scala | 8 + .../flowman/documentation/ProjectDoc.scala | 2 +- .../documentation/RelationCollector.scala | 28 +++- .../flowman/documentation/RelationDoc.scala | 16 +- .../documentation/TargetCollector.scala | 6 +- .../flowman/documentation/TestExecutor.scala | 67 +++++++-- .../flowman/documentation/TestResult.scala | 1 + .../com/dimajix/flowman/graph/nodes.scala | 12 ++ .../documentation/RelationCollectorTest.scala | 142 ++++++++++++++++++ .../exec/documentation/GenerateCommand.scala | 10 +- 11 files changed, 280 insertions(+), 47 deletions(-) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index 25dc41763..a45fb1c65 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -17,9 +17,11 @@ package com.dimajix.flowman.documentation import scala.collection.mutable +import scala.util.control.NonFatal import org.slf4j.LoggerFactory +import com.dimajix.common.ExceptionUtils.reasons import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.MappingRef @@ -80,7 +82,6 @@ class MappingCollector( */ private def document(execution: Execution, parent:Reference, mapping:Mapping, inputs:Map[MappingOutputIdentifier,MappingOutputDoc]) : MappingDoc = { val inputSchemas = inputs.map(kv => kv._1 -> kv._2.schema.map(_.toStruct).getOrElse(StructType(Seq()))) - val schemas = mapping.describe(execution, inputSchemas) val doc = MappingDoc( Some(parent), mapping.identifier, @@ -90,15 +91,29 @@ class MappingCollector( ) val ref = doc.reference - val outputs = schemas.map { case(output,schema) => - val doc = MappingOutputDoc( - Some(ref), - MappingOutputIdentifier(mapping.identifier, output), - None, - None - ) - val schemaDoc = SchemaDoc.ofStruct(doc.reference, schema) - doc.copy(schema = Some(schemaDoc)) + val outputs = try { + val schemas = mapping.describe(execution, inputSchemas) + schemas.map { case(output,schema) => + val doc = MappingOutputDoc( + Some(ref), + MappingOutputIdentifier(mapping.identifier, output), + None, + None + ) + val schemaDoc = SchemaDoc.ofStruct(doc.reference, schema) + doc.copy(schema = Some(schemaDoc)) + } + } catch { + case NonFatal(ex) => + logger.warn(s"Error while inferring schema description of mapping '${mapping.identifier}': ${reasons(ex)}") + mapping.outputs.map { output => + MappingOutputDoc( + Some(ref), + MappingOutputIdentifier(mapping.identifier, output), + None, + None + ) + } } val result = doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index c3699de73..5749cae96 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -94,6 +94,14 @@ final case class MappingOutputDoc( } +object MappingReference { + def of(parent:Reference, identifier:MappingIdentifier) : MappingReference = { + identifier.project match { + case None => MappingReference(Some(parent), identifier.name) + case Some(project) => MappingReference(Some(ProjectReference(project)), identifier.name) + } + } +} final case class MappingReference( override val parent:Option[Reference], name:String diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala index c297eeed5..775731885 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -32,7 +32,7 @@ final case class ProjectReference( final case class ProjectDoc( name: String, - version: Option[String], + version: Option[String] = None, description: Option[String] = None, targets:Map[TargetIdentifier,TargetDoc] = Map(), relations:Map[RelationIdentifier,RelationDoc] = Map(), diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 448f4fe01..3f02885e8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -16,16 +16,19 @@ package com.dimajix.flowman.documentation +import scala.util.Failure +import scala.util.Success +import scala.util.Try + import org.slf4j.LoggerFactory +import com.dimajix.common.ExceptionUtils.reasons import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.InputMapping -import com.dimajix.flowman.graph.MappingRef import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.graph.RelationRef import com.dimajix.flowman.graph.WriteRelation -import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation import com.dimajix.flowman.types.FieldValue @@ -55,7 +58,7 @@ class RelationCollector( case write:WriteRelation => write.input.incoming.flatMap { case map: InputMapping => - val mapref = MappingReference(Some(parent), map.input.name) + val mapref = MappingReference.of(parent, map.input.identifier) val outref = MappingOutputReference(Some(mapref), map.pin) Some(outref) case _ => None @@ -64,13 +67,13 @@ class RelationCollector( } val inputPartitions = node.outgoing.flatMap { case read:ReadRelation => - logger.debug(s"read partition ${relation.identifier}: ${read.input.relation.identifier} ${read.partitions}") + logger.debug(s"read partition ${relation.identifier}: ${read.input.identifier} ${read.partitions}") Some(read.partitions) case _ => None } val outputPartitions = node.incoming.flatMap { case write:WriteRelation => - logger.debug(s"write partition ${relation.identifier}: ${write.output.relation.identifier} ${write.partition}") + logger.debug(s"write partition ${relation.identifier}: ${write.output.identifier} ${write.partition}") Some(write.partition) case _ => None } @@ -88,7 +91,6 @@ class RelationCollector( ) val ref = doc.reference - val desc = SchemaDoc.ofStruct(ref, relation.describe(execution, partitions)) val schema = relation.schema.map { schema => val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) SchemaDoc( @@ -98,9 +100,19 @@ class RelationCollector( Seq() ) } - val mergedSchema = desc.merge(schema) + val mergedSchema = { + Try { + SchemaDoc.ofStruct(ref, relation.describe(execution, partitions)) + } match { + case Success(desc) => + Some(desc.merge(schema)) + case Failure(ex) => + logger.warn(s"Error while inferring schema description of relation '${relation.identifier}': ${reasons(ex)}") + schema + } + } - val result = doc.copy(schema = Some(mergedSchema)).merge(relation.documentation) + val result = doc.copy(schema = mergedSchema).merge(relation.documentation) if (executeTests) runTests(execution, relation, result) else diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index 01dc49d1e..6237086bf 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -21,6 +21,14 @@ import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.types.FieldValue +object RelationReference { + def of(parent:Reference, identifier:RelationIdentifier) : RelationReference = { + identifier.project match { + case None => RelationReference(Some(parent), identifier.name) + case Some(project) => RelationReference(Some(ProjectReference(project)), identifier.name) + } + } +} final case class RelationReference( parent:Option[Reference], name:String @@ -38,10 +46,10 @@ final case class RelationReference( final case class RelationDoc( parent:Option[Reference], identifier:RelationIdentifier, - description:Option[String], - schema:Option[SchemaDoc], - inputs:Seq[Reference], - provides:Seq[ResourceIdentifier], + description:Option[String] = None, + schema:Option[SchemaDoc] = None, + inputs:Seq[Reference] = Seq(), + provides:Seq[ResourceIdentifier] = Seq(), partitions:Map[String,FieldValue] = Map() ) extends EntityDoc { override def reference: RelationReference = RelationReference(parent, identifier.name) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index f6e7be8e9..c573c087d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -48,17 +48,17 @@ class TargetCollector extends Collector { val inputs = node.incoming.flatMap { case map: InputMapping => - val mapref = MappingReference(Some(parent), map.input.name) + val mapref = MappingReference.of(parent, map.input.identifier) val outref = MappingOutputReference(Some(mapref), map.pin) Some(outref) case read: ReadRelation => - val relref = RelationReference(Some(parent), read.input.name) + val relref = RelationReference.of(parent, read.input.identifier) Some(relref) case _ => None } val outputs = node.outgoing.flatMap { case write:WriteRelation => - val relref = RelationReference(Some(parent), write.output.name) + val relref = RelationReference.of(parent, write.output.identifier) Some(relref) case _ => None } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala index bb3bf1c04..885c045c6 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala @@ -16,9 +16,15 @@ package com.dimajix.flowman.documentation +import scala.util.Failure +import scala.util.Success +import scala.util.Try +import scala.util.control.NonFatal + import org.apache.spark.sql.DataFrame import org.slf4j.LoggerFactory +import com.dimajix.common.ExceptionUtils.reasons import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation @@ -39,8 +45,14 @@ class TestExecutor(execution: Execution) { val schemaDoc = doc.schema.map { schema => if (containsTests(schema)) { logger.info(s"Conducting tests on relation '${relation.identifier}'") - val df = relation.read(execution, doc.partitions) - runSchemaTests(df,schema) + try { + val df = relation.read(execution, doc.partitions) + runSchemaTests(df, schema) + } catch { + case NonFatal(ex) => + logger.warn(s"Error executing tests for relation '${relation.identifier}': ${reasons(ex)}") + failSchemaTests(schema) + } } else { schema @@ -60,8 +72,14 @@ class TestExecutor(execution: Execution) { val schema = output.schema.map { schema => if (containsTests(schema)) { logger.info(s"Conducting tests on mapping '${mapping.identifier}'") - val df = execution.instantiate(mapping, output.name) - runSchemaTests(df,schema) + try { + val df = execution.instantiate(mapping, output.name) + runSchemaTests(df, schema) + } catch { + case NonFatal(ex) => + logger.warn(s"Error executing tests for mapping '${mapping.identifier}': ${reasons(ex)}") + failSchemaTests(schema) + } } else { schema @@ -79,6 +97,22 @@ class TestExecutor(execution: Execution) { docs.exists(col => col.tests.nonEmpty || containsTests(col.children)) } + private def failSchemaTests(schema:SchemaDoc) : SchemaDoc = { + val columns = failColumnTests(schema.columns) + schema.copy(columns=columns) + } + private def failColumnTests(columns:Seq[ColumnDoc]) : Seq[ColumnDoc] = { + columns.map(col => failColumnTests(col)) + } + private def failColumnTests(column:ColumnDoc) : ColumnDoc = { + val tests = column.tests.map { test => + val result = TestResult(Some(test.reference), status = TestStatus.ERROR) + test.withResult(result) + } + val children = failColumnTests(column.children) + column.copy(children=children, tests=tests) + } + private def runSchemaTests(df:DataFrame, schema:SchemaDoc) : SchemaDoc = { val columns = runColumnTests(df, schema.columns) schema.copy(columns=columns) @@ -89,14 +123,23 @@ class TestExecutor(execution: Execution) { private def runColumnTests(df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { val columnPath = path + column.name val tests = column.tests.map { test => - val result = columnTestExecutors.flatMap(_.execute(execution, df, columnPath,test)).headOption - result match { - case None => - logger.warn(s"Could not find appropriate test executor for testing column $columnPath") - test - case Some(result) => - test.withResult(result.reparent(test.reference)) - } + val result = + try { + val result = columnTestExecutors.flatMap(_.execute(execution, df, columnPath, test)).headOption + result match { + case None => + logger.warn(s"Could not find appropriate test executor for testing column $columnPath") + TestResult(Some(test.reference), status = TestStatus.NOT_RUN) + case Some(result) => + result.reparent(test.reference) + } + } catch { + case NonFatal(ex) => + logger.warn(s"Error executing column test: ${reasons(ex)}") + TestResult(Some(test.reference), status = TestStatus.ERROR) + + } + test.withResult(result) } val children = runColumnTests(df, column.children, path + column.name + ".") column.copy(children=children, tests=tests) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala index 70ba31ac8..dbe08e4cc 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala @@ -22,6 +22,7 @@ object TestStatus { final case object FAILED extends TestStatus final case object SUCCESS extends TestStatus final case object ERROR extends TestStatus + final case object NOT_RUN extends TestStatus } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala index c6d47a98c..d58d347ec 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala @@ -20,9 +20,12 @@ import scala.collection.mutable import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target +import com.dimajix.flowman.model.TargetIdentifier sealed abstract class Node extends Product with Serializable { @@ -39,6 +42,7 @@ sealed abstract class Node extends Product with Serializable { def category : Category def kind : String def name : String + def project : Option[String] def provides : Set[ResourceIdentifier] def requires : Set[ResourceIdentifier] @@ -103,28 +107,35 @@ final case class MappingRef(id:Int, mapping:Mapping) extends Node { override def category: Category = Category.MAPPING override def kind: String = mapping.kind override def name: String = mapping.name + override def project: Option[String] = mapping.project.map(_.name) override def provides : Set[ResourceIdentifier] = Set() override def requires : Set[ResourceIdentifier] = mapping.requires + def identifier : MappingIdentifier = mapping.identifier } final case class TargetRef(id:Int, target:Target, phase:Phase) extends Node { override def category: Category = Category.TARGET override def kind: String = target.kind override def name: String = target.name + override def project: Option[String] = target.project.map(_.name) override def provides : Set[ResourceIdentifier] = target.provides(phase) override def requires : Set[ResourceIdentifier] = target.requires(phase) + def identifier : TargetIdentifier = target.identifier } final case class RelationRef(id:Int, relation:Relation) extends Node { override def category: Category = Category.RELATION override def kind: String = relation.kind override def name: String = relation.name + override def project: Option[String] = relation.project.map(_.name) override def provides : Set[ResourceIdentifier] = relation.provides override def requires : Set[ResourceIdentifier] = relation.requires + def identifier : RelationIdentifier = relation.identifier } final case class MappingColumn(id:Int, mapping: Mapping, output:String, column:String) extends Node { override def category: Category = Category.MAPPING_COLUMN override def kind: String = "mapping_column" override def name: String = mapping.name + "." + output + "." + column + override def project: Option[String] = mapping.project.map(_.name) override def provides : Set[ResourceIdentifier] = Set() override def requires : Set[ResourceIdentifier] = Set() } @@ -132,6 +143,7 @@ final case class RelationColumn(id:Int, relation: Relation, column:String) exten override def category: Category = Category.RELATION_COLUMN override def kind: String = "relation_column" override def name: String = relation.name + "." + column + override def project: Option[String] = relation.project.map(_.name) override def provides : Set[ResourceIdentifier] = Set() override def requires : Set[ResourceIdentifier] = Set() } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala new file mode 100644 index 000000000..2d9b1cbe0 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala @@ -0,0 +1,142 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.Linker +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.Target +import com.dimajix.flowman.model.TargetIdentifier +import com.dimajix.flowman.types.FieldValue +import com.dimajix.flowman.types.SingleValue +import com.dimajix.flowman.types.StructType + + +class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { + "RelationCollector.collect" should "work" in { + val mapping1 = mock[Mapping] + val mappingTemplate1 = mock[Prototype[Mapping]] + val mapping2 = mock[Mapping] + val mappingTemplate2 = mock[Prototype[Mapping]] + val sourceRelation = mock[Relation] + val sourceRelationTemplate = mock[Prototype[Relation]] + val targetRelation = mock[Relation] + val targetRelationTemplate = mock[Prototype[Relation]] + val target = mock[Target] + val targetTemplate = mock[Prototype[Target]] + + val project = Project( + name = "project", + mappings = Map( + "m1" -> mappingTemplate1, + "m2" -> mappingTemplate2 + ), + targets = Map( + "t" -> targetTemplate + ), + relations = Map( + "src" -> sourceRelationTemplate, + "tgt" -> targetRelationTemplate + ) + ) + val session = Session.builder().disableSpark().build() + val context = session.getContext(project) + val execution = session.execution + + (mappingTemplate1.instantiate _).expects(context).returns(mapping1) + (mapping1.context _).expects().returns(context) + (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) + + (mappingTemplate2.instantiate _).expects(context).returns(mapping2) + (mapping2.context _).expects().returns(context) + (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map("pcol"-> SingleValue("part1"))))) + + (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) + (sourceRelation.context _).expects().returns(context) + (sourceRelation.link _).expects(*).returns(Unit) + + (targetRelationTemplate.instantiate _).expects(context).returns(targetRelation) + (targetRelation.context _).expects().returns(context) + (targetRelation.link _).expects(*).returns(Unit) + + (targetTemplate.instantiate _).expects(context).returns(target) + (target.context _).expects().returns(context) + (target.link _).expects(*,*).onCall((l:Linker, _:Phase) => Some(1).foreach { _ => + l.input(MappingIdentifier("m1"), "main") + l.write(RelationIdentifier("tgt"), Map("outcol"-> SingleValue("part1"))) + }) + + val graph = Graph.ofProject(session, project, Phase.BUILD) + + (mapping1.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m1")) + //(mapping2.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m2")) + + (sourceRelation.identifier _).expects().atLeastOnce().returns(RelationIdentifier("project/src")) + (sourceRelation.description _).expects().atLeastOnce().returns(Some("source relation")) + (sourceRelation.documentation _).expects().returns(None) + (sourceRelation.provides _).expects().returns(Set()) + //(sourceRelation.requires _).expects().returns(Set()) + (sourceRelation.schema _).expects().returns(None) + (sourceRelation.describe _).expects(*,Map("pcol"-> SingleValue("part1"))).returns(StructType(Seq())) + + (targetRelation.identifier _).expects().atLeastOnce().returns(RelationIdentifier("project/tgt")) + (targetRelation.description _).expects().atLeastOnce().returns(Some("target relation")) + (targetRelation.documentation _).expects().returns(None) + (targetRelation.provides _).expects().returns(Set()) + //(targetRelation.requires _).expects().returns(Set()) + (targetRelation.schema _).expects().returns(None) + (targetRelation.describe _).expects(*,Map("outcol"-> SingleValue("part1"))).returns(StructType(Seq())) + + val collector = new RelationCollector() + val projectDoc = collector.collect(execution, graph, ProjectDoc(project.name)) + + val sourceRelationDoc = projectDoc.relations(RelationIdentifier("project/src")) + val targetRelationDoc = projectDoc.relations(RelationIdentifier("project/tgt")) + + sourceRelationDoc should be (RelationDoc( + parent = Some(ProjectReference("project")), + identifier = RelationIdentifier("project/src"), + description = Some("source relation"), + schema = Some(SchemaDoc( + parent = Some(RelationReference(Some(ProjectReference("project")), "src")) + )), + partitions = Map("pcol" -> SingleValue("part1")) + )) + + targetRelationDoc should be (RelationDoc( + parent = Some(ProjectReference("project")), + identifier = RelationIdentifier("project/tgt"), + description = Some("target relation"), + schema = Some(SchemaDoc( + parent = Some(RelationReference(Some(ProjectReference("project")), "tgt")) + )), + inputs = Seq(MappingOutputReference(Some(MappingReference(Some(ProjectReference("project")), "m1")), "main")), + partitions = Map("outcol" -> SingleValue("part1")) + )) + } +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala index 85fece935..6523ef6af 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala @@ -21,26 +21,18 @@ import scala.util.Success import scala.util.Try import scala.util.control.NonFatal -import org.apache.hadoop.fs.Path import org.kohsuke.args4j.Argument import org.slf4j.LoggerFactory import com.dimajix.common.ExceptionUtils.reasons import com.dimajix.flowman.common.ParserUtils.splitSettings -import com.dimajix.flowman.documentation.Documenter -import com.dimajix.flowman.documentation.MappingCollector -import com.dimajix.flowman.documentation.RelationCollector -import com.dimajix.flowman.documentation.TargetCollector import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Session import com.dimajix.flowman.execution.Status -import com.dimajix.flowman.hadoop.File import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.JobIdentifier import com.dimajix.flowman.model.Project -import com.dimajix.flowman.spec.documentation.FileGenerator import com.dimajix.flowman.tools.exec.Command -import com.dimajix.flowman.types.FieldValue class GenerateCommand extends Command { @@ -72,7 +64,7 @@ class GenerateCommand extends Command { Status.SUCCESS } catch { case NonFatal(ex) => - logger.error("Cannot generate documentation: " + reasons(ex)) + logger.error("Cannot generate documentation: ", ex) Status.FAILED } } From f280e44de1da880c4307fd02dd2d7e1db60b100c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 15 Feb 2022 09:05:50 +0100 Subject: [PATCH 39/95] Add new CONTRIBUTING.md --- CONTRIBUTING.md | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 50 +++++++++++++--------- 2 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..68d860396 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing to Flowman + +You want to contribute to Flowman? Welcome! Please read this document to understand what you can do: + * [Report an Issue](#report-an-issue) + * [Contribute Documentation](#contribute-documentation) + * [Contribute Code](#contribute-code) + + +## Report an Issue + +If you find a bug - behavior of Flowman code contradicting your expectation - you are welcome to report it. +We can only handle well-reported, actual bugs, so please follow the guidelines below. + +Once you have familiarized with the guidelines, you can go to the [GitHub issue tracker for Flowman](https://github.com/dimajix/flowman/issues/new) to report the issue. + +### Quick Checklist for Bug Reports + +Issue report checklist: + * Real, current bug + * No duplicate + * Reproducible + * Good summary + * Well-documented + * Minimal example + +### Issue handling process + +When an issue is reported, a committer will look at it and either confirm it as a real issue, close it if it is not an issue, or ask for more details. + +An issue that is about a real bug is closed as soon as the fix is committed. + +### Usage of Labels + +GitHub offers labels to categorize issues. We suggest the following labels: + +Labels for issue categories: + * bug: this issue is a bug in the code + * feature: this issue is a request for a new functionality or an enhancement request + * environment: this issue relates to supporting a specific runtime environment (Cloudera, specific Spark/Hadoop version, etc) + +Status of open issues: + * help wanted: the feature request is approved and you are invited to contribute + +Status/resolution of closed issues: + * wontfix: while acknowledged to be an issue, a fix cannot or will not be provided + +### Issue Reporting Disclaimer + +We want to improve the quality of Flowman and good bug reports are welcome! But our capacity is limited, thus we reserve the right to close or to not process insufficient bug reports in favor of those which are very cleanly documented and easy to reproduce. Even though we would like to solve each well-documented issue, there is always the chance that it will not happen - remember: Flowman is Open Source and comes without warranty. + +Bug report analysis support is very welcome! (e.g. pre-analysis or proposing solutions) + + + +## Contribute Documentation + +Flowman has many features implemented, unfortunately not all of them are well documented. So this is an area where we highly welcome contributions from users in order to improve the documentation. The documentation is contained in the "doc" subdirectory within the source code repository. This implies that when you want to contribute documentation, you have to follow the same procedure as for contributing code. + + + +## Contribute Code + +You are welcome to contribute code to Flowman in order to fix bugs or to implement new features. + +There are three important things to know: + +1. You must be aware of the Apache License (which describes contributions) and **agree to the Contributors License Agreement**. This is common practice in all major Open Source projects. + For company contributors special rules apply. See the respective section below for details. +2. Please ensure your contribution adopts Flowmans **code style, quality, and product standards**. The respective section below gives more details on the coding guidelines. +3. **Not all proposed contributions can be accepted**. Some features may e.g. just fit a third-party plugin better. The code must fit the overall direction of Flowman and really improve it. The more effort you invest, the better you should clarify in advance whether the contribution fits: the best way would be to just open an issue to discuss the feature you plan to implement (make it clear you intend to contribute). + +### Contributor License Agreement + +When you contribute (code, documentation, or anything else), you have to be aware that your contribution is covered by the same [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0) that is applied to Flowman itself. + +In particular, you need to agree to the [Flowman Contributors License Agreement](https://cla-assistant.io/dimajix/flowman), stating that you have the right and are okay to put your contribution under the license of this project. +CLA assistant will ask you to confirm that. + +This applies to all contributors, including those contributing on behalf of a company. +If you agree to its content, you simply have to click on the link posted by the CLA assistant as a comment to the pull request. Click it to check the CLA, then accept it on the following screen if you agree to it. +CLA assistant will save this decision for upcoming contributions and will notify you if there is any change to the CLA in the meantime. + +### Contribution Content Guidelines + +These are some rules we try to follow: + +- Apply a clean coding style adapted to the surrounding code, even though we are aware the existing code is not fully clean +- Use (4)spaces for indentation +- Use variable naming conventions like in the other files you are seeing (camelcase) +- No println - use SLF4J logging instead +- Comment your code where it gets non-trivial +- Write a unit test +- Do not do any incompatible changes, especially do not change or remove existing properties from YAML specs + +### How to contribute - the Process + +1. Make sure the change would be welcome (e.g. a bugfix or a useful feature); best do so by proposing it in a GitHub issue +2. Create a branch forking the flowman repository and do your change +3. Commit and push your changes on that branch +4. If your change fixes an issue reported at GitHub, add the following line to the commit message: + - ```Fixes #(issueNumber)``` +5. Create a Pull Request with the following information + - Describe the problem you fix with this change. + - Describe the effect that this change has from a user's point of view. App crashes and lockups are pretty convincing for example, but not all bugs are that obvious and should be mentioned in the text. + - Describe the technical details of what you changed. It is important to describe the change in a most understandable way so the reviewer is able to verify that the code is behaving as you intend it to. +6. Follow the link posted by the CLA assistant to your pull request and accept it, as described in detail above. +7. Wait for our code review and approval, possibly enhancing your change on request + - Note that the Flowman developers also have their regular duties, so depending on the required effort for reviewing, testing and clarification this may take a while +8. Once the change has been approved we will inform you in a comment +9. We will close the pull request, feel free to delete the now obsolete branch diff --git a/README.md b/README.md index 676cbbd61..5a580a064 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,11 @@ keep all aspects (like transformations and schema information) in a single place * Semantics of a build tool like Maven - just for data instead for applications * Declarative syntax in YAML files * Data model management (Create, Migrate and Destroy Hive tables, JDBC tables or file based storage) +* Generation of meaningful documentation * Flexible expression language * Jobs for managing build targets (like copying files or uploading data via sftp) * Automatic data dependency management within the execution of individual jobs -* Rich set of execution metrics -* Meaningful logging output +* Meaningful logging output & rich set of execution metrics * Powerful yet simple command line tools * Extendable via Plugins @@ -38,28 +38,21 @@ You can find the official homepage at [Flowman.io](https://flowman.io) # Installation -You can either grab an appropriate pre-build package at https://github.com/dimajix/flowman/releases or you -can build your own version via Maven with - - mvn clean install - -Please also read [BUILDING.md](BUILDING.md) for detailed instructions, specifically on build profiles. - +You can either grab an appropriate pre-build package at [GitHub](https://github.com/dimajix/flowman/releases) ## Installing the Packed Distribution The packed distribution file is called `flowman-{version}-bin.tar.gz` and can be extracted at any location using - - tar xvzf flowman-{version}-bin.tar.gz - +```shell +tar xvzf flowman-{version}-bin.tar.gz +``` ## Apache Spark Flowman does not bring its own Spark libraries, but relies on a correctly installed Spark distribution. You can download appropriate packages directly from [https://spark.apache.org](the Spark Homepage). - ## Hadoop Utils for Windows If you are trying to run the application on Windows, you also need the *Hadoop Winutils*, which is a set of @@ -70,7 +63,6 @@ Once you downloaded the appropriate version, you need to place the DLLs into a d * `PATH` should also contain `$HADOOP_HOME/bin` - # Command Line Utils The primary tool provided by Flowman is called `flowexec` and is located in the `bin` folder of the @@ -80,19 +72,37 @@ installation directory. The `flowexec` tool has several subcommands for working with objects and projects. The general pattern looks as follows - - flowexec [generic options] [specific options and arguments] +```shell +flowexec [generic options] [specific options and arguments] +``` For working with `flowexec`, either your current working directory needs to contain a Flowman project with a file `project.yml` or you need to specify the path to a valid project via - - flowexec -f /path/to/project/folder +```shell +flowexec -f /path/to/project/folder +``` ## Interactive Shell With version 0.14.0, Flowman also introduced a new interactive shell for executing data flows. The shell can be started via - - flowshell -f +```shell +flowshell -f +``` Within the shell, you can interactively build targets and inspect intermediate mappings. + + +# Building + +You can build your own version via Maven with +```shell +mvn clean install +``` +Please also read [BUILDING.md](BUILDING.md) for detailed instructions, specifically on build profiles. + + +# Contributing + +You want to contribute to Flowman? Welcome! Please read [CONTRIBUTING.md](CONTRIBUTING.md) to understand what you can +do. From 7a305d5037e3b25a83824c0298b239d900927c2f Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 16 Feb 2022 08:39:44 +0100 Subject: [PATCH 40/95] Add new html template for documentation --- examples/weather/documentation.yml | 1 + .../flowman/documentation/html/project.vtl | 148 ++++++++++++++++++ .../documentation/html/template.properties | 2 + .../documentation/text/template.properties | 2 + .../spec/documentation/FileGenerator.scala | 27 +++- .../documentation/TemplateGenerator.scala | 16 +- 6 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl create mode 100644 flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/template.properties create mode 100644 flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/template.properties diff --git a/examples/weather/documentation.yml b/examples/weather/documentation.yml index 6c6639cf8..1ccaa44dc 100644 --- a/examples/weather/documentation.yml +++ b/examples/weather/documentation.yml @@ -10,3 +10,4 @@ generators: # Create an output file in the project directory - kind: file location: ${project.basedir}/generated-documentation + template: html diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl new file mode 100644 index 000000000..555870c8c --- /dev/null +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -0,0 +1,148 @@ + + + + + Flowman Project '${project.name}' version ${project.version} + + + +

Flowman Project '${project.name}' version ${project.version}

+
Description: ${project.description}
+ +

Mappings

+ + +

Relations

+ + +

Targets

+ + + +

Mappings

+#foreach($mapping in ${project.mappings}) +

Mapping '${mapping.identifier}'

+
Description: ${mapping.description}
+

Inputs

+ #foreach($input in ${mapping.inputs}) + + + + + +
${input.kind}${input}
+ #end +

Outputs

+ #foreach($output in ${mapping.outputs}) +
${output.name}
+
Description: ${output.description}
+ + + + + + + + + + + #foreach($column in ${output.schema.columns}) + + + + + + + + #end +
ColumnData TypeAttributesDescriptionTests
${column.name}${column.catalogType}#if(!$column.nullable) NOT NULL #end${column.description}
+ #end +#end + +

Relations

+#foreach($relation in ${project.relations}) +

Relation '${relation.identifier}'

+
Description: ${relation.description}
+

Resources

+ + #foreach($resource in ${relation.resources}) + + + + + #end +
${resource.category}${resource.name}
+

Inputs

+ + #foreach($input in ${relation.inputs}) + + + + + #end +
${input}
+

Schema

+ + + + + + + + + + + #foreach($column in ${relation.schema.columns}) + + + + + + + + #end +
ColumnData TypeAttributesDescriptionTests
${column.name}${column.catalogType}#if(!$column.nullable) NOT NULL #end${column.description}
+#end + +

Targets

+#foreach($target in ${project.targets}) +

Target '${target.identifier}'

+
Description: ${target.description}
+

Inputs

+ #foreach($input in ${target.inputs}) + + + + + +
${input.kind}${input}
+ #end +

Outputs

+ #foreach($output in ${target.outputs}) + + + + + +
${output.kind}${output}
+ #end +

Phases

+
+ #foreach($phase in ${target.phases}) + ${phase.name} + #end +
+#end + diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/template.properties b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/template.properties new file mode 100644 index 000000000..9e0fa5bbd --- /dev/null +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/template.properties @@ -0,0 +1,2 @@ +template.project.input=project.vtl +template.project.output=project.html diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/template.properties b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/template.properties new file mode 100644 index 000000000..ea4cee436 --- /dev/null +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/text/template.properties @@ -0,0 +1,2 @@ +template.project.input=project.vtl +template.project.output=project.txt diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala index 316d5ee19..534a44565 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -16,9 +16,12 @@ package com.dimajix.flowman.spec.documentation +import java.io.StringReader import java.net.URL import java.nio.charset.Charset +import java.util.Properties +import scala.collection.JavaConverters._ import com.fasterxml.jackson.annotation.JsonProperty import com.google.common.io.Resources import org.apache.hadoop.fs.Path @@ -33,7 +36,8 @@ import com.dimajix.flowman.hadoop.File object FileGenerator { val textTemplate : URL = Resources.getResource(classOf[FileGenerator], "/com/dimajix/flowman/documentation/text") - val defaultTemplate : URL = textTemplate + val htmlTemplate : URL = Resources.getResource(classOf[FileGenerator], "/com/dimajix/flowman/documentation/html") + val defaultTemplate : URL = htmlTemplate } @@ -44,6 +48,9 @@ case class FileGenerator( private val logger = LoggerFactory.getLogger(classOf[FileGenerator]) override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + val props = new Properties() + props.load(new StringReader(loadResource("template.properties"))) + val fs = execution.fs val uri = location.toUri @@ -61,8 +68,15 @@ case class FileGenerator( } outputDir.mkdirs() - val projectDoc = renderProject(context, documentation) - writeFile(outputDir / "project.txt", projectDoc) + generateProjectFile(context, documentation, outputDir, props.asScala.toMap) + } + + private def generateProjectFile(context:Context, documentation: ProjectDoc, outputDir:File, properties: Map[String,String]) : Unit= { + val in = properties.getOrElse("template.project.input", "project.vtl") + val out = properties("template.project.output") + + val projectDoc = renderProject(context, documentation, in) + writeFile(outputDir / out, projectDoc) } private def writeFile(file:File, content:String) : Unit = { @@ -85,9 +99,14 @@ class FileGeneratorSpec extends GeneratorSpec { @JsonProperty(value="template", required=false) private var template:String = FileGenerator.defaultTemplate.toString override def instantiate(context: Context): Generator = { + val url = context.evaluate(template) match { + case "text" => FileGenerator.textTemplate + case "html" => FileGenerator.htmlTemplate + case str => new URL(str) + } FileGenerator( new Path(context.evaluate(location)), - new URL(context.evaluate(template)) + url ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala index a2eb9b181..c283228d3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala @@ -39,20 +39,20 @@ abstract class TemplateGenerator( ) extends BaseGenerator { override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit - protected def renderProject(context:Context, documentation: ProjectDoc) : String = { - val temp = loadResource("project.vtl") + protected def renderProject(context:Context, documentation: ProjectDoc, template:String="project.vtl") : String = { + val temp = loadResource(template) context.evaluate(temp, Map("project" -> ProjectDocWrapper(documentation))) } - protected def renderRelation(context:Context, documentation: RelationDoc) : String = { - val temp = loadResource("relation.vtl") + protected def renderRelation(context:Context, documentation: RelationDoc, template:String="relation.vtl") : String = { + val temp = loadResource(template) context.evaluate(temp, Map("relation" -> RelationDocWrapper(documentation))) } - protected def renderMapping(context:Context, documentation: MappingDoc) : String = { - val temp = loadResource("mapping.vtl") + protected def renderMapping(context:Context, documentation: MappingDoc, template:String="mapping.vtl") : String = { + val temp = loadResource(template) context.evaluate(temp, Map("mapping" -> MappingDocWrapper(documentation))) } - protected def renderTarget(context:Context, documentation: TargetDoc) : String = { - val temp = loadResource("target.vtl") + protected def renderTarget(context:Context, documentation: TargetDoc, template:String="target.vtl") : String = { + val temp = loadResource(template) context.evaluate(temp, Map("target" -> TargetDocWrapper(documentation))) } From 440db8957d453a6ad531ef2cde682eff9f6b359d Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 16 Feb 2022 21:12:21 +0100 Subject: [PATCH 41/95] Improve html documentation generation --- .../flowman/documentation/MappingDoc.scala | 16 +- .../flowman/documentation/ProjectDoc.scala | 6 +- .../flowman/documentation/velocity.scala | 20 ++ .../flowman/documentation/ColumnDocTest.scala | 2 +- .../documentation/ProjectDocTest.scala | 56 ++++ .../org/apache/spark/sql/DateTimeTest.scala | 23 ++ .../flowman/documentation/html/project.vtl | 291 ++++++++++++------ 7 files changed, 307 insertions(+), 107 deletions(-) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala create mode 100644 flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index 5749cae96..f5b24a0f9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -37,8 +37,8 @@ final case class MappingOutputReference( final case class MappingOutputDoc( parent:Some[Reference], identifier: MappingOutputIdentifier, - description: Option[String], - schema:Option[SchemaDoc] + description: Option[String] = None, + schema:Option[SchemaDoc] = None ) extends Fragment { override def reference: Reference = MappingOutputReference(parent, identifier.output) override def fragments: Seq[Fragment] = schema.toSeq @@ -103,7 +103,7 @@ object MappingReference { } } final case class MappingReference( - override val parent:Option[Reference], + override val parent:Option[Reference] = None, name:String ) extends Reference { override def toString: String = { @@ -117,16 +117,16 @@ final case class MappingReference( final case class MappingDoc( - parent:Option[Reference], + parent:Option[Reference] = None, identifier:MappingIdentifier, - description:Option[String], - inputs:Seq[Reference], - outputs:Seq[MappingOutputDoc] + description:Option[String] = None, + inputs:Seq[Reference] = Seq.empty, + outputs:Seq[MappingOutputDoc] = Seq.empty ) extends EntityDoc { override def reference: MappingReference = MappingReference(parent, identifier.name) override def fragments: Seq[Fragment] = outputs override def reparent(parent: Reference): MappingDoc = { - val ref = MappingOutputReference(Some(parent), identifier.name) + val ref = MappingReference(Some(parent), identifier.name) copy( parent=Some(parent), outputs=outputs.map(_.reparent(ref)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala index 775731885..a4395400d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.documentation +import scala.annotation.tailrec + import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.TargetIdentifier @@ -56,8 +58,10 @@ final case class ProjectDoc( case head :: tail => if (head != reference) None + else if (tail.isEmpty) + Some(this) else - resolve(head).flatMap(_.resolve(tail)) + fragments.find(_.reference == tail.head).flatMap(_.resolve(tail.tail)) case Nil => None } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index ef6d1f41b..e24d3845f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -132,6 +132,26 @@ final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(p def getName() : String = project.name def getVersion() : String = project.version.getOrElse("") + def resolve(reference:ReferenceWrapper) : FragmentWrapper = { + val x = project.resolve(reference.reference).map { + case m:MappingDoc => MappingDocWrapper(m) + case o:MappingOutputDoc => MappingOutputDocWrapper(o) + case r:RelationDoc => RelationDocWrapper(r) + case t:TargetDoc => TargetDocWrapper(t) + case p:TargetPhaseDoc => TargetPhaseDocWrapper(p) + case s:SchemaDoc => SchemaDocWrapper(s) + case t:TestResult => TestResultWrapper(t) + case c:ColumnDoc => ColumnDocWrapper(c) + case t:ColumnTest => ColumnTestWrapper(t) + case f:Fragment => new FragmentWrapper(f) + }.orNull + + if (x == null) + println("null") + + x + } + def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.values.map(MappingDocWrapper).toSeq.asJava def getRelations() : java.util.List[RelationDocWrapper] = project.relations.values.map(RelationDocWrapper).toSeq.asJava def getTargets() : java.util.List[TargetDocWrapper] = project.targets.values.map(TargetDocWrapper).toSeq.asJava diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala index 05e4299d9..acb40f279 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala @@ -25,7 +25,7 @@ import com.dimajix.flowman.types.NullType import com.dimajix.flowman.types.StringType -class ColumnDocTest extends AnyFlatSpec with Matchers { +class ColumnDocTest extends AnyFlatSpec with Matchers { "A ColumnDoc" should "support merge" in { val doc1 = ColumnDoc( None, diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala new file mode 100644 index 000000000..e012cd80c --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier + + +class ProjectDocTest extends AnyFlatSpec with Matchers { + "A ProjectDoc" should "support resolving" in { + val project = ProjectDoc( + name = "project" + ) + val projectRef = project.reference + val mapping = MappingDoc( + parent = Some(projectRef), + identifier = MappingIdentifier("project/m1") + ) + val mappingRef = mapping.reference + val output = MappingOutputDoc( + parent = Some(mappingRef), + identifier = MappingOutputIdentifier("project/m1:main") + ) + val outputRef = output.reference + val schema = SchemaDoc( + parent = Some(outputRef) + ) + val schemaRef = schema.reference + + val finalOutput = output.copy(schema = Some(schema)) + val finalMapping = mapping.copy(outputs = Seq(finalOutput)) + val finalProject = project.copy(mappings = Map(mapping.identifier -> finalMapping)) + + finalProject.resolve(projectRef) should be (Some(finalProject)) + finalProject.resolve(mappingRef) should be (Some(finalMapping)) + finalProject.resolve(outputRef) should be (Some(finalOutput)) + finalProject.resolve(schemaRef) should be (Some(schema)) + } +} diff --git a/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala b/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala new file mode 100644 index 000000000..938f49bc7 --- /dev/null +++ b/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala @@ -0,0 +1,23 @@ +package org.apache.spark.sql + +import java.sql.Timestamp + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.spark.testing.LocalSparkSession +import com.dimajix.util.DateTimeUtils + + +class DateTimeTest extends AnyFlatSpec with Matchers with LocalSparkSession { + def localTime(str:String) : Timestamp = new Timestamp(DateTimeUtils.stringToTime(str).getTime) + + "Date truncation" should "work in" in { + spark.conf.set("spark.sql.session.timeZone", "UTC") + val df = spark.createDataFrame(Seq((localTime("2019-02-01T12:34:03"), ""))) + + val result = df.selectExpr("date_trunc('day', _1)") + val collectedRows = result.collect() + result.show() + } +} diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 555870c8c..93ef18283 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -3,78 +3,212 @@ Flowman Project '${project.name}' version ${project.version} - + + + +#macro(schema $schema) + + + + + + + + + + + #foreach($column in ${schema.columns}) + + + + + + + + #end +
Column NameData TypeAttributesDescriptionTests
${column.name}${column.catalogType}#if(!$column.nullable) NOT NULL #end${column.description} + + #foreach($test in ${column.tests}) + + + + + #end +
${test.name}${test.status}
+
#end - -

Relations

- +#end + -

Targets

-
    - #foreach($target in ${project.targets}) -
  • '${target.identifier}'
    • + +
    +

    Flowman Project '${project.name}' version ${project.version}

    +
    Description: ${project.description}
    +
    + +
    +

    Index

    +

    Mappings

    + +
+ +

Relations

+ +

Targets

+ + -

Mappings

+

Mappings

#foreach($mapping in ${project.mappings}) +
+

Mapping '${mapping.identifier}'

Description: ${mapping.description}
+

Inputs

- #foreach($input in ${mapping.inputs}) - - - - - -
${input.kind}${input}
- #end + #references(${mapping.inputs})

Outputs

+
#foreach($output in ${mapping.outputs})
${output.name}
Description: ${output.description}
- - - - - - - - - - - #foreach($column in ${output.schema.columns}) - - - - - - - - #end -
ColumnData TypeAttributesDescriptionTests
${column.name}${column.catalogType}#if(!$column.nullable) NOT NULL #end${column.description}
+ #schema($output.schema) #end +
+
#end -

Relations

+

Relations

#foreach($relation in ${project.relations}) +
+

Relation '${relation.identifier}'

Description: ${relation.description}
+

Resources

#foreach($resource in ${relation.resources}) @@ -85,64 +219,27 @@ #end

Inputs

- - #foreach($input in ${relation.inputs}) - - - - - #end -
${input}
+ #references(${relation.inputs})

Schema

- - - - - - - - - - - #foreach($column in ${relation.schema.columns}) - - - - - - - - #end -
ColumnData TypeAttributesDescriptionTests
${column.name}${column.catalogType}#if(!$column.nullable) NOT NULL #end${column.description}
+ #schema($relation.schema) +
#end -

Targets

+

Targets

#foreach($target in ${project.targets}) +
+

Target '${target.identifier}'

Description: ${target.description}
+

Inputs

- #foreach($input in ${target.inputs}) - - - - - -
${input.kind}${input}
- #end + #references(${target.inputs})

Outputs

- #foreach($output in ${target.outputs}) - - - - - -
${output.kind}${output}
- #end + #references(${target.outputs})

Phases

-
#foreach($phase in ${target.phases}) - ${phase.name} +
${phase.name}
#end -
+
#end From 354223e3f80e6450d07627a7a9fa851bef06fc1c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 07:24:40 +0100 Subject: [PATCH 42/95] Remove wrong test --- .../org/apache/spark/sql/DateTimeTest.scala | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala diff --git a/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala b/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala deleted file mode 100644 index 938f49bc7..000000000 --- a/flowman-spark-extensions/src/test/scala/org/apache/spark/sql/DateTimeTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -package org.apache.spark.sql - -import java.sql.Timestamp - -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import com.dimajix.spark.testing.LocalSparkSession -import com.dimajix.util.DateTimeUtils - - -class DateTimeTest extends AnyFlatSpec with Matchers with LocalSparkSession { - def localTime(str:String) : Timestamp = new Timestamp(DateTimeUtils.stringToTime(str).getTime) - - "Date truncation" should "work in" in { - spark.conf.set("spark.sql.session.timeZone", "UTC") - val df = spark.createDataFrame(Seq((localTime("2019-02-01T12:34:03"), ""))) - - val result = df.selectExpr("date_trunc('day', _1)") - val collectedRows = result.collect() - result.show() - } -} From 98375d126a1e227113e3bdc3728f2359c0ece262 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 11:01:59 +0100 Subject: [PATCH 43/95] Improve html documentation generation --- docs/documenting/config.md | 40 ++++++++++++ examples/weather/documentation.yml | 5 ++ .../documentation/MappingCollector.scala | 7 +-- .../flowman/documentation/ProjectDoc.scala | 14 ++--- .../documentation/RelationCollector.scala | 2 +- .../documentation/TargetCollector.scala | 2 +- .../flowman/documentation/velocity.scala | 13 ++-- .../documentation/ProjectDocTest.scala | 2 +- .../documentation/RelationCollectorTest.scala | 14 ++--- .../flowman/documentation/html/project.vtl | 27 ++++++-- .../spec/documentation/FileGenerator.scala | 30 ++++++--- .../documentation/TemplateGenerator.scala | 62 ++++++++++++++++++- 12 files changed, 167 insertions(+), 51 deletions(-) diff --git a/docs/documenting/config.md b/docs/documenting/config.md index 3e2482e51..0844732a2 100644 --- a/docs/documenting/config.md +++ b/docs/documenting/config.md @@ -20,4 +20,44 @@ generators: # Create an output file in the project directory - kind: file location: ${project.basedir}/doc + # This will exclude all mappings + excludeMappings: ".*" + excludeRelations: + # You can either specify a name (without the project) + - "stations_raw" + # Or can also explicitly specify a name with the project + - ".*/measurements_raw" ``` + +## File Generator Fields + +The generator is used for generating the documentation. You can configure multiple generators for creating multiple +differently configured documentations. + +* `kind` **(mandatory)** *(type: string)*: `file` + +* `location` **(mandatory)** *(type: string)*: Specifies the output location + +* `includeMappings` **(optional)** *(type: list:regex)* *(default: ".*")*: +List of regular expressions which mappings to include. Per default all mappings will be included in the output. +The list of filters will be applied before the `excludeMappings` filter list. + +* `excludeMappings` **(optional)** *(type: list:regex)* + List of regular expressions which mappings to exclude. Per default no mapping will be excluded in the output. + The list of filters will be applied after the `includeMappings` filter list. + +* `includeTargets` **(optional)** *(type: list:regex)* *(default: ".*")*: + List of regular expressions which targets to include. Per default all targets will be included in the output. + The list of filters will be applied before the `excludeTargets` filter list. + +* `excludeTargets` **(optional)** *(type: list:regex)* + List of regular expressions which targets to exclude. Per default no target will be excluded in the output. + The list of filters will be applied after the `includeTargets` filter list. + +* `includeRelations` **(optional)** *(type: list:regex)* *(default: ".*")*: + List of regular expressions which relations to include. Per default all relations will be included in the output. + The list of filters will be applied before the `excludeRelations` filter list. + +* `excludeRelations` **(optional)** *(type: list:regex)* + List of regular expressions which relations to exclude. Per default no relation will be excluded in the output. + The list of filters will be applied after the `includeRelations` filter list. diff --git a/examples/weather/documentation.yml b/examples/weather/documentation.yml index 1ccaa44dc..859e0aa69 100644 --- a/examples/weather/documentation.yml +++ b/examples/weather/documentation.yml @@ -11,3 +11,8 @@ generators: - kind: file location: ${project.basedir}/generated-documentation template: html + excludeRelations: + # You can either specify a name (without the project) + - "stations_raw" + # Or can also explicitly specify a name with the project + - ".*/measurements_raw" diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index a45fb1c65..af9989a14 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -61,14 +61,13 @@ class MappingCollector( // Add additional inputs from non-mapping entities val incoming = node.incoming.collect { - case ReadRelation(input, _, _) => documentation.relations.get(input.relation.identifier).map(_.reference) + // TODO: The following logic is not correct in case of embedded relations. We would need an IdentityHashMap instead + case ReadRelation(input, _, _) => documentation.relations.find(_.identifier == input.relation.identifier).map(_.reference) }.flatten doc.copy(inputs=doc.inputs ++ incoming) } - val docs = graph.mappings.map { mapping => - mapping.mapping.identifier -> getMappingDoc(mapping) - }.toMap + val docs = graph.mappings.map(mapping => getMappingDoc(mapping)) documentation.copy(mappings=docs) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala index a4395400d..80cbccc49 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ProjectDoc.scala @@ -16,12 +16,6 @@ package com.dimajix.flowman.documentation -import scala.annotation.tailrec - -import com.dimajix.flowman.model.MappingIdentifier -import com.dimajix.flowman.model.RelationIdentifier -import com.dimajix.flowman.model.TargetIdentifier - final case class ProjectReference( name:String @@ -36,13 +30,13 @@ final case class ProjectDoc( name: String, version: Option[String] = None, description: Option[String] = None, - targets:Map[TargetIdentifier,TargetDoc] = Map(), - relations:Map[RelationIdentifier,RelationDoc] = Map(), - mappings:Map[MappingIdentifier,MappingDoc] = Map() + targets:Seq[TargetDoc] = Seq.empty, + relations:Seq[RelationDoc] = Seq.empty, + mappings:Seq[MappingDoc] = Seq.empty ) extends EntityDoc { override def reference: Reference = ProjectReference(name) override def parent: Option[Reference] = None - override def fragments: Seq[Fragment] = (targets.values ++ relations.values ++ mappings.values).toSeq + override def fragments: Seq[Fragment] = (targets ++ relations ++ mappings).toSeq override def resolve(path:Seq[Reference]) : Option[Fragment] = { if (path.isEmpty) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 3f02885e8..4089e488d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -40,7 +40,7 @@ class RelationCollector( override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference - val docs = graph.relations.map(t => t.relation.identifier -> document(execution, parent, t)).toMap + val docs = graph.relations.map(t => document(execution, parent, t)) documentation.copy(relations = docs) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index c573c087d..0655bc6d0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -32,7 +32,7 @@ class TargetCollector extends Collector { override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val parent = documentation.reference - val docs = graph.targets.map(t => t.target.identifier -> document(execution, parent, t)).toMap + val docs = graph.targets.map(t => document(execution, parent, t)) documentation.copy(targets = docs) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index e24d3845f..6cfc5275e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -133,7 +133,7 @@ final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(p def getVersion() : String = project.version.getOrElse("") def resolve(reference:ReferenceWrapper) : FragmentWrapper = { - val x = project.resolve(reference.reference).map { + project.resolve(reference.reference).map { case m:MappingDoc => MappingDocWrapper(m) case o:MappingOutputDoc => MappingOutputDocWrapper(o) case r:RelationDoc => RelationDocWrapper(r) @@ -145,14 +145,9 @@ final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(p case t:ColumnTest => ColumnTestWrapper(t) case f:Fragment => new FragmentWrapper(f) }.orNull - - if (x == null) - println("null") - - x } - def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.values.map(MappingDocWrapper).toSeq.asJava - def getRelations() : java.util.List[RelationDocWrapper] = project.relations.values.map(RelationDocWrapper).toSeq.asJava - def getTargets() : java.util.List[TargetDocWrapper] = project.targets.values.map(TargetDocWrapper).toSeq.asJava + def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.map(MappingDocWrapper).asJava + def getRelations() : java.util.List[RelationDocWrapper] = project.relations.map(RelationDocWrapper).asJava + def getTargets() : java.util.List[TargetDocWrapper] = project.targets.map(TargetDocWrapper).asJava } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala index e012cd80c..8fa324fd6 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ProjectDocTest.scala @@ -46,7 +46,7 @@ class ProjectDocTest extends AnyFlatSpec with Matchers { val finalOutput = output.copy(schema = Some(schema)) val finalMapping = mapping.copy(outputs = Seq(finalOutput)) - val finalProject = project.copy(mappings = Map(mapping.identifier -> finalMapping)) + val finalProject = project.copy(mappings = Seq(finalMapping)) finalProject.resolve(projectRef) should be (Some(finalProject)) finalProject.resolve(mappingRef) should be (Some(finalMapping)) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala index 2d9b1cbe0..e242733d6 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala @@ -31,8 +31,6 @@ import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.Target -import com.dimajix.flowman.model.TargetIdentifier -import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.SingleValue import com.dimajix.flowman.types.StructType @@ -115,10 +113,10 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { val collector = new RelationCollector() val projectDoc = collector.collect(execution, graph, ProjectDoc(project.name)) - val sourceRelationDoc = projectDoc.relations(RelationIdentifier("project/src")) - val targetRelationDoc = projectDoc.relations(RelationIdentifier("project/tgt")) + val sourceRelationDoc = projectDoc.relations.find(_.identifier == RelationIdentifier("project/src")) + val targetRelationDoc = projectDoc.relations.find(_.identifier == RelationIdentifier("project/tgt")) - sourceRelationDoc should be (RelationDoc( + sourceRelationDoc should be (Some(RelationDoc( parent = Some(ProjectReference("project")), identifier = RelationIdentifier("project/src"), description = Some("source relation"), @@ -126,9 +124,9 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { parent = Some(RelationReference(Some(ProjectReference("project")), "src")) )), partitions = Map("pcol" -> SingleValue("part1")) - )) + ))) - targetRelationDoc should be (RelationDoc( + targetRelationDoc should be (Some(RelationDoc( parent = Some(ProjectReference("project")), identifier = RelationIdentifier("project/tgt"), description = Some("target relation"), @@ -137,6 +135,6 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { )), inputs = Seq(MappingOutputReference(Some(MappingReference(Some(ProjectReference("project")), "m1")), "main")), partitions = Map("outcol" -> SingleValue("part1")) - )) + ))) } } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 93ef18283..13a9e4dc4 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -141,14 +141,19 @@ #end #macro(references $refs) - #foreach($input in ${refs}) + #foreach($input in ${refs}) + #if(${project.resolve($input)}) + #else + + + #end -
${input.kind} ${project.resolve($input)}${input.kind}${input}
#end + #end @@ -160,28 +165,35 @@

Index

+ #if(${project.mappings})

Mappings

+ #end + #if(${project.relations})

Relations

+ #end + #if(${project.targets})

Targets

+ #end
+#if(${project.mappings})

Mappings

#foreach($mapping in ${project.mappings})
@@ -201,7 +213,9 @@
#end +#end +#if(${project.relations})

Relations

#foreach($relation in ${project.relations})
@@ -224,7 +238,9 @@ #schema($relation.schema)
#end +#end +#if(${project.targets})

Targets

#foreach($target in ${project.targets})
@@ -242,4 +258,5 @@ #end
#end +#end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala index 534a44565..2b492fdc7 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -22,6 +22,8 @@ import java.nio.charset.Charset import java.util.Properties import scala.collection.JavaConverters._ +import scala.util.matching.Regex + import com.fasterxml.jackson.annotation.JsonProperty import com.google.common.io.Resources import org.apache.hadoop.fs.Path @@ -43,11 +45,17 @@ object FileGenerator { case class FileGenerator( location:Path, - template:URL = FileGenerator.defaultTemplate -) extends TemplateGenerator(template) { + template:URL = FileGenerator.defaultTemplate, + includeRelations:Seq[Regex] = Seq.empty, + excludeRelations:Seq[Regex] = Seq.empty, + includeMappings:Seq[Regex] = Seq.empty, + excludeMappings:Seq[Regex] = Seq.empty, + includeTargets:Seq[Regex] = Seq.empty, + excludeTargets:Seq[Regex] = Seq.empty +) extends TemplateGenerator(template, includeRelations, excludeRelations, includeMappings, excludeMappings, includeTargets, excludeTargets) { private val logger = LoggerFactory.getLogger(classOf[FileGenerator]) - override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + protected override def generateInternal(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { val props = new Properties() props.load(new StringReader(loadResource("template.properties"))) @@ -94,19 +102,21 @@ case class FileGenerator( } -class FileGeneratorSpec extends GeneratorSpec { +class FileGeneratorSpec extends TemplateGeneratorSpec { @JsonProperty(value="location", required=true) private var location:String = _ @JsonProperty(value="template", required=false) private var template:String = FileGenerator.defaultTemplate.toString override def instantiate(context: Context): Generator = { - val url = context.evaluate(template) match { - case "text" => FileGenerator.textTemplate - case "html" => FileGenerator.htmlTemplate - case str => new URL(str) - } + val url = getTemplateUrl(context) FileGenerator( new Path(context.evaluate(location)), - url + url, + includeRelations = includeRelations.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r), + excludeRelations = excludeRelations.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r), + includeMappings = includeMappings.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r), + excludeMappings = excludeMappings.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r), + includeTargets = includeTargets.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r), + excludeTargets = excludeTargets.map(context.evaluate).map(_.trim).filter(_.nonEmpty).map(_.r) ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala index c283228d3..bc53c09c3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TemplateGenerator.scala @@ -19,9 +19,14 @@ package com.dimajix.flowman.spec.documentation import java.net.URL import java.nio.charset.Charset +import scala.util.matching.Regex + +import com.fasterxml.jackson.annotation.JsonProperty import com.google.common.io.Resources +import org.apache.hadoop.fs.Path import com.dimajix.flowman.documentation.BaseGenerator +import com.dimajix.flowman.documentation.Generator import com.dimajix.flowman.documentation.MappingDoc import com.dimajix.flowman.documentation.MappingDocWrapper import com.dimajix.flowman.documentation.ProjectDoc @@ -32,12 +37,45 @@ import com.dimajix.flowman.documentation.TargetDoc import com.dimajix.flowman.documentation.TargetDocWrapper import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.model.Identifier abstract class TemplateGenerator( - template:URL + template:URL, + includeRelations:Seq[Regex] = Seq(".*".r), + excludeRelations:Seq[Regex] = Seq.empty, + includeMappings:Seq[Regex] = Seq(".*".r), + excludeMappings:Seq[Regex] = Seq.empty, + includeTargets:Seq[Regex] = Seq(".*".r), + excludeTargets:Seq[Regex] = Seq.empty ) extends BaseGenerator { - override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit + override def generate(context:Context, execution: Execution, documentation: ProjectDoc): Unit = { + def checkRegex(id:Identifier[_], regex:Regex) : Boolean = { + regex.unapplySeq(id.toString).nonEmpty || regex.unapplySeq(id.name).nonEmpty + } + // Apply all filters + val relations = documentation.relations.filter { relation => + includeRelations.exists(regex => checkRegex(relation.identifier, regex)) && + !excludeRelations.exists(regex => checkRegex(relation.identifier, regex)) + } + val mappings = documentation.mappings.filter { mapping => + includeMappings.exists(regex => checkRegex(mapping.identifier, regex)) && + !excludeMappings.exists(regex => checkRegex(mapping.identifier, regex)) + } + val targets = documentation.targets.filter { target => + includeTargets.exists(regex => checkRegex(target.identifier, regex)) && + !excludeTargets.exists(regex => checkRegex(target.identifier, regex)) + } + val doc = documentation.copy( + relations = relations, + mappings = mappings, + targets = targets + ) + + generateInternal(context:Context, execution: Execution, doc) + } + + protected def generateInternal(context:Context, execution: Execution, documentation: ProjectDoc): Unit protected def renderProject(context:Context, documentation: ProjectDoc, template:String="project.vtl") : String = { val temp = loadResource(template) @@ -66,3 +104,23 @@ abstract class TemplateGenerator( Resources.toString(url, Charset.forName("UTF-8")) } } + + +abstract class TemplateGeneratorSpec extends GeneratorSpec { + @JsonProperty(value="template", required=false) private var template:String = FileGenerator.defaultTemplate.toString + @JsonProperty(value="includeRelations", required=false) protected var includeRelations:Seq[String] = Seq(".*") + @JsonProperty(value="excludeRelations", required=false) protected var excludeRelations:Seq[String] = Seq.empty + @JsonProperty(value="includeMappings", required=false) protected var includeMappings:Seq[String] = Seq(".*") + @JsonProperty(value="excludeMappings", required=false) protected var excludeMappings:Seq[String] = Seq.empty + @JsonProperty(value="includeTargets", required=false) protected var includeTargets:Seq[String] = Seq(".*") + @JsonProperty(value="excludeTargets", required=false) protected var excludeTargets:Seq[String] = Seq.empty + + protected def getTemplateUrl(context: Context): URL = { + context.evaluate(template) match { + case "text" => FileGenerator.textTemplate + case "html" => FileGenerator.htmlTemplate + case str => new URL(str) + } + } + +} From c7808ed4df3806dd1adf620c692d1d20687c1ebb Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 11:46:46 +0100 Subject: [PATCH 44/95] Improve schema inference for documenting relations --- .../documentation/RelationCollector.scala | 41 +++++++++++++++---- .../flowman/documentation/html/project.vtl | 4 ++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 4089e488d..5ca403145 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -92,14 +92,16 @@ class RelationCollector( val ref = doc.reference val schema = relation.schema.map { schema => - val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) - SchemaDoc( - Some(ref), - schema.description, - fieldsDoc.columns, - Seq() - ) - } + val fieldsDoc = SchemaDoc.ofFields(parent, schema.fields) + SchemaDoc( + Some(ref), + description = schema.description, + columns = fieldsDoc.columns + ) + }.orElse { + // Try to infer schema from input + getInputSchema(execution, ref, node) + } val mergedSchema = { Try { SchemaDoc.ofStruct(ref, relation.describe(execution, partitions)) @@ -123,4 +125,27 @@ class RelationCollector( val executor = new TestExecutor(execution) executor.executeTests(relation, doc) } + + private def getInputSchema(execution:Execution, parent:Reference, node:RelationRef) : Option[SchemaDoc] = { + // Try to infer schema from input + val schema = node.incoming.flatMap { + case write:WriteRelation => + write.input.incoming.flatMap { + case map: InputMapping => + Try { + execution.describe(map.input.mapping, map.pin) + }.toOption + case _ => None + } + case _ => Seq() + }.headOption + + schema.map { schema => + val fieldsDoc = SchemaDoc.ofFields(parent.parent.get, schema.fields) + SchemaDoc( + Some(parent), + columns = fieldsDoc.columns + ) + } + } } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 13a9e4dc4..ca0bc5d64 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -119,6 +119,8 @@ Tests + + #if($schema) #foreach($column in ${schema.columns}) ${column.name} @@ -137,6 +139,8 @@ #end + #end + #end From 414cd1709f16cff91b82efd725e17b65c1e7fc72 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 12:57:21 +0100 Subject: [PATCH 45/95] Update documentation --- docs/documenting/config.md | 8 ++++++++ docs/documenting/index.md | 9 +++++++++ docs/documenting/mappings.md | 1 - 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/documenting/config.md b/docs/documenting/config.md index 0844732a2..c7e265bfb 100644 --- a/docs/documenting/config.md +++ b/docs/documenting/config.md @@ -29,6 +29,14 @@ generators: - ".*/measurements_raw" ``` +## Collectors + +Flowman uses so called *collectors* which create an internal model of the documentation from the core entities like +relations, mappings and build targets. The default configuration uses the three collectors `relations`, `mappings` +and `targets`, with each of them being responsible for one entity type. If you really do not require documentation +for one of these targets, you may want to simply remove the corresponding collector from that list. + + ## File Generator Fields The generator is used for generating the documentation. You can configure multiple generators for creating multiple diff --git a/docs/documenting/index.md b/docs/documenting/index.md index be7c527b2..00112ceb9 100644 --- a/docs/documenting/index.md +++ b/docs/documenting/index.md @@ -26,3 +26,12 @@ Generating the documentation is as easy as running [flowexec](../cli/flowexec.md ```shell flowexec -f my_project_directory documentation generate ``` + +Since generating documentation also requires a job context (which may contain additional parameters and environment +variables), you can also explicitly specify the job which is used for instantiating all entities like relations, +mappings and targets as follows: + +```shell +flowexec -f my_project_directory documentation generate +``` +If no job is specified, Flowman will use the `main` job diff --git a/docs/documenting/mappings.md b/docs/documenting/mappings.md index da75c31b5..b60893407 100644 --- a/docs/documenting/mappings.md +++ b/docs/documenting/mappings.md @@ -36,7 +36,6 @@ mappings: description: "The time when the measurement was made" - name: report_type description: "The report type of the measurement" - description: "The quality indicator of the wind speed. 1 means trustworthy quality." - name: air_temperature description: "The air temperature in degree Celsius" - name: air_temperature_qual From 946a3fc9285e721942aee56683329d9ae7583569 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 19:12:46 +0100 Subject: [PATCH 46/95] Fix some UI issues in Flowman History Server --- .../src/components/JobDetails.vue | 25 ++++++++----------- .../src/components/MetricTable.vue | 12 +++++++-- .../src/components/TargetDetails.vue | 2 +- flowman-server-ui/src/views/Home.vue | 8 +++--- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/flowman-server-ui/src/components/JobDetails.vue b/flowman-server-ui/src/components/JobDetails.vue index 5f3e73396..ac6ff9c0b 100644 --- a/flowman-server-ui/src/components/JobDetails.vue +++ b/flowman-server-ui/src/components/JobDetails.vue @@ -2,7 +2,7 @@ gavel - Job '{{properties.project}}/{{properties.name}}' {{ properties.phase }} id {{job}} status {{properties.status}} + Job '{{ details.project }}/{{ details.job }}' {{ details.phase }} id {{ job }} status {{ details.status }} @@ -80,7 +80,7 @@ {{ p[0] }} : {{ p[1] }} @@ -120,6 +120,8 @@ import EnvironmentTable from '@/components/EnvironmentTable.vue' import MetricTable from '@/components/MetricTable.vue' import moment from "moment"; +let hash = require('object-hash'); + export default { name: 'JobDetails', components: {Status,EnvironmentTable,MetricTable}, @@ -130,7 +132,7 @@ export default { data () { return { - properties: {}, + details: {}, metrics: [], targets: [], environment: [] @@ -144,18 +146,7 @@ export default { methods: { refresh() { this.$api.getJobDetails(this.job).then(response => { - this.properties = { - namespace: response.namespace, - project: response.project, - name: response.job, - args: response.args, - phase: response.phase, - status: response.status, - startDt: response.startDateTime, - endDt: response.endDateTime, - parameters: response.args, - metrics: response.metrics - } + this.details = response this.metrics = response.metrics }) @@ -175,6 +166,10 @@ export default { }, duration(dt) { return moment.duration(dt).humanize() + }, + + hash(obj) { + return hash(obj) } } } diff --git a/flowman-server-ui/src/components/MetricTable.vue b/flowman-server-ui/src/components/MetricTable.vue index 3d57d8f6a..b64b684f7 100644 --- a/flowman-server-ui/src/components/MetricTable.vue +++ b/flowman-server-ui/src/components/MetricTable.vue @@ -17,13 +17,13 @@ {{ item[1].name }} {{ p[0] }} : {{ p[1] }} @@ -36,11 +36,19 @@ diff --git a/flowman-server-ui/src/components/TargetDetails.vue b/flowman-server-ui/src/components/TargetDetails.vue index 9fdee0c91..ca37c592e 100644 --- a/flowman-server-ui/src/components/TargetDetails.vue +++ b/flowman-server-ui/src/components/TargetDetails.vue @@ -2,7 +2,7 @@ gavel - Target '{{details.project}}/{{details.name}}' {{ details.phase }} id {{target}} status {{details.status}} + Target '{{details.project}}/{{details.target}}' {{ details.phase }} id {{target}} status {{details.status}} Jobs - + - + @@ -22,10 +22,10 @@ Targets - + - + From f33b60b320dbcadc2ea4ceaaaa740fb50c23f74d Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 17 Feb 2022 19:12:59 +0100 Subject: [PATCH 47/95] Improve naming of formaly anonymous jobs and targets --- .../com/dimajix/flowman/execution/Runner.scala | 9 ++++++--- .../flowman/spec/documentation/CollectorSpec.scala | 13 +++++++++++-- .../flowman/spec/target/TruncateTarget.scala | 1 + .../flowman/tools/exec/model/PhaseCommand.scala | 7 ++----- .../flowman/tools/exec/target/PhaseCommand.scala | 2 +- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala index bcff74137..9fca652c3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Runner.scala @@ -51,6 +51,8 @@ import com.dimajix.flowman.model.TargetResult import com.dimajix.flowman.model.Test import com.dimajix.flowman.model.TestWrapper import com.dimajix.flowman.spi.LogFilter +import com.dimajix.flowman.types.FieldType +import com.dimajix.flowman.types.LongType import com.dimajix.flowman.util.ConsoleColors._ import com.dimajix.spark.SparkUtils.withJobGroup @@ -613,7 +615,7 @@ final class Runner( * @param phases * @return */ - def executeTargets(targets:Seq[Target], phases:Seq[Phase], force:Boolean, keepGoing:Boolean=false, dryRun:Boolean=false, isolated:Boolean=true) : Status = { + def executeTargets(targets:Seq[Target], phases:Seq[Phase], jobName:String="execute-target", force:Boolean, keepGoing:Boolean=false, dryRun:Boolean=false, isolated:Boolean=true) : Status = { if (targets.nonEmpty) { val context = targets.head.context @@ -623,11 +625,12 @@ final class Runner( .withTargets(targets.map(tgt => (tgt.name, Prototype.of(tgt))).toMap) .build() val job = Job.builder(jobContext) - .setName("execute-target-" + Clock.systemUTC().millis()) + .setName(jobName) .setTargets(targets.map(_.identifier)) + .setParameters(Seq(Job.Parameter("execution_ts", LongType))) .build() - executeJob(job, phases, force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=isolated) + executeJob(job, phases, args=Map("execution_ts" -> Clock.systemUTC().millis()), force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=isolated) } else { Status.SUCCESS diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala index 195fe9ba2..3e60e1a04 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.spec.documentation +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo @@ -38,14 +39,22 @@ abstract class CollectorSpec extends Spec[Collector] { } final class MappingCollectorSpec extends CollectorSpec { + @JsonProperty(value="executeTests", required=true) private var executeTests:String = "true" + override def instantiate(context: Context): MappingCollector = { - new MappingCollector() + new MappingCollector( + context.evaluate(executeTests).toBoolean + ) } } final class RelationCollectorSpec extends CollectorSpec { + @JsonProperty(value="executeTests", required=true) private var executeTests:String = "true" + override def instantiate(context: Context): RelationCollector = { - new RelationCollector() + new RelationCollector( + context.evaluate(executeTests).toBoolean + ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala index 9f1feba97..f58d48250 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TruncateTarget.scala @@ -81,6 +81,7 @@ case class TruncateTarget( project.map(_.name).getOrElse(""), name, phase, + // TODO: Maybe here should be a partition or a list of partitions.... Map() ) } diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/PhaseCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/PhaseCommand.scala index c407fad77..515a182b0 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/PhaseCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/PhaseCommand.scala @@ -59,15 +59,12 @@ class PhaseCommand(phase:Phase) extends Command { project.relations.keys.toSeq val partition = ParserUtils.parseDelimitedKeyValues(this.partition) val targets = toRun.map { rel => - // Create Properties without a project. Otherwise the lookup of the relation will fail, since its identifier - // will refer to the project. And since the relation are not part of the project, this is also really correct - val name = rel + "-" + Clock.systemUTC().millis() - val props = Target.Properties(context.root, name, "relation") + val props = Target.Properties(context.root, rel, "relation") RelationTarget(props, RelationIdentifier(rel, project.name), MappingOutputIdentifier.empty, partition) } val runner = session.runner - runner.executeTargets(targets, Seq(phase), force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=false) + runner.executeTargets(targets, Seq(phase), jobName="cli-tools", force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=false) } } diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/target/PhaseCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/target/PhaseCommand.scala index ee4a61c3d..1c6104738 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/target/PhaseCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/target/PhaseCommand.scala @@ -62,7 +62,7 @@ class PhaseCommand(phase:Phase) extends Command { context.getTarget(TargetIdentifier(t)) } val runner = session.runner - runner.executeTargets(allTargets, lifecycle, force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=false) + runner.executeTargets(allTargets, lifecycle, jobName="cli-tools", force=force, keepGoing=keepGoing, dryRun=dryRun, isolated=false) } } From 7afa94e9ab97ded35b15f3ca916272b27713a263 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 08:24:03 +0100 Subject: [PATCH 48/95] Refactor documentation generator --- docs/documenting/config.md | 2 + examples/weather/documentation.yml | 2 + .../flowman/documentation/ColumnTest.scala | 11 +- .../flowman/documentation/Documenter.scala | 3 +- .../documentation/MappingCollector.scala | 27 +---- .../documentation/RelationCollector.scala | 25 ++--- .../documentation/TargetCollector.scala | 12 +-- .../flowman/documentation/TargetDoc.scala | 14 +-- .../flowman/documentation/TestCollector.scala | 101 ++++++++++++++++++ .../flowman/documentation/TestExecutor.scala | 22 ++-- .../flowman/spi/ColumnTestExecutor.scala | 12 ++- .../documentation/ColumnTestTest.scala | 52 +++++---- .../spec/documentation/CollectorSpec.scala | 23 ++-- .../spec/documentation/MappingDocSpec.scala | 13 +-- .../spec/documentation/RelationDocSpec.scala | 11 +- .../spec/documentation/SchemaDocSpec.scala | 4 +- .../spec/documentation/TargetDocSpec.scala | 5 +- 17 files changed, 204 insertions(+), 135 deletions(-) create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala diff --git a/docs/documenting/config.md b/docs/documenting/config.md index c7e265bfb..f5a6b05cf 100644 --- a/docs/documenting/config.md +++ b/docs/documenting/config.md @@ -15,6 +15,8 @@ collectors: - kind: mappings # Collect documentation of build targets - kind: targets + # Execute all tests + - kind: tests generators: # Create an output file in the project directory diff --git a/examples/weather/documentation.yml b/examples/weather/documentation.yml index 859e0aa69..db6312873 100644 --- a/examples/weather/documentation.yml +++ b/examples/weather/documentation.yml @@ -5,6 +5,8 @@ collectors: - kind: mappings # Collect documentation of build targets - kind: targets + # Execute all tests + - kind: tests generators: # Create an output file in the project directory diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 540104e8e..758fe5342 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -20,6 +20,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame import org.apache.spark.sql.functions.lit +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.spi.ColumnTestExecutor @@ -110,10 +111,10 @@ final case class ValuesColumnTest( class DefaultColumnTestExecutor extends ColumnTestExecutor { - override def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = { + override def execute(execution: Execution, context:Context, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = { test match { case _: NotNullColumnTest => - executePredicateTest(df, column, test, df(column).isNotNull) + executePredicateTest(df, test, df(column).isNotNull) case _: UniqueColumnTest => val agg = df.filter(df(column).isNotNull).groupBy(df(column)).count() val result = agg.filter(agg(agg.columns(1)) > 1).orderBy(agg(agg.columns(1)).desc).limit(6).collect() @@ -122,17 +123,17 @@ class DefaultColumnTestExecutor extends ColumnTestExecutor { case v: ValuesColumnTest => val dt = df.schema(column).dataType val values = v.values.map(v => lit(v).cast(dt)) - executePredicateTest(df.filter(df(column).isNotNull), column, test, df(column).isin(values:_*)) + executePredicateTest(df.filter(df(column).isNotNull), test, df(column).isin(values:_*)) case v: RangeColumnTest => val dt = df.schema(column).dataType val lower = lit(v.lower).cast(dt) val upper = lit(v.upper).cast(dt) - executePredicateTest(df.filter(df(column).isNotNull), column, test, df(column).between(lower, upper)) + executePredicateTest(df.filter(df(column).isNotNull), test, df(column).between(lower, upper)) case _ => None } } - private def executePredicateTest(df: DataFrame, column:String, test:ColumnTest, predicate:Column) : Option[TestResult] = { + private def executePredicateTest(df: DataFrame, test:ColumnTest, predicate:Column) : Option[TestResult] = { val result = df.groupBy(predicate).count().collect() val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala index 8c2c4ebda..effc4f649 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala @@ -41,7 +41,8 @@ object Documenter { val collectors = Seq( new RelationCollector(), new MappingCollector(), - new TargetCollector() + new TargetCollector(), + new TestCollector() ) Documenter( collectors=collectors diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index af9989a14..718c87bde 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -32,9 +32,7 @@ import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.types.StructType -class MappingCollector( - executeTests:Boolean = true -) extends Collector { +class MappingCollector extends Collector { private val logger = LoggerFactory.getLogger(getClass) override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { @@ -84,9 +82,7 @@ class MappingCollector( val doc = MappingDoc( Some(parent), mapping.identifier, - None, - inputs.map(_._2.reference).toSeq, - Seq() + inputs = inputs.map(_._2.reference).toSeq, ) val ref = doc.reference @@ -95,9 +91,7 @@ class MappingCollector( schemas.map { case(output,schema) => val doc = MappingOutputDoc( Some(ref), - MappingOutputIdentifier(mapping.identifier, output), - None, - None + MappingOutputIdentifier(mapping.identifier, output) ) val schemaDoc = SchemaDoc.ofStruct(doc.reference, schema) doc.copy(schema = Some(schemaDoc)) @@ -108,22 +102,11 @@ class MappingCollector( mapping.outputs.map { output => MappingOutputDoc( Some(ref), - MappingOutputIdentifier(mapping.identifier, output), - None, - None + MappingOutputIdentifier(mapping.identifier, output) ) } } - val result = doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) - if (executeTests) - runTests(execution, mapping, result) - else - result - } - - private def runTests(execution: Execution, mapping:Mapping, doc:MappingDoc) : MappingDoc = { - val executor = new TestExecutor(execution) - executor.executeTests(mapping, doc) + doc.copy(outputs=outputs.toSeq).merge(mapping.documentation) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 5ca403145..9bd5b0bda 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -29,13 +29,10 @@ import com.dimajix.flowman.graph.InputMapping import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.graph.RelationRef import com.dimajix.flowman.graph.WriteRelation -import com.dimajix.flowman.model.Relation import com.dimajix.flowman.types.FieldValue -class RelationCollector( - executeTests:Boolean = true -) extends Collector { +class RelationCollector extends Collector { private val logger = LoggerFactory.getLogger(getClass) override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { @@ -83,11 +80,10 @@ class RelationCollector( val doc = RelationDoc( Some(parent), relation.identifier, - relation.description, - None, - inputs, - relation.provides.toSeq, - partitions + description = relation.description, + inputs = inputs, + provides = relation.provides.toSeq, + partitions = partitions ) val ref = doc.reference @@ -114,16 +110,7 @@ class RelationCollector( } } - val result = doc.copy(schema = mergedSchema).merge(relation.documentation) - if (executeTests) - runTests(execution, relation, result) - else - result - } - - private def runTests(execution: Execution, relation:Relation, doc:RelationDoc) : RelationDoc = { - val executor = new TestExecutor(execution) - executor.executeTests(relation, doc) + doc.copy(schema = mergedSchema).merge(relation.documentation) } private def getInputSchema(execution:Execution, parent:Reference, node:RelationRef) : Option[SchemaDoc] = { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index 0655bc6d0..524a0d7a6 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -66,10 +66,9 @@ class TargetCollector extends Collector { val doc = TargetDoc( Some(parent), target.identifier, - target.description, - Seq(), - inputs, - outputs + description = target.description, + inputs = inputs, + outputs = outputs ) val ref = doc.reference @@ -77,9 +76,8 @@ class TargetCollector extends Collector { TargetPhaseDoc( Some(ref), p, - None, - target.provides(p).toSeq, - target.requires(p).toSeq + provides = target.provides(p).toSeq, + requires = target.requires(p).toSeq ) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala index 7cbfb3c85..fb0a20a9f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetDoc.scala @@ -38,9 +38,9 @@ final case class TargetPhaseReference( final case class TargetPhaseDoc( parent:Option[Reference], phase:Phase, - description:Option[String], - provides:Seq[ResourceIdentifier], - requires:Seq[ResourceIdentifier] + description:Option[String] = None, + provides:Seq[ResourceIdentifier] = Seq.empty, + requires:Seq[ResourceIdentifier] = Seq.empty ) extends Fragment { override def reference: Reference = TargetPhaseReference(parent, phase) override def fragments: Seq[Fragment] = Seq() @@ -67,10 +67,10 @@ final case class TargetReference( final case class TargetDoc( parent:Option[Reference], identifier:TargetIdentifier, - description:Option[String], - phases:Seq[TargetPhaseDoc], - inputs:Seq[Reference], - outputs:Seq[Reference] + description:Option[String] = None, + phases:Seq[TargetPhaseDoc] = Seq.empty, + inputs:Seq[Reference] = Seq.empty, + outputs:Seq[Reference] = Seq.empty ) extends EntityDoc { override def reference: TargetReference = TargetReference(parent, identifier.name) override def fragments: Seq[Fragment] = phases diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala new file mode 100644 index 000000000..4df55c67b --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala @@ -0,0 +1,101 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.slf4j.LoggerFactory + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier + + +class TestCollector extends Collector { + private val logger = LoggerFactory.getLogger(getClass) + + /** + * This will execute all tests and change the documentation accordingly + * @param execution + * @param graph + * @param documentation + * @return + */ + override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { + val executor = new TestExecutor(execution) + val mappings = documentation.mappings.map { m => + resolveMapping(graph, m.reference) match { + case None => + // This should not happen - but who knows... + logger.warn(s"Cannot find mapping for document reference '${m.reference.toString}'") + m + case Some(mapping) => + executor.executeTests(mapping, m) + } + } + val relations = documentation.relations.map { r => + resolveRelation(graph, r.reference) match { + case None => + // This should not happen - but who knows... + logger.warn(s"Cannot find relation for document reference '${r.reference.toString}'") + r + case Some(relation) => + executor.executeTests(relation, r) + } + } + + documentation.copy( + mappings = mappings, + relations = relations + ) + } + + /** + * Resolve a mapping via its documentation reference in the graph + * @param graph + * @param ref + * @return + */ + private def resolveMapping(graph: Graph, ref:MappingReference) : Option[Mapping] = { + ref.parent match { + case None => + graph.mappings.find(m => m.name == ref.name).map(_.mapping) + case Some(ProjectReference(project)) => + val id = MappingIdentifier(ref.name, project) + graph.mappings.find(m => m.identifier == id).map(_.mapping) + case _ => None + } + } + + /** + * Resolve a relation via its documentation reference in the graph + * @param graph + * @param ref + * @return + */ + private def resolveRelation(graph: Graph, ref:RelationReference) : Option[Relation] = { + ref.parent match { + case None => + graph.relations.find(m => m.name == ref.name).map(_.relation) + case Some(ProjectReference(project)) => + val id = RelationIdentifier(ref.name, project) + graph.relations.find(m => m.identifier == id).map(_.relation) + case _ => None + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala index 885c045c6..7511d6feb 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala @@ -16,15 +16,13 @@ package com.dimajix.flowman.documentation -import scala.util.Failure -import scala.util.Success -import scala.util.Try import scala.util.control.NonFatal import org.apache.spark.sql.DataFrame import org.slf4j.LoggerFactory import com.dimajix.common.ExceptionUtils.reasons +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation @@ -47,7 +45,7 @@ class TestExecutor(execution: Execution) { logger.info(s"Conducting tests on relation '${relation.identifier}'") try { val df = relation.read(execution, doc.partitions) - runSchemaTests(df, schema) + runSchemaTests(relation.context, df, schema) } catch { case NonFatal(ex) => logger.warn(s"Error executing tests for relation '${relation.identifier}': ${reasons(ex)}") @@ -74,7 +72,7 @@ class TestExecutor(execution: Execution) { logger.info(s"Conducting tests on mapping '${mapping.identifier}'") try { val df = execution.instantiate(mapping, output.name) - runSchemaTests(df, schema) + runSchemaTests(mapping.context, df, schema) } catch { case NonFatal(ex) => logger.warn(s"Error executing tests for mapping '${mapping.identifier}': ${reasons(ex)}") @@ -113,19 +111,19 @@ class TestExecutor(execution: Execution) { column.copy(children=children, tests=tests) } - private def runSchemaTests(df:DataFrame, schema:SchemaDoc) : SchemaDoc = { - val columns = runColumnTests(df, schema.columns) + private def runSchemaTests(context:Context, df:DataFrame, schema:SchemaDoc) : SchemaDoc = { + val columns = runColumnTests(context, df, schema.columns) schema.copy(columns=columns) } - private def runColumnTests(df:DataFrame, columns:Seq[ColumnDoc], path:String = "") : Seq[ColumnDoc] = { - columns.map(col => runColumnTests(df, col, path)) + private def runColumnTests(context:Context, df:DataFrame, columns:Seq[ColumnDoc], path:String = "") : Seq[ColumnDoc] = { + columns.map(col => runColumnTests(context, df, col, path)) } - private def runColumnTests(df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { + private def runColumnTests(context:Context, df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { val columnPath = path + column.name val tests = column.tests.map { test => val result = try { - val result = columnTestExecutors.flatMap(_.execute(execution, df, columnPath, test)).headOption + val result = columnTestExecutors.flatMap(_.execute(execution, context, df, columnPath, test)).headOption result match { case None => logger.warn(s"Could not find appropriate test executor for testing column $columnPath") @@ -141,7 +139,7 @@ class TestExecutor(execution: Execution) { } test.withResult(result) } - val children = runColumnTests(df, column.children, path + column.name + ".") + val children = runColumnTests(context, df, column.children, path + column.name + ".") column.copy(children=children, tests=tests) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala index 68ed556bb..bfc83905d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.DataFrame import com.dimajix.flowman.documentation.ColumnTest import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution @@ -36,5 +37,14 @@ object ColumnTestExecutor { } trait ColumnTestExecutor { - def execute(execution: Execution, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] + /** + * Executes a column test + * @param execution - execution to use + * @param context - context that can be used for resource lookups like relations or mappings + * @param df - DataFrame containing the output to test + * @param column - Path of the column to test + * @param test - Test to execute + * @return + */ + def execute(execution: Execution, context:Context, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala index ad4ea474c..a2e0b53e9 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -29,16 +29,17 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq((Some(1),2), (None,3))) val test = NotNullColumnTest(None) - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_3", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_3", test)) } "A UniqueColumnTest" should "be executable" in { @@ -46,6 +47,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq( @@ -55,13 +57,13 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) val test = UniqueColumnTest(None) - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result3 = testExecutor.execute(execution, df, "_3", test) + val result3 = testExecutor.execute(execution, context, df, "_3", test) result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } "A ValuesColumnTest" should "be executable" in { @@ -69,6 +71,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq( @@ -77,13 +80,13 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) val test = ValuesColumnTest(None, values=Seq(1,2)) - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result3 = testExecutor.execute(execution, df, "_3", test) + val result3 = testExecutor.execute(execution, context, df, "_3", test) result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } it should "use correct data types" in { @@ -91,6 +94,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq( @@ -99,13 +103,13 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) val test = ValuesColumnTest(None, values=Seq(1,2)) - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result3 = testExecutor.execute(execution, df, "_3", test) + val result3 = testExecutor.execute(execution, context, df, "_3", test) result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } "A RangeColumnTest" should "be executable" in { @@ -113,6 +117,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq( @@ -121,13 +126,13 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) val test = RangeColumnTest(None, lower=1, upper=2) - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result3 = testExecutor.execute(execution, df, "_3", test) + val result3 = testExecutor.execute(execution, context, df, "_3", test) result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } it should "use correct data types" in { @@ -135,6 +140,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { .withSparkSession(spark) .build() val execution = session.execution + val context = session.context val testExecutor = new DefaultColumnTestExecutor val df = spark.createDataFrame(Seq( @@ -143,12 +149,12 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) val test = RangeColumnTest(None, lower="1.0", upper="2.2") - val result1 = testExecutor.execute(execution, df, "_1", test) + val result1 = testExecutor.execute(execution, context, df, "_1", test) result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - val result2 = testExecutor.execute(execution, df, "_2", test) + val result2 = testExecutor.execute(execution, context, df, "_2", test) result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) - val result3 = testExecutor.execute(execution, df, "_3", test) + val result3 = testExecutor.execute(execution, context, df, "_3", test) result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) - an[Exception] should be thrownBy(testExecutor.execute(execution, df, "_4", test)) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala index 3e60e1a04..9b4b4e531 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala @@ -16,7 +16,6 @@ package com.dimajix.flowman.spec.documentation -import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo @@ -24,6 +23,7 @@ import com.dimajix.flowman.documentation.Collector import com.dimajix.flowman.documentation.MappingCollector import com.dimajix.flowman.documentation.RelationCollector import com.dimajix.flowman.documentation.TargetCollector +import com.dimajix.flowman.documentation.TestCollector import com.dimajix.flowman.execution.Context import com.dimajix.flowman.spec.Spec @@ -32,29 +32,22 @@ import com.dimajix.flowman.spec.Spec @JsonSubTypes(value = Array( new JsonSubTypes.Type(name = "mappings", value = classOf[MappingCollectorSpec]), new JsonSubTypes.Type(name = "relations", value = classOf[RelationCollectorSpec]), - new JsonSubTypes.Type(name = "targets", value = classOf[TargetCollectorSpec]) + new JsonSubTypes.Type(name = "targets", value = classOf[TargetCollectorSpec]), + new JsonSubTypes.Type(name = "tests", value = classOf[TestCollectorSpec]) )) abstract class CollectorSpec extends Spec[Collector] { override def instantiate(context: Context): Collector } final class MappingCollectorSpec extends CollectorSpec { - @JsonProperty(value="executeTests", required=true) private var executeTests:String = "true" - override def instantiate(context: Context): MappingCollector = { - new MappingCollector( - context.evaluate(executeTests).toBoolean - ) + new MappingCollector() } } final class RelationCollectorSpec extends CollectorSpec { - @JsonProperty(value="executeTests", required=true) private var executeTests:String = "true" - override def instantiate(context: Context): RelationCollector = { - new RelationCollector( - context.evaluate(executeTests).toBoolean - ) + new RelationCollector() } } @@ -63,3 +56,9 @@ final class TargetCollectorSpec extends CollectorSpec { new TargetCollector() } } + +final class TestCollectorSpec extends CollectorSpec { + override def instantiate(context: Context): TestCollector = { + new TestCollector() + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala index 092fc1715..08859a53b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala @@ -79,9 +79,7 @@ class MappingDocSpec extends Spec[MappingDoc] { val doc = MappingDoc( None, MappingIdentifier.empty, - context.evaluate(description), - Seq(), - Seq() + description = context.evaluate(description) ) val ref = doc.reference @@ -89,17 +87,12 @@ class MappingDocSpec extends Spec[MappingDoc] { if (columns.nonEmpty || tests.nonEmpty) { val output = MappingOutputDoc( Some(ref), - MappingOutputIdentifier.empty.copy(output="main"), - None, - None + MappingOutputIdentifier.empty.copy(output="main") ) val ref2 = output.reference val schema = SchemaDoc( - Some(ref2), - None, - Seq(), - Seq() + Some(ref2) ) val ref3 = schema.reference val cols = columns.map(_.instantiate(context, ref3)) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala index 0ac1e007d..dec6b36bb 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala @@ -34,21 +34,14 @@ class RelationDocSpec extends Spec[RelationDoc] { val doc = RelationDoc( None, RelationIdentifier.empty, - context.evaluate(description), - None, - Seq(), - Seq(), - Map() + description = context.evaluate(description) ) val ref = doc.reference val schema = if (columns.nonEmpty || tests.nonEmpty) { val schema = SchemaDoc( - Some(ref), - None, - Seq(), - Seq() + Some(ref) ) val ref2 = schema.reference val cols = columns.map(_.instantiate(context, ref2)) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala index b45eeb24f..38436fd02 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala @@ -31,9 +31,7 @@ class SchemaDocSpec { def instantiate(context: Context, parent:Reference): SchemaDoc = { val doc = SchemaDoc( Some(parent), - context.evaluate(description), - Seq(), - Seq() + description = context.evaluate(description) ) val ref = doc.reference diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala index cd83cd8e6..c31084f7d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/TargetDocSpec.scala @@ -31,10 +31,7 @@ class TargetDocSpec extends Spec[TargetDoc] { val doc = TargetDoc( None, TargetIdentifier.empty, - context.evaluate(description), - Seq(), - Seq(), - Seq() + description = context.evaluate(description) ) doc } From 9a3dd8c49870fec1baa4d2d70643c2a796ed5c14 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 17:29:26 +0100 Subject: [PATCH 49/95] Add more column tests for documentation --- docs/documenting/config.md | 4 +- docs/documenting/tests.md | 15 ++ examples/weather/mapping/aggregates.yml | 14 ++ examples/weather/model/aggregates.yml | 15 ++ .../flowman/documentation/ColumnDoc.scala | 8 + .../flowman/documentation/ColumnTest.scala | 67 +++++++- .../flowman/documentation/MappingDoc.scala | 15 ++ .../documentation/ReferenceResolver.scala | 60 +++++++ .../flowman/documentation/RelationDoc.scala | 7 + .../flowman/documentation/SchemaDoc.scala | 8 + .../flowman/documentation/TestCollector.scala | 43 +---- .../flowman/documentation/velocity.scala | 7 + .../com/dimajix/flowman/graph/Graph.scala | 4 + .../flowman/spi/ColumnTestExecutor.scala | 2 +- .../flowman/documentation/ColumnDocTest.scala | 152 ++++++++++++++++++ .../documentation/ColumnTestTest.scala | 131 ++++++++++++--- .../flowman/documentation/html/project.vtl | 3 +- .../spec/documentation/ColumnTestSpec.scala | 34 +++- .../spec/documentation/ColumnTestTest.scala | 19 +++ 19 files changed, 536 insertions(+), 72 deletions(-) create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/documentation/ReferenceResolver.scala diff --git a/docs/documenting/config.md b/docs/documenting/config.md index f5a6b05cf..2929702c5 100644 --- a/docs/documenting/config.md +++ b/docs/documenting/config.md @@ -25,9 +25,9 @@ generators: # This will exclude all mappings excludeMappings: ".*" excludeRelations: - # You can either specify a name (without the project) + # You can either specify a name or regular expression (without the project) - "stations_raw" - # Or can also explicitly specify a name with the project + # Or can also explicitly specify a name with the project. Note that the entries actually are regular expressions - ".*/measurements_raw" ``` diff --git a/docs/documenting/tests.md b/docs/documenting/tests.md index e756c889d..1e1b9ad97 100644 --- a/docs/documenting/tests.md +++ b/docs/documenting/tests.md @@ -40,6 +40,12 @@ relations: - kind: notNull - kind: values values: [0,1,2,3,4,5,6,7,8,9] + - name: air_temperature + tests: + - kind: expression + expression: "air_temperature >= -100 OR air_temperature_qual <> 1" + - kind: expression + expression: "air_temperature <= 100 OR air_temperature_qual <> 1" ``` ## Available Column Tests @@ -79,3 +85,12 @@ want to specify both `notNUll` and `range`. * `kind` **(mandatory)** *(string)*: `range` * `lower` **(mandatory)** *(string)*: Lower value (inclusive) * `upper` **(mandatory)** *(string)*: Upper value (inclusive) + + +### SQL Expression + +A very flexible test is provided with the SQL expression test. This test allows you to specify any simple SQL expression +(which may also use different columns), which should evaluate to `TRUE` for all records passing the test. + +* `kind` **(mandatory)** *(string)*: `expression` +* `expression` **(mandatory)** *(string)*: Boolean SQL Expression diff --git a/examples/weather/mapping/aggregates.yml b/examples/weather/mapping/aggregates.yml index f4091e1c6..5b0f482c8 100644 --- a/examples/weather/mapping/aggregates.yml +++ b/examples/weather/mapping/aggregates.yml @@ -22,5 +22,19 @@ mappings: - kind: unique - name: min_wind_speed description: Minimum wind speed + tests: + - kind: expression + expression: "min_wind_speed >= 0" - name: max_wind_speed description: Maximum wind speed + tests: + - kind: expression + expression: "max_wind_speed <= 60" + - name: min_temperature + tests: + - kind: expression + expression: "min_temperature >= -100" + - name: max_temperature + tests: + - kind: expression + expression: "max_temperature <= 100" diff --git a/examples/weather/model/aggregates.yml b/examples/weather/model/aggregates.yml index a5c405c24..ca85e9de1 100644 --- a/examples/weather/model/aggregates.yml +++ b/examples/weather/model/aggregates.yml @@ -33,3 +33,18 @@ relations: type: FLOAT - name: avg_temperature type: FLOAT + + documentation: + columns: + - name: min_wind_speed + tests: + - kind: expression + expression: "min_wind_speed >= 0" + - name: min_temperature + tests: + - kind: expression + expression: "min_temperature >= -100" + - name: max_temperature + tests: + - kind: expression + expression: "max_temperature <= 100" diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 7b6f06e7e..293bee278 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -34,6 +34,14 @@ final case class ColumnReference( } } override def kind : String = "column" + + def sql : String = { + parent match { + case Some(schema:SchemaReference) => schema.sql + "." + name + case Some(col:ColumnReference) => col.sql + "." + name + case _ => name + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 758fe5342..96317fccc 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -18,10 +18,14 @@ package com.dimajix.flowman.documentation import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.expr import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.types.BooleanType import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.spi.ColumnTestExecutor @@ -106,8 +110,35 @@ final case class ValuesColumnTest( } } -//case class ForeignKeyColumnTest() extends ColumnTest -//case class ExpressionColumnTest() extends ColumnTest +case class ForeignKeyColumnTest( + parent:Option[Reference], + description: Option[String] = None, + relation: Option[RelationIdentifier] = None, + mapping: Option[MappingOutputIdentifier] = None, + column: Option[String] = None, + result:Option[TestResult] = None +) extends ColumnTest { + override def name : String = s"FOREIGN KEY (${column.getOrElse("")}) REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")}" + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ForeignKeyColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} + +case class ExpressionColumnTest( + parent:Option[Reference], + description: Option[String] = None, + expression: String, + result:Option[TestResult] = None +) extends ColumnTest { + override def name: String = expression + override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) + override def reparent(parent: Reference): ExpressionColumnTest = { + val ref = ColumnTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} class DefaultColumnTestExecutor extends ColumnTestExecutor { @@ -115,20 +146,43 @@ class DefaultColumnTestExecutor extends ColumnTestExecutor { test match { case _: NotNullColumnTest => executePredicateTest(df, test, df(column).isNotNull) + case _: UniqueColumnTest => val agg = df.filter(df(column).isNotNull).groupBy(df(column)).count() - val result = agg.filter(agg(agg.columns(1)) > 1).orderBy(agg(agg.columns(1)).desc).limit(6).collect() - val status = if (result.isEmpty) TestStatus.SUCCESS else TestStatus.FAILED - Some(TestResult(Some(test.reference), status, None, None)) + val result = agg.groupBy(agg(agg.columns(1)) > 1).count().collect() + val numSuccess = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) + val numFailed = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) + val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val description = s"$numSuccess values are unique, $numFailed values are non-unique" + Some(TestResult(Some(test.reference), status, Some(description))) + case v: ValuesColumnTest => val dt = df.schema(column).dataType val values = v.values.map(v => lit(v).cast(dt)) executePredicateTest(df.filter(df(column).isNotNull), test, df(column).isin(values:_*)) + case v: RangeColumnTest => val dt = df.schema(column).dataType val lower = lit(v.lower).cast(dt) val upper = lit(v.upper).cast(dt) executePredicateTest(df.filter(df(column).isNotNull), test, df(column).between(lower, upper)) + + case v: ExpressionColumnTest => + executePredicateTest(df, test, expr(v.expression).cast(BooleanType)) + + case f:ForeignKeyColumnTest => + val otherDf = + f.relation.map { rel => + val relation = context.getRelation(rel) + relation.read(execution) + }.orElse(f.mapping.map { map=> + val mapping = context.getMapping(map.mapping) + execution.instantiate(mapping, map.output) + }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test of column '$column' in test ${test.reference.toString}")) + val otherColumn = f.column.getOrElse(column) + val joined = df.join(otherDf, df(column) === otherDf(otherColumn), "left") + executePredicateTest(joined.filter(df(column).isNotNull), test,otherDf(otherColumn).isNotNull) + case _ => None } } @@ -138,6 +192,7 @@ class DefaultColumnTestExecutor extends ColumnTestExecutor { val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS - Some(TestResult(Some(test.reference), status, None, None)) + val description = s"$numSuccess records passed, $numFailed records failed" + Some(TestResult(Some(test.reference), status, Some(description))) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala index f5b24a0f9..870927aaa 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingDoc.scala @@ -31,6 +31,14 @@ final case class MappingOutputReference( } } override def kind : String = "mapping_output" + + def sql : String = { + parent match { + case Some(MappingReference(Some(ProjectReference(project)), mapping)) => s"$project/[$mapping:$name]" + case Some(p:MappingReference) => s"[${p.sql}:$name]" + case _ => s"[:$name]" + } + } } @@ -113,6 +121,13 @@ final case class MappingReference( } } override def kind: String = "mapping" + + def sql : String = { + parent match { + case Some(ProjectReference(project)) => project + "/" + name + case _ => name + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ReferenceResolver.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ReferenceResolver.scala new file mode 100644 index 000000000..abc921911 --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ReferenceResolver.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier + + +class ReferenceResolver(graph:Graph) { + /** + * Resolve a mapping via its documentation reference in the graph + * @param graph + * @param ref + * @return + */ + def resolve(ref:MappingReference) : Option[Mapping] = { + ref.parent match { + case None => + graph.mappings.find(m => m.name == ref.name).map(_.mapping) + case Some(ProjectReference(project)) => + val id = MappingIdentifier(ref.name, project) + graph.mappings.find(m => m.identifier == id).map(_.mapping) + case _ => None + } + } + + /** + * Resolve a relation via its documentation reference in the graph + * @param graph + * @param ref + * @return + */ + def resolve(ref:RelationReference) : Option[Relation] = { + ref.parent match { + case None => + graph.relations.find(m => m.name == ref.name).map(_.relation) + case Some(ProjectReference(project)) => + val id = RelationIdentifier(ref.name, project) + graph.relations.find(m => m.identifier == id).map(_.relation) + case _ => None + } + } +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index 6237086bf..7ebdc15b2 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -40,6 +40,13 @@ final case class RelationReference( } } override def kind : String = "relation" + + def sql : String = { + parent match { + case Some(ProjectReference(project)) => project + "/" + name + case _ => name + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index 11c30fbaf..47eecffd7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -36,6 +36,14 @@ final case class SchemaReference( } } override def kind : String = "schema" + + def sql : String = { + parent match { + case Some(rel:RelationReference) => rel.sql + case Some(map:MappingOutputReference) => map.sql + case _ => "" + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala index 4df55c67b..5006ddf47 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala @@ -20,10 +20,6 @@ import org.slf4j.LoggerFactory import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph -import com.dimajix.flowman.model.Mapping -import com.dimajix.flowman.model.MappingIdentifier -import com.dimajix.flowman.model.Relation -import com.dimajix.flowman.model.RelationIdentifier class TestCollector extends Collector { @@ -37,9 +33,10 @@ class TestCollector extends Collector { * @return */ override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { + val resolver = new ReferenceResolver(graph) val executor = new TestExecutor(execution) val mappings = documentation.mappings.map { m => - resolveMapping(graph, m.reference) match { + resolver.resolve(m.reference) match { case None => // This should not happen - but who knows... logger.warn(s"Cannot find mapping for document reference '${m.reference.toString}'") @@ -49,7 +46,7 @@ class TestCollector extends Collector { } } val relations = documentation.relations.map { r => - resolveRelation(graph, r.reference) match { + resolver.resolve(r.reference) match { case None => // This should not happen - but who knows... logger.warn(s"Cannot find relation for document reference '${r.reference.toString}'") @@ -64,38 +61,4 @@ class TestCollector extends Collector { relations = relations ) } - - /** - * Resolve a mapping via its documentation reference in the graph - * @param graph - * @param ref - * @return - */ - private def resolveMapping(graph: Graph, ref:MappingReference) : Option[Mapping] = { - ref.parent match { - case None => - graph.mappings.find(m => m.name == ref.name).map(_.mapping) - case Some(ProjectReference(project)) => - val id = MappingIdentifier(ref.name, project) - graph.mappings.find(m => m.identifier == id).map(_.mapping) - case _ => None - } - } - - /** - * Resolve a relation via its documentation reference in the graph - * @param graph - * @param ref - * @return - */ - private def resolveRelation(graph: Graph, ref:RelationReference) : Option[Relation] = { - ref.parent match { - case None => - graph.relations.find(m => m.name == ref.name).map(_.relation) - case Some(ProjectReference(project)) => - val id = RelationIdentifier(ref.name, project) - graph.relations.find(m => m.identifier == id).map(_.relation) - case _ => None - } - } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 6cfc5275e..8b8f86829 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -26,6 +26,13 @@ final case class ReferenceWrapper(reference:Reference) { def getParent() : ReferenceWrapper = reference.parent.map(ReferenceWrapper).orNull def getKind() : String = reference.kind + def getSql() : String = reference match { + case m:MappingReference => m.sql + case m:RelationReference => m.sql + case m:ColumnReference => m.sql + case m:SchemaReference => m.sql + case _ => "" + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala index 59c5d94e0..f7462c606 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala @@ -32,6 +32,10 @@ import com.dimajix.flowman.model.TargetIdentifier object Graph { + def empty(context:Context) : Graph = { + Graph(context, Seq.empty, Seq.empty, Seq.empty) + } + /** * Creates a Graph from a given project. The [[Context]] required for lookups and instantiation is retrieved from * the given [[Session]] diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala index bfc83905d..f4dd910be 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala @@ -20,13 +20,13 @@ import java.util.ServiceLoader import scala.collection.JavaConverters._ -import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame import com.dimajix.flowman.documentation.ColumnTest import com.dimajix.flowman.documentation.TestResult import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.graph.Graph object ColumnTestExecutor { diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala index acb40f279..b36db9d37 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnDocTest.scala @@ -19,6 +19,9 @@ package com.dimajix.flowman.documentation import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.types.DoubleType import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.NullType @@ -79,4 +82,153 @@ class ColumnDocTest extends AnyFlatSpec with Matchers { ) )) } + + it should "support sql with no parent" in { + val doc = ColumnDoc( + None, + Field("col1", NullType, description = Some("Some desc 1")) + ) + val doc2 = doc.copy(children = Seq( + ColumnDoc(Some(doc.reference), Field("child1", StringType)), + ColumnDoc(Some(doc.reference), Field("child2", StringType)) + )) + + doc2.reference.sql should be ("col1") + doc2.children(0).reference.sql should be ("col1.child1") + doc2.children(1).reference.sql should be ("col1.child2") + } + + it should "support sql with a relation parent" in { + val doc0 = RelationDoc( + None, + RelationIdentifier("project/rel1") + ) + val doc1 = SchemaDoc( + Some(doc0.reference) + ) + val doc2 = ColumnDoc( + Some(doc1.reference), + Field("col1", NullType, description = Some("Some desc 1")) + ) + val doc2p = doc2.copy(children = Seq( + ColumnDoc(Some(doc2.reference), Field("child1", StringType)), + ColumnDoc(Some(doc2.reference), Field("child2", StringType)) + )) + val doc1p = doc1.copy( + columns = Seq(doc2p) + ) + val doc0p = doc0.copy( + schema = Some(doc1p) + ) + + doc0p.schema.get.columns(0).reference.sql should be ("rel1.col1") + doc0p.schema.get.columns(0).children(0).reference.sql should be ("rel1.col1.child1") + doc0p.schema.get.columns(0).children(1).reference.sql should be ("rel1.col1.child2") + } + + it should "support sql with a relation parent and a project" in { + val doc0 = ProjectDoc("project") + val doc1 = RelationDoc( + Some(doc0.reference), + RelationIdentifier("project/rel1") + ) + val doc2 = SchemaDoc( + Some(doc1.reference) + ) + val doc3 = ColumnDoc( + Some(doc2.reference), + Field("col1", NullType, description = Some("Some desc 1")) + ) + val doc3p = doc3.copy(children = Seq( + ColumnDoc(Some(doc3.reference), Field("child1", StringType)), + ColumnDoc(Some(doc3.reference), Field("child2", StringType)) + )) + val doc2p = doc2.copy( + columns = Seq(doc3p) + ) + val doc1p = doc1.copy( + schema = Some(doc2p) + ) + val doc0p = doc0.copy( + relations = Seq(doc1p) + ) + + doc0p.relations(0).schema.get.columns(0).reference.sql should be ("project/rel1.col1") + doc0p.relations(0).schema.get.columns(0).children(0).reference.sql should be ("project/rel1.col1.child1") + doc0p.relations(0).schema.get.columns(0).children(1).reference.sql should be ("project/rel1.col1.child2") + } + + it should "support sql with a mapping parent and a no project" in { + val doc1 = MappingDoc( + None, + MappingIdentifier("project/map1") + ) + val doc2 = MappingOutputDoc( + Some(doc1.reference), + MappingOutputIdentifier("project/map1:lala") + ) + val doc3 = SchemaDoc( + Some(doc2.reference) + ) + val doc4 = ColumnDoc( + Some(doc3.reference), + Field("col1", NullType, description = Some("Some desc 1")) + ) + val doc4p = doc4.copy(children = Seq( + ColumnDoc(Some(doc4.reference), Field("child1", StringType)), + ColumnDoc(Some(doc4.reference), Field("child2", StringType)) + )) + val doc3p = doc3.copy( + columns = Seq(doc4p) + ) + val doc2p = doc2.copy( + schema = Some(doc3p) + ) + val doc1p = doc1.copy( + outputs = Seq(doc2p) + ) + + doc1p.outputs(0).schema.get.columns(0).reference.sql should be ("[map1:lala].col1") + doc1p.outputs(0).schema.get.columns(0).children(0).reference.sql should be ("[map1:lala].col1.child1") + doc1p.outputs(0).schema.get.columns(0).children(1).reference.sql should be ("[map1:lala].col1.child2") + } + + it should "support sql with a mapping parent and a project" in { + val doc0 = ProjectDoc("project") + val doc1 = MappingDoc( + Some(doc0.reference), + MappingIdentifier("project/map1") + ) + val doc2 = MappingOutputDoc( + Some(doc1.reference), + MappingOutputIdentifier("project/map1:lala") + ) + val doc3 = SchemaDoc( + Some(doc2.reference) + ) + val doc4 = ColumnDoc( + Some(doc3.reference), + Field("col1", NullType, description = Some("Some desc 1")) + ) + val doc4p = doc4.copy(children = Seq( + ColumnDoc(Some(doc4.reference), Field("child1", StringType)), + ColumnDoc(Some(doc4.reference), Field("child2", StringType)) + )) + val doc3p = doc3.copy( + columns = Seq(doc4p) + ) + val doc2p = doc2.copy( + schema = Some(doc3p) + ) + val doc1p = doc1.copy( + outputs = Seq(doc2p) + ) + val doc0p = doc0.copy( + mappings = Seq(doc1p) + ) + + doc0p.mappings(0).outputs(0).schema.get.columns(0).reference.sql should be ("project/[map1:lala].col1") + doc0p.mappings(0).outputs(0).schema.get.columns(0).children(0).reference.sql should be ("project/[map1:lala].col1.child1") + doc0p.mappings(0).outputs(0).schema.get.columns(0).children(1).reference.sql should be ("project/[map1:lala].col1.child2") + } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala index a2e0b53e9..4c1d28b07 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -16,14 +16,21 @@ package com.dimajix.flowman.documentation +import org.apache.spark.storage.StorageLevel +import org.scalamock.scalatest.MockFactory import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype import com.dimajix.spark.testing.LocalSparkSession -class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { +class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { "A NotNullColumnTest" should "be executable" in { val session = Session.builder() .withSparkSession(spark) @@ -36,9 +43,9 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = NotNullColumnTest(None) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_3", test)) } @@ -58,11 +65,11 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = UniqueColumnTest(None) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 values are unique, 0 values are non-unique")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 values are unique, 1 values are non-unique")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("3 values are unique, 0 values are non-unique")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -81,11 +88,11 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = ValuesColumnTest(None, values=Seq(1,2)) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -104,11 +111,11 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = ValuesColumnTest(None, values=Seq(1,2)) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -127,11 +134,11 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = RangeColumnTest(None, lower=1, upper=2) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -150,11 +157,99 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with LocalSparkSession { val test = RangeColumnTest(None, lower="1.0", upper="2.2") val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED))) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS))) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + } + + "An ExpressionColumnTest" should "succeed" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val context = session.context + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = ExpressionColumnTest(None, expression="_2 > _3") + val result1 = testExecutor.execute(execution, context, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + val result2 = testExecutor.execute(execution, context, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + val result4 = testExecutor.execute(execution, context, df, "_4", test) + result4 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + } + + it should "fail" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val context = session.context + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test = ExpressionColumnTest(None, expression="_2 < _3") + val result1 = testExecutor.execute(execution, context, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + val result2 = testExecutor.execute(execution, context, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + val result4 = testExecutor.execute(execution, context, df, "_4", test) + result4 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + } + + "A ForeignKeyColumnTest" should "work" in { + val mappingSpec = mock[Prototype[Mapping]] + val mapping = mock[Mapping] + + val session = Session.builder() + .withSparkSession(spark) + .build() + val project = Project( + name = "project", + mappings = Map("mapping" -> mappingSpec) + ) + val context = session.getContext(project) + val execution = session.execution + + val testExecutor = new DefaultColumnTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),1,1), + (None,2,3) + )) + val otherDf = spark.createDataFrame(Seq( + (1,1), + (2,2) + )) + + (mappingSpec.instantiate _).expects(*).returns(mapping) + (mapping.context _).expects().returns(context) + (mapping.inputs _).expects().returns(Seq()) + (mapping.outputs _).expects().atLeastOnce().returns(Seq("main")) + (mapping.broadcast _).expects().returns(false) + (mapping.cache _).expects().returns(StorageLevel.NONE) + (mapping.checkpoint _).expects().returns(false) + (mapping.identifier _).expects().returns(MappingIdentifier("project/mapping")) + (mapping.execute _).expects(*,*).returns(Map("main" -> otherDf)) + + val test = ForeignKeyColumnTest(None, mapping=Some(MappingOutputIdentifier("mapping")), column=Some("_1")) + val result1 = testExecutor.execute(execution, context, df, "_1", test) + result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + val result2 = testExecutor.execute(execution, context, df, "_2", test) + result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + val result3 = testExecutor.execute(execution, context, df, "_3", test) + result3 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index ca0bc5d64..ad31a94d6 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -114,7 +114,7 @@ Column Name Data Type - Attributes + Constraints Description Tests @@ -133,6 +133,7 @@ ${test.name} ${test.status} + #if(${test.result})${test.result.description}#end #end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala index e1db805c2..3d9a71840 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala @@ -23,11 +23,15 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo import com.dimajix.common.TypeRegistry import com.dimajix.flowman.documentation.ColumnReference import com.dimajix.flowman.documentation.ColumnTest +import com.dimajix.flowman.documentation.ExpressionColumnTest +import com.dimajix.flowman.documentation.ForeignKeyColumnTest import com.dimajix.flowman.documentation.NotNullColumnTest import com.dimajix.flowman.documentation.RangeColumnTest import com.dimajix.flowman.documentation.UniqueColumnTest import com.dimajix.flowman.documentation.ValuesColumnTest import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.spec.annotation.ColumnTestType import com.dimajix.flowman.spi.ClassAnnotationHandler @@ -38,6 +42,8 @@ object ColumnTestSpec extends TypeRegistry[ColumnTestSpec] { @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") @JsonSubTypes(value = Array( + new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionColumnTestSpec]), + new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeyColumnTestSpec]), new JsonSubTypes.Type(name = "notNull", value = classOf[NotNullColumnTestSpec]), new JsonSubTypes.Type(name = "unique", value = classOf[UniqueColumnTestSpec]), new JsonSubTypes.Type(name = "range", value = classOf[RangeColumnTestSpec]), @@ -57,16 +63,16 @@ class ColumnTestSpecAnnotationHandler extends ClassAnnotationHandler { class NotNullColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = NotNullColumnTest(Some(parent)) + override def instantiate(context: Context, parent:ColumnReference): NotNullColumnTest = NotNullColumnTest(Some(parent)) } class UniqueColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = UniqueColumnTest(Some(parent)) + override def instantiate(context: Context, parent:ColumnReference): UniqueColumnTest = UniqueColumnTest(Some(parent)) } class RangeColumnTestSpec extends ColumnTestSpec { @JsonProperty(value="lower", required=true) private var lower:String = "" @JsonProperty(value="upper", required=true) private var upper:String = "" - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = RangeColumnTest( + override def instantiate(context: Context, parent:ColumnReference): RangeColumnTest = RangeColumnTest( Some(parent), None, context.evaluate(lower), @@ -76,8 +82,28 @@ class RangeColumnTestSpec extends ColumnTestSpec { class ValuesColumnTestSpec extends ColumnTestSpec { @JsonProperty(value="values", required=false) private var values:Seq[String] = Seq() - override def instantiate(context: Context, parent:ColumnReference): ColumnTest = ValuesColumnTest( + override def instantiate(context: Context, parent:ColumnReference): ValuesColumnTest = ValuesColumnTest( Some(parent), values=values.map(context.evaluate) ) } +class ExpressionColumnTestSpec extends ColumnTestSpec { + @JsonProperty(value="expression", required=true) private var expression:String = _ + + override def instantiate(context: Context, parent:ColumnReference): ExpressionColumnTest = ExpressionColumnTest( + Some(parent), + expression=context.evaluate(expression) + ) +} +class ForeignKeyColumnTestSpec extends ColumnTestSpec { + @JsonProperty(value="mapping", required=false) private var mapping:Option[String] = None + @JsonProperty(value="relation", required=false) private var relation:Option[String] = None + @JsonProperty(value="column", required=false) private var column:Option[String] = None + + override def instantiate(context: Context, parent:ColumnReference): ForeignKeyColumnTest = ForeignKeyColumnTest( + Some(parent), + relation=context.evaluate(relation).map(RelationIdentifier(_)), + mapping=context.evaluate(mapping).map(MappingOutputIdentifier(_)), + column=context.evaluate(column) + ) +} diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala index b10d30bf4..a327edf47 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala @@ -20,6 +20,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.documentation.ColumnReference +import com.dimajix.flowman.documentation.ExpressionColumnTest import com.dimajix.flowman.documentation.RangeColumnTest import com.dimajix.flowman.documentation.UniqueColumnTest import com.dimajix.flowman.documentation.ValuesColumnTest @@ -81,4 +82,22 @@ class ColumnTestTest extends AnyFlatSpec with Matchers { values = Seq("a", "12", null) )) } + + "A ExpressionColumnTest" should "be deserializable" in { + val yaml = + """ + |kind: expression + |expression: "col1 < col2" + """.stripMargin + + val spec = ObjectMapper.parse[ColumnTestSpec](yaml) + spec shouldBe a[ExpressionColumnTestSpec] + + val context = RootContext.builder().build() + val test = spec.instantiate(context, ColumnReference(None, "col0")) + test should be (ExpressionColumnTest( + Some(ColumnReference(None, "col0")), + expression = "col1 < col2" + )) + } } From 232c0deb11cea6417c80449911a26b3d15b6d6c5 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 18:25:10 +0100 Subject: [PATCH 50/95] Implement schema tests for documentation --- examples/weather/model/measurements.yml | 10 ++ examples/weather/model/stations.yml | 7 ++ .../flowman/documentation/ColumnTest.scala | 4 +- .../flowman/documentation/SchemaTest.scala | 100 ++++++++++++++++-- .../flowman/documentation/TestExecutor.scala | 32 +++++- .../flowman/documentation/velocity.scala | 10 ++ .../flowman/spi/SchemaTestExecutor.scala | 3 +- .../flowman/documentation/html/project.vtl | 20 ++++ .../spec/documentation/SchemaTestSpec.scala | 43 +++++++- 9 files changed, 216 insertions(+), 13 deletions(-) diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index 409b36574..e5cd84408 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -41,3 +41,13 @@ relations: - kind: notNull - kind: values values: [0,1,2,3,4,5,6,7,8,9] + # Schema Tests, which might involve multiple columns + tests: + kind: foreignKey + relation: stations + columns: + - usaf + - wban + references: + - usaf + - wban diff --git a/examples/weather/model/stations.yml b/examples/weather/model/stations.yml index c7f21f0d7..95a70dee8 100644 --- a/examples/weather/model/stations.yml +++ b/examples/weather/model/stations.yml @@ -6,3 +6,10 @@ relations: schema: kind: avro file: "${project.basedir}/schema/stations.avsc" + + documentation: + tests: + kind: primaryKey + columns: + - usaf + - wban diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 96317fccc..84535c84f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -110,7 +110,7 @@ final case class ValuesColumnTest( } } -case class ForeignKeyColumnTest( +final case class ForeignKeyColumnTest( parent:Option[Reference], description: Option[String] = None, relation: Option[RelationIdentifier] = None, @@ -126,7 +126,7 @@ case class ForeignKeyColumnTest( } } -case class ExpressionColumnTest( +final case class ExpressionColumnTest( parent:Option[Reference], description: Option[String] = None, expression: String, diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala index dd65cde5e..fa4c013f5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -16,14 +16,20 @@ package com.dimajix.flowman.documentation +import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.types.BooleanType +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.spi.SchemaTestExecutor final case class SchemaTestReference( - override val parent:Option[SchemaReference] + override val parent:Option[Reference] ) extends Reference { override def toString: String = { parent match { @@ -35,21 +41,103 @@ final case class SchemaTestReference( } -sealed abstract class SchemaTest extends Fragment with Product with Serializable { +abstract class SchemaTest extends Fragment with Product with Serializable { + def name : String def result : Option[TestResult] def withResult(result:TestResult) : SchemaTest - override def parent: Option[SchemaReference] + override def parent: Option[Reference] override def reference: SchemaTestReference = SchemaTestReference(parent) override def fragments: Seq[Fragment] = result.toSeq override def reparent(parent: Reference): SchemaTest } +final case class PrimaryKeySchemaTest( + parent:Option[Reference], + description: Option[String] = None, + columns:Seq[String] = Seq.empty, + result:Option[TestResult] = None +) extends SchemaTest { + override def name : String = s"PRIMARY KEY(${columns.mkString(",")})" + override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) + override def reparent(parent: Reference): PrimaryKeySchemaTest = { + val ref = SchemaTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} -//case class ExpressionSchemaTest( -//) extends SchemaTest +final case class ForeignKeySchemaTest( + parent:Option[Reference], + description: Option[String] = None, + columns: Seq[String] = Seq.empty, + relation: Option[RelationIdentifier] = None, + mapping: Option[MappingOutputIdentifier] = None, + references: Seq[String] = Seq.empty, + result:Option[TestResult] = None +) extends SchemaTest { + override def name : String = s"FOREIGN KEY (${columns.mkString(",")}) REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")}(${references.mkString(",")})" + override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) + override def reparent(parent: Reference): ForeignKeySchemaTest = { + val ref = SchemaTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} + +final case class ExpressionSchemaTest( + parent:Option[Reference], + description: Option[String] = None, + expression: String, + result:Option[TestResult] = None +) extends SchemaTest { + override def name: String = expression + override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) + override def reparent(parent: Reference): ExpressionSchemaTest = { + val ref = SchemaTestReference(Some(parent)) + copy(parent=Some(parent), result=result.map(_.reparent(ref))) + } +} class DefaultSchemaTestExecutor extends SchemaTestExecutor { - override def execute(execution: Execution, df: DataFrame, test: SchemaTest): Option[TestResult] = ??? + override def execute(execution: Execution, context:Context, df: DataFrame, test: SchemaTest): Option[TestResult] = { + test match { + case p:PrimaryKeySchemaTest => + val cols = p.columns.map(df(_)) + val agg = df.filter(cols.map(_.isNotNull).reduce(_ && _)).groupBy(cols:_*).count() + val result = agg.groupBy(agg(agg.columns(cols.length)) > 1).count().collect() + val numSuccess = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) + val numFailed = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) + val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val description = s"$numSuccess keys are unique, $numFailed keys are non-unique" + Some(TestResult(Some(test.reference), status, Some(description))) + + case f:ForeignKeySchemaTest => + val otherDf = + f.relation.map { rel => + val relation = context.getRelation(rel) + relation.read(execution) + }.orElse(f.mapping.map { map=> + val mapping = context.getMapping(map.mapping) + execution.instantiate(mapping, map.output) + }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test ${test.reference.toString}")) + val cols = f.columns.map(df(_)) + val otherCols = f.references.map(otherDf(_)) + val joined = df.join(otherDf, cols.zip(otherCols).map(lr => lr._1 === lr._2).reduce(_ && _), "left") + executePredicateTest(joined, test, otherCols.map(_.isNotNull).reduce(_ || _)) + + case e:ExpressionSchemaTest => + executePredicateTest(df, test, expr(e.expression).cast(BooleanType)) + + case _ => None + } + } + + private def executePredicateTest(df: DataFrame, test:SchemaTest, predicate:Column) : Option[TestResult] = { + val result = df.groupBy(predicate).count().collect() + val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) + val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) + val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val description = s"$numSuccess records passed, $numFailed records failed" + Some(TestResult(Some(test.reference), status, Some(description))) + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala index 7511d6feb..44570d1d9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala @@ -27,11 +27,13 @@ import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation import com.dimajix.flowman.spi.ColumnTestExecutor +import com.dimajix.flowman.spi.SchemaTestExecutor class TestExecutor(execution: Execution) { private val logger = LoggerFactory.getLogger(getClass) private val columnTestExecutors = ColumnTestExecutor.executors + private val schemaTestExecutors = SchemaTestExecutor.executors /** * Executes all tests for a relation as defined within the documentation @@ -97,7 +99,11 @@ class TestExecutor(execution: Execution) { private def failSchemaTests(schema:SchemaDoc) : SchemaDoc = { val columns = failColumnTests(schema.columns) - schema.copy(columns=columns) + val tests = schema.tests.map { test => + val result = TestResult(Some(test.reference), status = TestStatus.ERROR) + test.withResult(result) + } + schema.copy(columns=columns, tests=tests) } private def failColumnTests(columns:Seq[ColumnDoc]) : Seq[ColumnDoc] = { columns.map(col => failColumnTests(col)) @@ -113,7 +119,28 @@ class TestExecutor(execution: Execution) { private def runSchemaTests(context:Context, df:DataFrame, schema:SchemaDoc) : SchemaDoc = { val columns = runColumnTests(context, df, schema.columns) - schema.copy(columns=columns) + val tests = schema.tests.map { test => + logger.info(s" - Executing schema test '${test.name}'") + val result = + try { + val result = schemaTestExecutors.flatMap(_.execute(execution, context, df, test)).headOption + result match { + case None => + logger.warn(s"Could not find appropriate test executor for testing schema") + TestResult(Some(test.reference), status = TestStatus.NOT_RUN) + case Some(result) => + result.reparent(test.reference) + } + } catch { + case NonFatal(ex) => + logger.warn(s"Error executing column test: ${reasons(ex)}") + TestResult(Some(test.reference), status = TestStatus.ERROR) + + } + test.withResult(result) + } + + schema.copy(columns=columns, tests=tests) } private def runColumnTests(context:Context, df:DataFrame, columns:Seq[ColumnDoc], path:String = "") : Seq[ColumnDoc] = { columns.map(col => runColumnTests(context, df, col, path)) @@ -121,6 +148,7 @@ class TestExecutor(execution: Execution) { private def runColumnTests(context:Context, df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { val columnPath = path + column.name val tests = column.tests.map { test => + logger.info(s" - Executing test '${test.name}' on column ${columnPath}") val result = try { val result = columnTestExecutors.flatMap(_.execute(execution, context, df, columnPath, test)).headOption diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 8b8f86829..81bb2786a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -73,8 +73,18 @@ final case class ColumnDocWrapper(column:ColumnDoc) extends FragmentWrapper(colu } +final case class SchemaTestWrapper(test:SchemaTest) extends FragmentWrapper(test) { + override def toString: String = test.name + + def getName() : String = test.name + def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull + def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") +} + + final case class SchemaDocWrapper(schema:SchemaDoc) extends FragmentWrapper(schema) { def getColumns() : java.util.List[ColumnDocWrapper] = schema.columns.map(ColumnDocWrapper).asJava + def getTests() : java.util.List[SchemaTestWrapper] = schema.tests.map(SchemaTestWrapper).asJava } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala index 00837f734..255e3900b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.DataFrame import com.dimajix.flowman.documentation.SchemaTest import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution @@ -35,5 +36,5 @@ object SchemaTestExecutor { } trait SchemaTestExecutor { - def execute(execution: Execution, df:DataFrame, test:SchemaTest) : Option[TestResult] + def execute(execution: Execution, context:Context, df:DataFrame, test:SchemaTest) : Option[TestResult] } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index ad31a94d6..63c3adceb 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -143,6 +143,26 @@ #end +#if($schema.tests) + + + + + + + + + + #foreach($test in ${schema.tests}) + + + + + + #end + +
Schema TestResultRemarks
${test.name}${test.status}#if(${test.result})${test.result.description}#end
+#end #end #macro(references $refs) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala index f36bba463..897f310de 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala @@ -16,13 +16,19 @@ package com.dimajix.flowman.spec.documentation +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import com.dimajix.common.TypeRegistry +import com.dimajix.flowman.documentation.ExpressionSchemaTest +import com.dimajix.flowman.documentation.ForeignKeySchemaTest +import com.dimajix.flowman.documentation.PrimaryKeySchemaTest import com.dimajix.flowman.documentation.SchemaReference import com.dimajix.flowman.documentation.SchemaTest import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.spec.annotation.SchemaTestType import com.dimajix.flowman.spi.ClassAnnotationHandler @@ -33,17 +39,50 @@ object SchemaTestSpec extends TypeRegistry[SchemaTestSpec] { @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") @JsonSubTypes(value = Array( - new JsonSubTypes.Type(name = "file", value = classOf[FileGeneratorSpec]) + new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionSchemaTestSpec]), + new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeySchemaTestSpec]), + new JsonSubTypes.Type(name = "primaryKey", value = classOf[PrimaryKeySchemaTestSpec]) )) abstract class SchemaTestSpec { def instantiate(context: Context, parent:SchemaReference): SchemaTest } - class SchemaTestSpecAnnotationHandler extends ClassAnnotationHandler { override def annotation: Class[_] = classOf[SchemaTestType] override def register(clazz: Class[_]): Unit = SchemaTestSpec.register(clazz.getAnnotation(classOf[SchemaTestType]).kind(), clazz.asInstanceOf[Class[_ <: SchemaTestSpec]]) } + + +class PrimaryKeySchemaTestSpec extends SchemaTestSpec { + @JsonProperty(value="columns", required=false) private var columns:Seq[String] = Seq.empty + + override def instantiate(context: Context, parent:SchemaReference): PrimaryKeySchemaTest = PrimaryKeySchemaTest( + Some(parent), + columns = columns.map(context.evaluate) + ) +} +class ExpressionSchemaTestSpec extends SchemaTestSpec { + @JsonProperty(value="expression", required=true) private var expression:String = _ + + override def instantiate(context: Context, parent:SchemaReference): ExpressionSchemaTest = ExpressionSchemaTest( + Some(parent), + expression = context.evaluate(expression) + ) +} +class ForeignKeySchemaTestSpec extends SchemaTestSpec { + @JsonProperty(value="mapping", required=false) private var mapping:Option[String] = None + @JsonProperty(value="relation", required=false) private var relation:Option[String] = None + @JsonProperty(value="columns", required=false) private var columns:Seq[String] = Seq.empty + @JsonProperty(value="references", required=false) private var references:Seq[String] = Seq.empty + + override def instantiate(context: Context, parent:SchemaReference): ForeignKeySchemaTest = ForeignKeySchemaTest( + Some(parent), + columns=columns.map(context.evaluate), + relation=context.evaluate(relation).map(RelationIdentifier(_)), + mapping=context.evaluate(mapping).map(MappingOutputIdentifier(_)), + references=references.map(context.evaluate) + ) +} From 4e0138997ba0098cc2db18e40d2ca42e675bbd3e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 20:02:31 +0100 Subject: [PATCH 51/95] Add new 'document' target for generating documentation within a normal job --- .../{deltaVacuum.md => delta-vacuum.md} | 0 docs/spec/target/document.md | 59 +++++++++ examples/weather/job/main.yml | 1 + examples/weather/mapping/measurements.yml | 2 +- examples/weather/model/aggregates.yml | 12 ++ examples/weather/model/measurements.yml | 12 ++ examples/weather/model/stations.yml | 1 + examples/weather/project.yml | 7 ++ examples/weather/target/documentation.yml | 5 + .../flowman/documentation/Documenter.scala | 5 +- .../documentation/MappingCollector.scala | 2 +- .../flowman/documentation/html/project.vtl | 20 +-- .../documentation/DocumenterLoader.scala | 9 +- .../flowman/spec/target/DocumentTarget.scala | 114 ++++++++++++++++++ .../flowman/spec/target/TargetSpec.scala | 2 + .../spec/documentation/DocumenterTest.scala | 54 +++++++++ .../spec/target/DocumentTargetTest.scala | 59 +++++++++ .../exec/documentation/GenerateCommand.scala | 3 +- 18 files changed, 349 insertions(+), 18 deletions(-) rename docs/spec/target/{deltaVacuum.md => delta-vacuum.md} (100%) create mode 100644 docs/spec/target/document.md create mode 100644 examples/weather/target/documentation.yml rename {flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec => flowman-spec/src/main/scala/com/dimajix/flowman/spec}/documentation/DocumenterLoader.scala (82%) create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DocumentTarget.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala diff --git a/docs/spec/target/deltaVacuum.md b/docs/spec/target/delta-vacuum.md similarity index 100% rename from docs/spec/target/deltaVacuum.md rename to docs/spec/target/delta-vacuum.md diff --git a/docs/spec/target/document.md b/docs/spec/target/document.md new file mode 100644 index 000000000..349e5ba55 --- /dev/null +++ b/docs/spec/target/document.md @@ -0,0 +1,59 @@ +# Document Target + +The `document` (or equivalently `documentation`) target is used to build a documentation of the current project. +You can find more details about that feature in the [documentation section](../../documenting/index.md). You can either +generate the project documentation via `flowexec documentation generate`, or you also generate the documentation via +this special target, which will be executed as part of the `VERIFY` phsae (after the `BUILD` phase has finished). + +## Example + +```yaml +targets: + documentation: + kind: documentation + collectors: + # Collect documentation of relations + - kind: relations + # Collect documentation of mappings + - kind: mappings + # Collect documentation of build targets + - kind: targets + # Execute all tests + - kind: tests + + generators: + # Create an output file in the project directory + - kind: file + location: ${project.basedir}/generated-documentation + template: html + excludeRelations: + # You can either specify a name (without the project) + - "stations_raw" + # Or can also explicitly specify a name with the project + - ".*/measurements_raw" +``` + +## Fields + +* `kind` **(mandatory)** *(type: string)*: `documentation` or `document` + +* `description` **(optional)** *(type: string)*: + Optional descriptive text of the build target + +* `collectors` **(optional)** *(type: list:collector)*: + List of documentation collectors + +* `generators` **(optional)** *(type: list:generator)*: + List of documentation generators + + +## Configuration + +When no explicit configuration is provided via `generators` or `collectors`, then Flowman will use the +[documentation configuration](../../documenting/config.md) provided in `documentation.yml`. If that file does not +exist, Flowman will fall back to some default configuration, which creates a html based documentation in a +subdirectory `generated-documentation` within the projects base directory. + + +## Supported Phases +* `VERIFY` - This will generate the documentation diff --git a/examples/weather/job/main.yml b/examples/weather/job/main.yml index 5424d77e6..b3e8a034d 100644 --- a/examples/weather/job/main.yml +++ b/examples/weather/job/main.yml @@ -13,3 +13,4 @@ jobs: - stations - aggregates - validate_stations_raw + - documentation diff --git a/examples/weather/mapping/measurements.yml b/examples/weather/mapping/measurements.yml index ca96ea6bc..5fdbc9eab 100644 --- a/examples/weather/mapping/measurements.yml +++ b/examples/weather/mapping/measurements.yml @@ -16,7 +16,7 @@ mappings: date: "TO_DATE(SUBSTR(raw_data,16,8), 'yyyyMMdd')" time: "SUBSTR(raw_data,24,4)" report_type: "SUBSTR(raw_data,42,5)" - wind_direction: "SUBSTR(raw_data,61,3)" + wind_direction: "CAST(SUBSTR(raw_data,61,3) AS INT)" wind_direction_qual: "SUBSTR(raw_data,64,1)" wind_observation: "SUBSTR(raw_data,65,1)" wind_speed: "CAST(CAST(SUBSTR(raw_data,66,4) AS FLOAT)/10 AS FLOAT)" diff --git a/examples/weather/model/aggregates.yml b/examples/weather/model/aggregates.yml index ca85e9de1..6740ce84d 100644 --- a/examples/weather/model/aggregates.yml +++ b/examples/weather/model/aggregates.yml @@ -35,7 +35,14 @@ relations: type: FLOAT documentation: + description: "The aggregate table contains min/max temperature value per year and country" columns: + - name: country + tests: + - kind: notNull + - name: year + tests: + - kind: notNull - name: min_wind_speed tests: - kind: expression @@ -48,3 +55,8 @@ relations: tests: - kind: expression expression: "max_temperature <= 100" + tests: + kind: primaryKey + columns: + - country + - year diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index e5cd84408..c54364ae4 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -19,11 +19,15 @@ relations: documentation: description: "This model contains all individual measurements" + # This section contains additional documentation to the columns, including some simple test cases columns: - name: year description: "The year of the measurement, used for partitioning the data" tests: - kind: notNull + - kind: range + lower: 1901 + upper: 2022 - name: usaf tests: - kind: notNull @@ -36,6 +40,14 @@ relations: - name: time tests: - kind: notNull + - name: wind_direction_qual + tests: + - kind: notNull + - name: wind_direction + tests: + - kind: notNull + - kind: expression + expression: "(wind_direction >= 0 AND wind_direction <= 360) OR wind_direction_qual <> 1" - name: air_temperature_qual tests: - kind: notNull diff --git a/examples/weather/model/stations.yml b/examples/weather/model/stations.yml index 95a70dee8..d87b730d1 100644 --- a/examples/weather/model/stations.yml +++ b/examples/weather/model/stations.yml @@ -1,6 +1,7 @@ relations: stations: kind: file + description: "The 'stations' table contains meta data on all weather stations" format: parquet location: "$basedir/stations/" schema: diff --git a/examples/weather/project.yml b/examples/weather/project.yml index 45979cd4e..ee1536669 100644 --- a/examples/weather/project.yml +++ b/examples/weather/project.yml @@ -1,6 +1,13 @@ name: "weather" version: "1.0" +description: " + This is a simple but very comprehensive example project for Flowman using publicly available weather data. + The project will demonstrate many features of Flowman, like reading and writing data, performing data transformations, + joining, filtering and aggregations. The project will also create a meaningful documentation containing data quality + tests. + " +# The following modules simply contain a list of subdirectories containing the specification files modules: - model - mapping diff --git a/examples/weather/target/documentation.yml b/examples/weather/target/documentation.yml new file mode 100644 index 000000000..7f0949506 --- /dev/null +++ b/examples/weather/target/documentation.yml @@ -0,0 +1,5 @@ +targets: + # This target will create a documentation in the VERIFY phase + documentation: + kind: documentation + # We do not specify any additional configuration, so the project's documentation.yml file will be used diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala index effc4f649..57779e3b8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala @@ -102,11 +102,12 @@ final case class Documenter( } } } - private def execute(context:Context, execution: Execution, project:Project) : Unit = { + def execute(context:Context, execution: Execution, project:Project) : Unit = { // 1. Get Project documentation val projectDoc = ProjectDoc( project.name, - project.version + version = project.version, + description = project.description ) // 2. Apply all other collectors diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index 718c87bde..b085abeca 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -82,7 +82,7 @@ class MappingCollector extends Collector { val doc = MappingDoc( Some(parent), mapping.identifier, - inputs = inputs.map(_._2.reference).toSeq, + inputs = inputs.map(_._2.reference).toSeq ) val ref = doc.reference diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 63c3adceb..8271f706c 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -97,6 +97,9 @@ padding: 20px; border-radius: 10px; } + div.description { + padding: 10px 20px; + } div.bubble { display: inline; @@ -185,7 +188,8 @@

Flowman Project '${project.name}' version ${project.version}

-
Description: ${project.description}
+
Description: ${project.description}
+
Generated at ${Timestamp.now()}
@@ -224,15 +228,15 @@

Mapping '${mapping.identifier}'

-
Description: ${mapping.description}
+
Description: ${mapping.description}

Inputs

#references(${mapping.inputs})

Outputs

#foreach($output in ${mapping.outputs}) -
${output.name}
-
Description: ${output.description}
+

Output '${output.name}'

+ #if(${output.description})
Description: ${output.description}
#end #schema($output.schema) #end
@@ -245,8 +249,8 @@ #foreach($relation in ${project.relations})
-

Relation '${relation.identifier}'

-
Description: ${relation.description}
+

Relation '${relation.identifier}'

+
Description: ${relation.description}

Resources

@@ -270,8 +274,8 @@ #foreach($target in ${project.targets})
-

Target '${target.identifier}'

-
Description: ${target.description}
+

Target '${target.identifier}'

+
Description: ${target.description}

Inputs

#references(${target.inputs}) diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterLoader.scala similarity index 82% rename from flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterLoader.scala index 2796b1cac..9ff1d727a 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/DocumenterLoader.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/DocumenterLoader.scala @@ -14,18 +14,17 @@ * limitations under the License. */ -package com.dimajix.flowman.tools.exec.documentation +package com.dimajix.flowman.spec.documentation import org.apache.hadoop.fs.Path import com.dimajix.flowman.documentation.Documenter import com.dimajix.flowman.execution.Context import com.dimajix.flowman.model.Project -import com.dimajix.flowman.spec.documentation.FileGenerator object DocumenterLoader { - def load(project:Project, context:Context) : Documenter = { + def load(context: Context, project: Project): Documenter = { project.basedir.flatMap { basedir => val docpath = basedir / "documentation.yml" if (docpath.isFile()) { @@ -40,10 +39,10 @@ object DocumenterLoader { } } - private def defaultDocumenter(outputDir:Path) : Documenter = { + private def defaultDocumenter(outputDir: Path): Documenter = { val generators = Seq( new FileGenerator(outputDir) ) - Documenter.read.default().copy(generators=generators) + Documenter.read.default().copy(generators = generators) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DocumentTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DocumentTarget.scala new file mode 100644 index 000000000..811a7ecfa --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/DocumentTarget.scala @@ -0,0 +1,114 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.target + +import com.fasterxml.jackson.annotation.JsonProperty +import org.slf4j.LoggerFactory + +import com.dimajix.common.No +import com.dimajix.common.Trilean +import com.dimajix.common.Yes +import com.dimajix.flowman.documentation.Collector +import com.dimajix.flowman.documentation.Documenter +import com.dimajix.flowman.documentation.Generator +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.model.BaseTarget +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.model.Target +import com.dimajix.flowman.spec.documentation.CollectorSpec +import com.dimajix.flowman.spec.documentation.DocumenterLoader +import com.dimajix.flowman.spec.documentation.GeneratorSpec + + +case class DocumentTarget( + instanceProperties:Target.Properties, + collectors:Seq[Collector] = Seq(), + generators:Seq[Generator] = Seq() +) extends BaseTarget { + private val logger = LoggerFactory.getLogger(getClass) + + /** + * Returns all phases which are implemented by this target in the execute method + * @return + */ + override def phases : Set[Phase] = Set(Phase.VERIFY) + + /** + * Returns a list of physical resources required by this target + * @return + */ + override def requires(phase: Phase) : Set[ResourceIdentifier] = Set() + + /** + * Returns the state of the target, specifically of any artifacts produces. If this method return [[Yes]], + * then an [[execute]] should update the output, such that the target is not 'dirty' any more. + * @param execution + * @param phase + * @return + */ + override def dirty(execution: Execution, phase: Phase) : Trilean = { + phase match { + case Phase.VERIFY => Yes + case _ => No + } + } + + /** + * Build the documentation target + * + * @param execution + */ + override def verify(execution:Execution) : Unit = { + require(execution != null) + + project match { + case Some(project) => + document(execution, project) + case None => + logger.warn("Cannot generator documentation without project") + } + } + + private def document(execution:Execution, project:Project) : Unit = { + val documenter = + if (collectors.isEmpty && generators.isEmpty) { + DocumenterLoader.load(context, project) + } + else { + Documenter(collectors, generators) + } + + documenter.execute(context, execution, project) + } +} + + +class DocumentTargetSpec extends TargetSpec { + @JsonProperty(value="collectors") private var collectors: Seq[CollectorSpec] = Seq() + @JsonProperty(value="generators") private var generators: Seq[GeneratorSpec] = Seq() + + override def instantiate(context: Context): DocumentTarget = { + DocumentTarget( + instanceProperties(context), + collectors.map(_.instantiate(context)), + generators.map(_.instantiate(context)) + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala index 793c90fec..1e86d53e6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/TargetSpec.scala @@ -49,6 +49,8 @@ object TargetSpec extends TypeRegistry[TargetSpec] { new JsonSubTypes.Type(name = "copyFile", value = classOf[CopyFileTargetSpec]), new JsonSubTypes.Type(name = "count", value = classOf[CountTargetSpec]), new JsonSubTypes.Type(name = "deleteFile", value = classOf[DeleteFileTargetSpec]), + new JsonSubTypes.Type(name = "document", value = classOf[DocumentTargetSpec]), + new JsonSubTypes.Type(name = "documentation", value = classOf[DocumentTargetSpec]), new JsonSubTypes.Type(name = "drop", value = classOf[DropTargetSpec]), new JsonSubTypes.Type(name = "file", value = classOf[FileTargetSpec]), new JsonSubTypes.Type(name = "getFile", value = classOf[GetFileTargetSpec]), diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala new file mode 100644 index 000000000..ca2e51dda --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.documentation + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.spec.ObjectMapper + + +class DocumenterTest extends AnyFlatSpec with Matchers { + "A DocumenterSpec" should "be parsable" in { + val yaml = + """ + |collectors: + | # Collect documentation of relations + | - kind: relations + | # Collect documentation of mappings + | - kind: mappings + | # Collect documentation of build targets + | - kind: targets + | # Execute all tests + | - kind: tests + | + |generators: + | # Create an output file in the project directory + | - kind: file + | location: ${project.basedir}/generated-documentation + | template: html + | excludeRelations: + | # You can either specify a name (without the project) + | - "stations_raw" + | # Or can also explicitly specify a name with the project + | - ".*/measurements_raw" + |""".stripMargin + + val spec = ObjectMapper.parse[DocumenterSpec](yaml) + spec shouldBe a[DocumenterSpec] + } +} diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala new file mode 100644 index 000000000..fece1a85b --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.target + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.model.Module + + +class DocumentTargetTest extends AnyFlatSpec with Matchers { + "A DocumentTarget" should "be parseable" in { + val spec = + """ + |targets: + | docu: + | kind: document + | collectors: + | # Collect documentation of relations + | - kind: relations + | # Collect documentation of mappings + | - kind: mappings + | # Collect documentation of build targets + | - kind: targets + | # Execute all tests + | - kind: tests + | + | generators: + | # Create an output file in the project directory + | - kind: file + | location: ${project.basedir}/generated-documentation + | template: html + | excludeRelations: + | # You can either specify a name (without the project) + | - "stations_raw" + | # Or can also explicitly specify a name with the project + | - ".*/measurements_raw" + |""".stripMargin + + val module = Module.read.string(spec) + val target = module.targets("docu") + target shouldBe an[DocumentTargetSpec] + } + +} diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala index 6523ef6af..f82cd3d38 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/documentation/GenerateCommand.scala @@ -32,6 +32,7 @@ import com.dimajix.flowman.execution.Status import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.JobIdentifier import com.dimajix.flowman.model.Project +import com.dimajix.flowman.spec.documentation.DocumenterLoader import com.dimajix.flowman.tools.exec.Command @@ -58,7 +59,7 @@ class GenerateCommand extends Command { } private def generateDoc(session: Session, project:Project, job:Job, args:Map[String,Any]) : Status = { - val documenter = DocumenterLoader.load(project, job.context) + val documenter = DocumenterLoader.load(job.context, project) try { documenter.execute(session, job, args) Status.SUCCESS From b3b955d9bc03dce0d6a2eae95fc4f76ca7941db7 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 20:08:04 +0100 Subject: [PATCH 52/95] Update documentation --- docs/documenting/index.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/documenting/index.md b/docs/documenting/index.md index 00112ceb9..e8eabbbeb 100644 --- a/docs/documenting/index.md +++ b/docs/documenting/index.md @@ -19,7 +19,7 @@ Although Flowman will generate many valuable documentation bits by inspecting th documentation will override any automatically inferred information. -### Generating Documentation +### Generating Documentation via Command Line Generating the documentation is as easy as running [flowexec](../cli/flowexec.md) as follows: @@ -35,3 +35,34 @@ mappings and targets as follows: flowexec -f my_project_directory documentation generate ``` If no job is specified, Flowman will use the `main` job + + +### Generating Documentation via Build Target + +The section above descirbes how to explicitly generate the project documentation by invoking +`flowexec documentation generate`. As an alternative, Flowman offers a [document](../spec/target/document.md) +targets, which allows one to generate the documentation during the `VERIFY` phase (after the `BUILD` phase has +finished) of a normal Flowman project. + +This can be easily configured as follows + +```yaml +targets: + # This target will create a documentation in the VERIFY phase + doc: + kind: documentation + # We do not specify any additional configuration, so the project's documentation.yml file will be used +``` + +Then you only need to add that build target `doc` to your job as follows: + +```yaml +jobs: + main: + targets: + # List all targets which should be built as part of the `main` job + - measurements + - ... + # Finally add the "doc" job for generating the documentation + - doc +``` From 7cfe7c99eb46f5a9f0def53f0d0172365de67f69 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 20:24:22 +0100 Subject: [PATCH 53/95] Minor documentation update --- docs/spec/template/connection.md | 13 ++++++++++++- docs/spec/template/index.md | 14 ++++++++++++++ docs/spec/template/mapping.md | 5 +++++ docs/spec/template/relation.md | 6 ++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/spec/template/connection.md b/docs/spec/template/connection.md index 9f2c333a2..e9896a5d8 100644 --- a/docs/spec/template/connection.md +++ b/docs/spec/template/connection.md @@ -5,6 +5,7 @@ # All template definitions (independent of their kind) go into the templates section templates: default_connection: + # The template is a connection template kind: connection parameters: - name: dir @@ -49,5 +50,15 @@ relations: * FROM line_item li " - ``` + +Once a relation template is defined, you can create instances of the template at any place where a connection can be +specified. You need to use the special syntax `template/` when creating an instance of the template. +The template instance then can also contain values for all parameters defined in the template. + + +## Fields + +* `kind` **(mandatory)** *(type: string)*: `connection` +* `parameters` **(optional)** *(type: list[parameter])*: list of parameter definitions. +* `template` **(mandatory)** *(type: mapping)*: The actual definition of a connection. diff --git a/docs/spec/template/index.md b/docs/spec/template/index.md index 7bfdef7ea..bc10a4256 100644 --- a/docs/spec/template/index.md +++ b/docs/spec/template/index.md @@ -12,17 +12,24 @@ There are some differences when creating an instance of a template as opposed to ## Example ```yaml +# The 'templates' section contains template definitions for relation, mappings, connections and more templates: + # Define a new template called "key_value" key_value: + # The template is a mapping template kind: mapping + # Specify a list of template parameters, which then can be provided during instantiation parameters: - name: key type: string - name: value type: int default: 12 + # Now comes the template definition itself. template: + # Specify the kind within the "mapping" entitiy class kind: values + # The following settings are all specific to the "values" mapping kind records: - ["$key",$value] schema: @@ -33,10 +40,17 @@ templates: - name: value_column type: integer +# Now we can use the "key_value" template in the "mappings" section mappings: + # First instance mapping_1: + # You need to prefix the template name with "template/" kind: template/key_value + # Provide a value for the "key" parameter. + # The "value" parameter has a default value, so it doesn't need to be provided key: some_value + + # Second instance mapping_2: kind: template/key_value key: some_other_value diff --git a/docs/spec/template/mapping.md b/docs/spec/template/mapping.md index b924c5c0a..f8637e645 100644 --- a/docs/spec/template/mapping.md +++ b/docs/spec/template/mapping.md @@ -5,7 +5,9 @@ # All template definitions (independent of their kind) go into the templates section templates: key_value: + # The template is a mapping template kind: mapping + # Specify a list of template parameters, which then can be provided during instantiation parameters: - name: key type: string @@ -28,9 +30,12 @@ templates: # Now you can create instances of the template in the corresponding entity section or at any other place where # a mapping is allowed mappings: + # First instance mapping_1: kind: template/key_value key: some_value + + # Second instance mapping_2: kind: template/key_value key: some_other_value diff --git a/docs/spec/template/relation.md b/docs/spec/template/relation.md index dc4cc1e0b..0022d15a7 100644 --- a/docs/spec/template/relation.md +++ b/docs/spec/template/relation.md @@ -5,7 +5,9 @@ # All template definitions (independent of their kind) go into the templates section templates: key_value: + # The template is a relation template kind: relation + # Specify a list of template parameters, which then can be provided during instantiation parameters: - name: key type: string @@ -28,9 +30,12 @@ templates: # Now you can create instances of the template in the corresponding entity section or at any other place where # a relation is allowed relation: + # First instance source_1: kind: template/key_value key: some_value + + # Second instance source_2: kind: template/key_value key: some_other_value @@ -41,6 +46,7 @@ Once a relation template is defined, you can create instances of the template at specified. You need to use the special syntax `template/` when creating an instance of the template. The template instance then can also contain values for all parameters defined in the template. + ## Fields * `kind` **(mandatory)** *(type: string)*: `relation` From a2f18518ead11145eb653c5747124cc9130156a9 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 20:26:22 +0100 Subject: [PATCH 54/95] Minor documentation update --- docs/spec/target/document.md | 34 ++++++++++++++++---------------- docs/spec/template/connection.md | 16 +++++++-------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/spec/target/document.md b/docs/spec/target/document.md index 349e5ba55..539423ab8 100644 --- a/docs/spec/target/document.md +++ b/docs/spec/target/document.md @@ -12,25 +12,25 @@ targets: documentation: kind: documentation collectors: - # Collect documentation of relations - - kind: relations - # Collect documentation of mappings - - kind: mappings - # Collect documentation of build targets - - kind: targets - # Execute all tests - - kind: tests + # Collect documentation of relations + - kind: relations + # Collect documentation of mappings + - kind: mappings + # Collect documentation of build targets + - kind: targets + # Execute all tests + - kind: tests generators: - # Create an output file in the project directory - - kind: file - location: ${project.basedir}/generated-documentation - template: html - excludeRelations: - # You can either specify a name (without the project) - - "stations_raw" - # Or can also explicitly specify a name with the project - - ".*/measurements_raw" + # Create an output file in the project directory + - kind: file + location: ${project.basedir}/generated-documentation + template: html + excludeRelations: + # You can either specify a name (without the project) + - "stations_raw" + # Or can also explicitly specify a name with the project + - ".*/measurements_raw" ``` ## Fields diff --git a/docs/spec/template/connection.md b/docs/spec/template/connection.md index e9896a5d8..81eb4b872 100644 --- a/docs/spec/template/connection.md +++ b/docs/spec/template/connection.md @@ -33,14 +33,14 @@ relations: dir: /opt/flowman/derby_new table: "advertiser_setting" schema: - kind: embedded - fields: - - name: id - type: Integer - - name: business_rule_id - type: Integer - - name: rtb_advertiser_id - type: Integer + kind: embedded + fields: + - name: id + type: Integer + - name: business_rule_id + type: Integer + - name: rtb_advertiser_id + type: Integer rel_2: kind: jdbc From 21fb52754d1838cf60c979a2a50cba5241303b8d Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 20:46:07 +0100 Subject: [PATCH 55/95] More documentation updates --- docs/cookbook/kerberos.md | 2 +- docs/cookbook/metrics.md | 3 +++ docs/cookbook/validation.md | 13 +++++++++++++ docs/lifecycle.md | 4 ++-- docs/spec/target/blackhole.md | 4 +++- docs/spec/target/compare.md | 4 +++- docs/spec/target/console.md | 5 ++++- docs/spec/target/copy-file.md | 4 +++- docs/spec/target/copy.md | 4 +++- docs/spec/target/count.md | 6 ++++-- docs/spec/target/delete-file.md | 4 +++- docs/spec/target/delta-vacuum.md | 4 +++- docs/spec/target/document.md | 4 +++- docs/spec/target/drop.md | 4 +++- docs/spec/target/file.md | 12 +++++++----- docs/spec/target/hive-database.md | 4 +++- docs/spec/target/local.md | 12 +++++++----- docs/spec/target/measure.md | 6 ++++++ docs/spec/target/merge-files.md | 4 +++- docs/spec/target/merge.md | 4 +++- docs/spec/target/null.md | 4 +++- docs/spec/target/relation.md | 4 +++- docs/spec/target/sftp-upload.md | 6 ++++++ docs/spec/target/template.md | 4 ++++ docs/spec/target/truncate.md | 4 +++- docs/spec/target/validate.md | 4 +++- docs/spec/target/verify.md | 4 +++- 27 files changed, 105 insertions(+), 32 deletions(-) diff --git a/docs/cookbook/kerberos.md b/docs/cookbook/kerberos.md index a98a37271..ee031ffb6 100644 --- a/docs/cookbook/kerberos.md +++ b/docs/cookbook/kerberos.md @@ -14,7 +14,7 @@ KRB_PRINCIPAL={{KRB_PRINCIPAL}}@MY-REALM.NET KRB_KEYTAB=$FLOWMAN_CONF_DIR/{{KRB_PRINCIPAL}}.keytab ``` -Of course this way, Flowman will always use the same Kerberos principal for all projects. Currently there is no other +Of course this way, Flowman will always use the same Kerberos principal for all projects. Currently, there is no other way, since Spark and Hadoop need to have the Kerberos principal set at startup. But you can simply use different config directories and switch between them by setting the `FLOWMAN_CONF_DIR` environment variable. diff --git a/docs/cookbook/metrics.md b/docs/cookbook/metrics.md index 84892d3c0..2246ea5bc 100644 --- a/docs/cookbook/metrics.md +++ b/docs/cookbook/metrics.md @@ -21,7 +21,10 @@ jobs: targets: - my_target - my_other_target + # The following section configures the metric board, which selects the Flowman metrics of interest and also + # maps the Flowman metric names to possibly different names metrics: + # Define labels which are attached to all published metrics below labels: force: ${force} status: ${status} diff --git a/docs/cookbook/validation.md b/docs/cookbook/validation.md index 2e6505b34..1b9d69895 100644 --- a/docs/cookbook/validation.md +++ b/docs/cookbook/validation.md @@ -48,3 +48,16 @@ The example above will validate assumptions on `some_table` mapping, which reads All `validate` targets are executed during the [VALIDATE](../lifecycle.md) phase, which is executed before any other build phase. If one of these targets fail, Flowman will stop execution on return an error. This helps to prevent building invalid data. + + +## Verification + +In addition to *validating* data quality before a FLowman job starts its main work later in the `CREATE` and +`BUILD` phase, Flowman also provides the ability to *verify* the results of all data transformations after the +the `BUILD` execution phase, namely in the `VERIFY` phase. In order to implement a verification, you simply need +to use a [verify](../spec/target/verify.md) target, which works precisely like the `validate` target with the only +difference that it is executed after the `BUILD` phase. + +Note that when you are concerned about the quality of the data produced by your Flowman job, the `verify` target +is only one of multiple possibilities to implement meaningful checks. Read more in the +[data quality cookbook](data-qualioty.md) about available options. diff --git a/docs/lifecycle.md b/docs/lifecycle.md index b590542cd..89f68fc8a 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -5,7 +5,7 @@ multiple different phases, each of them representing one stage of the whole life ## Lifecycle Phases -The full lifecycle consists out of specific phases, as follows: +The full lifecycle consists out of specific execution phases, as follows: 1. **VALIDATE**. This first phase is used for validation and any error will stop the next steps. A validation step might for example @@ -29,7 +29,7 @@ some specific user defined tests that compare data. If verification fails, the b tables (i.e. it will delete data), but it will keep tables alive. 6. **DESTROY**. -The final phase *destroy* is used to phyiscally remove relations including their data. This will also remove table +The final phase *destroy* is used to physically remove relations including their data. This will also remove table definitions, views and directories. It performs the opposite operation than the *create* phase. diff --git a/docs/spec/target/blackhole.md b/docs/spec/target/blackhole.md index d034a497d..b9e4cc274 100644 --- a/docs/spec/target/blackhole.md +++ b/docs/spec/target/blackhole.md @@ -24,5 +24,7 @@ Optional descriptive text of the build target Specifies the name of the mapping output to be materialized -## Supported Phases +## Supported Execution Phases * `BUILD` - In the build phase, all records of the specified mapping will be materialized + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/compare.md b/docs/spec/target/compare.md index 6edd3ed74..610722d10 100644 --- a/docs/spec/target/compare.md +++ b/docs/spec/target/compare.md @@ -36,6 +36,8 @@ a mapping. Specifies the data set containing the expected data. In most cases you probably will use a file data set referencing some predefined results -## Supported Phases +## Supported Execution Phases * `VERIFY` - Comparison will be performed in the *verify* build phase. If the comparison fails, the build will stop with an error + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/console.md b/docs/spec/target/console.md index af7e883ef..2b7bbe69a 100644 --- a/docs/spec/target/console.md +++ b/docs/spec/target/console.md @@ -23,5 +23,8 @@ Specified the [dataset](../dataset/index.md) containing the records to be dumped * `limit` **(optional)** *(type: integer)* *(default: 100)*: Specified the number of records to be displayed -## Supported Phases + +## Supported Execution Phases * `BUILD` - The target will only be executed in the *build* phase + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/copy-file.md b/docs/spec/target/copy-file.md index 40c5e6823..9adf75541 100644 --- a/docs/spec/target/copy-file.md +++ b/docs/spec/target/copy-file.md @@ -11,8 +11,10 @@ * `target` **(mandatory)** *(type: string)*: -## Supported Phases +## Supported Execution Phases * `BUILD` * `VERIFY` * `TRUNCATE` * `DESTROY` + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/copy.md b/docs/spec/target/copy.md index 3d20f7291..39edfd6d5 100644 --- a/docs/spec/target/copy.md +++ b/docs/spec/target/copy.md @@ -53,8 +53,10 @@ and output file will contain approximately the same number of records. The defau Flowman config variable `floman.default.target.rebalance`. -## Supported Phases +## Supported Execution Phases * `BUILD` - The *build* phase will perform the copy operation * `VERIFY` - The *verify* phase will ensure that the target exists * `TRUNCATE` - The *truncate* phase will remove the target * `DESTROY` - The *destroy* phase will remove the target + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/count.md b/docs/spec/target/count.md index d388f8a16..42b8e57ae 100644 --- a/docs/spec/target/count.md +++ b/docs/spec/target/count.md @@ -16,5 +16,7 @@ targets: Specifies the name of the input mapping to be counted -## Supported Phases -* `BUILD` +## Supported Execution Phases +* `BUILD` - Counting records of a mapping will be executed as part of the `BUILD` phase + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/delete-file.md b/docs/spec/target/delete-file.md index 060d2c458..c4e066366 100644 --- a/docs/spec/target/delete-file.md +++ b/docs/spec/target/delete-file.md @@ -19,5 +19,7 @@ targets: * `location` **(mandatory)** *(type: string)*: -## Supported Phases +## Supported Execution Phases * `BUILD` - This will remove the specified location + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/delta-vacuum.md b/docs/spec/target/delta-vacuum.md index d985b5bca..8fe9b0855 100644 --- a/docs/spec/target/delta-vacuum.md +++ b/docs/spec/target/delta-vacuum.md @@ -58,5 +58,7 @@ targets: will be performed. -## Supported Phases +## Supported Execution Phases * `BUILD` - This will execute the vacuum operation + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/document.md b/docs/spec/target/document.md index 539423ab8..474b643e1 100644 --- a/docs/spec/target/document.md +++ b/docs/spec/target/document.md @@ -55,5 +55,7 @@ exist, Flowman will fall back to some default configuration, which creates a htm subdirectory `generated-documentation` within the projects base directory. -## Supported Phases +## Supported Execution Phases * `VERIFY` - This will generate the documentation + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/drop.md b/docs/spec/target/drop.md index 3e8bf7497..121e7e2c4 100644 --- a/docs/spec/target/drop.md +++ b/docs/spec/target/drop.md @@ -56,7 +56,9 @@ The `drop` target will drop a relation and all its contents. It will be executed during the `DESTROY` phase. -## Supported Phases +## Supported Execution Phases * `CREATE` - This will drop the target relation or migrate it to the newest schema (if possible). * `VERIFY` - This will verify that the target relation does not exist any more * `DESTROY` - This will also drop the relation itself and all its content. + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/file.md b/docs/spec/target/file.md index 7749a034c..accc58526 100644 --- a/docs/spec/target/file.md +++ b/docs/spec/target/file.md @@ -56,11 +56,13 @@ Flowman config variable `floman.default.target.rebalance`. ## Supported Phases -* `CREATE` -* `BUILD` -* `VERIFY` -* `TRUNCATE` -* `DESTROY` +* `CREATE` - creates the target directory +* `BUILD` - build the target files containing records +* `VERIFY` - verifies that the target file exists +* `TRUNCATE` - removes the target file, but keeps the directory +* `DESTROY` - recursively removes the target directory and all files inside + +Read more about [execution phases](../../lifecycle.md). ## Provided Metrics diff --git a/docs/spec/target/hive-database.md b/docs/spec/target/hive-database.md index 88a9139a5..885439c80 100644 --- a/docs/spec/target/hive-database.md +++ b/docs/spec/target/hive-database.md @@ -23,7 +23,9 @@ targets: Name of the Hive database to be created -## Supported Phases +## Supported Execution Phases * `CREATE` - Ensures that the specified Hive database exists and creates one if it is not found * `VERIFY` - Verifies that the specified Hive database exists * `DESTROY` - Drops the Hive database + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/local.md b/docs/spec/target/local.md index c81446811..8e9c92dde 100644 --- a/docs/spec/target/local.md +++ b/docs/spec/target/local.md @@ -32,11 +32,13 @@ targets: * `columns` **(optional)** *(list)* *(default: [])*: -## Supported Phases -* `BUILD` -* `VERIFY` -* `TRUNCATE` -* `DESTROY` +## Supported Execution Phases +* `BUILD` - build the target files containing records +* `VERIFY` - verifies that the target file exists +* `TRUNCATE` - removes the target file +* `DESTROY` - removes the target file, equivalent to `TRUNCATE` + +Read more about [execution phases](../../lifecycle.md). ## Provided Metrics diff --git a/docs/spec/target/measure.md b/docs/spec/target/measure.md index 4f992afcd..67793ad5b 100644 --- a/docs/spec/target/measure.md +++ b/docs/spec/target/measure.md @@ -21,3 +21,9 @@ targets: This example will provide two metrics, `record_count` and `column_sum`, which then can be sent to a [metric sink](../metric) configured in the [namespace](../namespace.md). + + +## Supported Execution Phases +* `VERIFY` - The evaluation of all measures will only be performed in the `VERIFY` phase + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/merge-files.md b/docs/spec/target/merge-files.md index 9b5af4e4b..9334ebcec 100644 --- a/docs/spec/target/merge-files.md +++ b/docs/spec/target/merge-files.md @@ -23,8 +23,10 @@ targets: * `overwrite` **(optional)** *(boolean)* *(default: true)*: -## Supported Phases +## Supported Execution Phases * `BUILD` * `VERIFY` * `TRUNCATE` * `DESTROY` + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/merge.md b/docs/spec/target/merge.md index 1b26a555a..c0ca7da1c 100644 --- a/docs/spec/target/merge.md +++ b/docs/spec/target/merge.md @@ -102,7 +102,7 @@ relations: Flowman config variable `floman.default.target.rebalance`. -## Supported Phases +## Supported Execution Phases * `CREATE` - This will create the target relation or migrate it to the newest schema (if possible). * `BUILD` - This will write the output of the specified mapping into the relation. If no mapping is specified, nothing will be done. @@ -111,6 +111,8 @@ relations: if the relation refers to a Hive table) * `DESTROY` - This drops the relation itself and all its content. +Read more about [execution phases](../../lifecycle.md). + ## Provided Metrics The relation target also provides some metric containing the number of records written: diff --git a/docs/spec/target/null.md b/docs/spec/target/null.md index 586c91a57..56c5cf6ec 100644 --- a/docs/spec/target/null.md +++ b/docs/spec/target/null.md @@ -20,10 +20,12 @@ targets: Optional descriptive text of the build target -## Supported Phases +## Supported Execution Phases * `CREATE` * `MIGRATE` * `BUILD` * `VERIFY` * `TRUNCATE` * `DESTROY` + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/relation.md b/docs/spec/target/relation.md index 9665e826a..4c13f0360 100644 --- a/docs/spec/target/relation.md +++ b/docs/spec/target/relation.md @@ -113,7 +113,7 @@ the relation during the `CREATE`, `TRUNCATE` and `DESTROY` phase. In this case, target. -## Supported Phases +## Supported Execution Phases * `CREATE` - This will create the target relation or migrate it to the newest schema (if possible). * `BUILD` - This will write the output of the specified mapping into the relation. If no mapping is specified, nothing will be done. @@ -122,6 +122,8 @@ target. if the relation refers to a Hive table) * `DESTROY` - This drops the relation itself and all its content. +Read more about [execution phases](../../lifecycle.md). + ## Provided Metrics The relation target also provides some metric containing the number of records written: diff --git a/docs/spec/target/sftp-upload.md b/docs/spec/target/sftp-upload.md index a0ff8ef24..cdb9a0780 100644 --- a/docs/spec/target/sftp-upload.md +++ b/docs/spec/target/sftp-upload.md @@ -76,3 +76,9 @@ Set to `true` in order to overwrite existing files on the SFTP server. Otherwise file will result in an error. ## Description + + +## Supported Execution Phases +* `BUILD` - This will upload the specified file via SFTP + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/template.md b/docs/spec/target/template.md index 8bcda8e0c..6d6f0e926 100644 --- a/docs/spec/target/template.md +++ b/docs/spec/target/template.md @@ -15,3 +15,7 @@ targets: environment: - table=fee ``` + +## Supported Execution Phases + +The supported execution phases are determined by the referenced target. diff --git a/docs/spec/target/truncate.md b/docs/spec/target/truncate.md index cf213efe7..5b4070cc5 100644 --- a/docs/spec/target/truncate.md +++ b/docs/spec/target/truncate.md @@ -68,7 +68,9 @@ targets: Specifies the partition (or multiple partitions) to truncate. -## Supported Phases +## Supported Execution Phases * `BUILD` - This will truncate the specified relation. * `VERIFY` - This will verify that the relation (and any specified partition) actually contains no data. * `TRUNCATE` - This will truncate the specified relation. + +Read more about [execution phases](../../lifecycle.md). diff --git a/docs/spec/target/validate.md b/docs/spec/target/validate.md index 31a563ab6..d7240292d 100644 --- a/docs/spec/target/validate.md +++ b/docs/spec/target/validate.md @@ -40,9 +40,11 @@ targets: Specify how to proceed in case individual assertions fail. Possible values are `failFast`, `failAtEnd` and `failNever` -## Supported Phases +## Supported Execution Phases * `VALIDATE` - The specified assertions will be run in the `VALIDATE` phase before the `CREATE` and `BUILD` phases. +Read more about [execution phases](../../lifecycle.md). + ## Remarks diff --git a/docs/spec/target/verify.md b/docs/spec/target/verify.md index db4681c54..4a8156eca 100644 --- a/docs/spec/target/verify.md +++ b/docs/spec/target/verify.md @@ -42,9 +42,11 @@ targets: Specify how to proceed in case individual assertions fail. Possible values are `failFast`, `failAtEnd` and `failNever` -## Supported Phases +## Supported Execution Phases * `VERIDY` - The specified assertions will be run in the `VERIFY` phase after the `CREATE` and `BUILD` phases. +Read more about [execution phases](../../lifecycle.md). + ## Remarks From 8ec60220d0de774e578993f661687354afec5f2c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 18 Feb 2022 21:02:48 +0100 Subject: [PATCH 56/95] Fix build for Scala 2.11 --- .../dimajix/flowman/documentation/MappingCollector.scala | 3 ++- .../dimajix/flowman/documentation/RelationCollector.scala | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index b085abeca..fb146d000 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -82,7 +82,8 @@ class MappingCollector extends Collector { val doc = MappingDoc( Some(parent), mapping.identifier, - inputs = inputs.map(_._2.reference).toSeq + None, + inputs.map(_._2.reference).toSeq ) val ref = doc.reference diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 9bd5b0bda..a89ce9ec3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -81,9 +81,10 @@ class RelationCollector extends Collector { Some(parent), relation.identifier, description = relation.description, - inputs = inputs, - provides = relation.provides.toSeq, - partitions = partitions + None, + inputs, + relation.provides.toSeq, + partitions ) val ref = doc.reference From 85b47181a931f2b62b3c57f677ffa1c8236f5d04 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 19 Feb 2022 10:25:09 +0100 Subject: [PATCH 57/95] Use temporary staging table for writing to a SQL Server relation --- .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 19 +++- .../spec/relation/SqlServerRelation.scala | 76 +++++++++++-- .../flowman/spec/relation/JdbcRelation.scala | 107 ++++++++++++------ 3 files changed, 159 insertions(+), 43 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index 576493b60..3edc9fbbd 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,6 +71,23 @@ object JdbcUtils { factory() } + def withTransaction[T](con:java.sql.Connection)(fn: => T) : T = { + val oldMode = con.getAutoCommit + con.setAutoCommit(false) + try { + val result = fn + con.commit() + result + } catch { + case ex:SQLException => + logger.error(s"SQL transaction failed, rolling back: ${ex.getMessage}") + con.rollback() + throw ex + } finally { + con.setAutoCommit(oldMode) + } + } + def withStatement[T](conn:Connection, options: JDBCOptions)(fn:Statement => T) : T = { val statement = conn.createStatement() try { diff --git a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala index 057ca7e59..5edf2db12 100644 --- a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala +++ b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala @@ -21,10 +21,14 @@ import scala.collection.mutable import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.jdbc.JdbcUtils +import com.dimajix.flowman.jdbc.SqlDialects +import com.dimajix.flowman.jdbc.TableDefinition import com.dimajix.flowman.model.Connection import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Reference @@ -33,12 +37,13 @@ import com.dimajix.flowman.model.Schema import com.dimajix.flowman.spec.annotation.RelationType import com.dimajix.flowman.spec.connection.ConnectionReferenceSpec import com.dimajix.flowman.spec.connection.JdbcConnection +import com.dimajix.flowman.types.StructType -class SqlServerRelation( - instanceProperties:Relation.Properties, - schema:Option[Schema] = None, - partitions: Seq[PartitionField] = Seq(), +case class SqlServerRelation( + override val instanceProperties:Relation.Properties, + override val schema:Option[Schema] = None, + override val partitions: Seq[PartitionField] = Seq(), connection: Reference[Connection], properties: Map[String,String] = Map(), database: Option[String] = None, @@ -46,14 +51,69 @@ class SqlServerRelation( query: Option[String] = None, mergeKey: Seq[String] = Seq(), primaryKey: Seq[String] = Seq() -) extends JdbcRelation(instanceProperties, schema, partitions, connection, properties, database, table, query, mergeKey, primaryKey) { - override protected def doWrite(execution: Execution, df:DataFrame): Unit = { +) extends JdbcRelationBase(instanceProperties, schema, partitions, connection, properties, database, table, query, mergeKey, primaryKey) { + private val tempTableIdentifier = TableIdentifier(s"##${tableIdentifier.table}_temp_staging") + + override protected def doOverwriteAll(execution: Execution, df:DataFrame) : Unit = { + withConnection { (con, options) => + createTempTable(con, options, StructType.of(df.schema)) + logger.info(s"Writing new data into temporary staging table '${tempTableIdentifier}'") + appendTable(execution, df, tempTableIdentifier) + + withTransaction(con) { + withStatement(con, options) { case (statement, options) => + val dialect = SqlDialects.get(options.url) + logger.info(s"Truncating table '${tableIdentifier}'") + statement.executeUpdate(s"TRUNCATE TABLE ${dialect.quote(tableIdentifier)}") + logger.info(s"Copying data from temporary staging table '${tempTableIdentifier}' into table '${tableIdentifier}'") + statement.executeUpdate(s"INSERT INTO ${dialect.quote(tableIdentifier)} SELECT * FROM ${dialect.quote(tempTableIdentifier)}") + logger.info(s"Dropping temporary staging table '${tempTableIdentifier}'") + statement.executeUpdate(s"DROP TABLE ${dialect.quote(tempTableIdentifier)}") + } + } + } + } + override protected def doAppend(execution: Execution, df:DataFrame): Unit = { + withConnection { (con, options) => + createTempTable(con, options, StructType.of(df.schema)) + logger.info(s"Writing new data into temporary staging table '${tempTableIdentifier}'") + appendTable(execution, df, tempTableIdentifier) + + withTransaction(con) { + withStatement(con, options) { case (statement, options) => + val dialect = SqlDialects.get(options.url) + logger.info(s"Copying data from temporary staging table '${tempTableIdentifier}' into table '${tableIdentifier}'") + statement.executeUpdate(s"INSERT INTO ${dialect.quote(tableIdentifier)} SELECT * FROM ${dialect.quote(tempTableIdentifier)}") + logger.info(s"Dropping temporary staging table '${tempTableIdentifier}'") + statement.executeUpdate(s"DROP TABLE ${dialect.quote(tempTableIdentifier)}") + } + } + } + } + + private def appendTable(execution: Execution, df:DataFrame, table:TableIdentifier): Unit = { val (_,props) = createConnectionProperties() this.writer(execution, df, "com.microsoft.sqlserver.jdbc.spark", Map(), SaveMode.Append) - .options(props) - .option(JDBCOptions.JDBC_TABLE_NAME, tableIdentifier.unquotedString) + .options(props ++ Map("tableLock" -> "true", "mssqlIsolationLevel" -> "READ_UNCOMMITTED")) + .option(JDBCOptions.JDBC_TABLE_NAME, table.unquotedString) .save() } + private def createTempTable(con:java.sql.Connection,options: JDBCOptions, schema:StructType) : Unit = { + logger.info(s"Creating temporary staging table '${tempTableIdentifier}' with schema\n${schema.treeString}") + + // First drop temp table if it already exists + withStatement(con, options) { case (statement, options) => + val dialect = SqlDialects.get(options.url) + statement.executeUpdate(s"DROP TABLE IF EXISTS ${dialect.quote(tempTableIdentifier)}") + } + + // Create temp table with specified schema, but without any primary key or indices + val table = TableDefinition( + tempTableIdentifier, + schema.fields + ) + JdbcUtils.createTable(con, table, options) + } override protected def createConnectionProperties() : (String,Map[String,String]) = { val connection = this.connection.value.asInstanceOf[JdbcConnection] diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 10d43c6a5..bb192ca27 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -31,7 +31,6 @@ import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException -import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions @@ -72,10 +71,9 @@ import com.dimajix.flowman.spec.connection.JdbcConnection import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.SingleValue import com.dimajix.flowman.types.{StructType => FlowmanStructType} -import com.dimajix.spark.sql.SchemaUtils -case class JdbcRelation( +class JdbcRelationBase( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, override val partitions: Seq[PartitionField] = Seq(), @@ -87,9 +85,8 @@ case class JdbcRelation( mergeKey: Seq[String] = Seq(), primaryKey: Seq[String] = Seq() ) extends BaseRelation with PartitionedRelation with SchemaRelation { - private val logger = LoggerFactory.getLogger(classOf[JdbcRelation]) - - def tableIdentifier : TableIdentifier = TableIdentifier(table.getOrElse(""), database) + protected val logger = LoggerFactory.getLogger(getClass) + protected val tableIdentifier : TableIdentifier = TableIdentifier(table.getOrElse(""), database) if (query.nonEmpty && table.nonEmpty) throw new IllegalArgumentException(s"JDBC relation '$identifier' cannot have both a table and a SQL query defined") @@ -220,43 +217,53 @@ case class JdbcRelation( // Write partition into DataBase mode match { case OutputMode.OVERWRITE if partition.isEmpty => - withConnection { (con, options) => - JdbcUtils.truncateTable(con, tableIdentifier, options) - } - doWrite(execution, dfExt) + doOverwriteAll(execution, dfExt) case OutputMode.OVERWRITE => - withStatement { (statement, options) => - val dialect = SqlDialects.get(options.url) - val condition = partitionCondition(dialect, partition) - val query = "DELETE FROM " + dialect.quote(tableIdentifier) + " WHERE " + condition - statement.executeUpdate(query) - } - doWrite(execution, dfExt) + doOverwritePartition(execution, dfExt, partition) case OutputMode.APPEND => - doWrite(execution, dfExt) + doAppend(execution, dfExt) case OutputMode.IGNORE_IF_EXISTS => if (!checkPartition(partition)) { - doWrite(execution, dfExt) + doAppend(execution, dfExt) } case OutputMode.ERROR_IF_EXISTS => if (!checkPartition(partition)) { - doWrite(execution, dfExt) + doAppend(execution, dfExt) } else { throw new PartitionAlreadyExistsException(database.getOrElse(""), table.get, partition.mapValues(_.value)) } + case OutputMode.UPDATE => + doUpdate(execution, df) case _ => throw new IllegalArgumentException(s"Unsupported save mode: '$mode'. " + - "Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + "Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'update', 'errorifexists'.") } } - protected def doWrite(execution: Execution, df:DataFrame): Unit = { + protected def doOverwriteAll(execution: Execution, df:DataFrame) : Unit = { + withConnection { (con, options) => + JdbcUtils.truncateTable(con, tableIdentifier, options) + } + doAppend(execution, df) + } + protected def doOverwritePartition(execution: Execution, df:DataFrame, partition:Map[String,SingleValue]) : Unit = { + withStatement { (statement, options) => + val dialect = SqlDialects.get(options.url) + val condition = partitionCondition(dialect, partition) + val query = "DELETE FROM " + dialect.quote(tableIdentifier) + " WHERE " + condition + statement.executeUpdate(query) + } + doAppend(execution, df) + } + protected def doAppend(execution: Execution, df:DataFrame): Unit = { val (_,props) = createConnectionProperties() this.writer(execution, df, "jdbc", Map(), SaveMode.Append) .options(props) .option(JDBCOptions.JDBC_TABLE_NAME, tableIdentifier.unquotedString) .save() } - + protected def doUpdate(execution: Execution, df:DataFrame): Unit = { + throw new IllegalArgumentException(s"Unsupported save mode: 'UPDATE' for generic JDBC relation.") + } /** * Performs a merge operation. Either you need to specify a [[mergeKey]], or the relation needs to provide some @@ -417,7 +424,7 @@ case class JdbcRelation( } } - private def doCreate(con:java.sql.Connection, options:JDBCOptions): Unit = { + protected def doCreate(con:java.sql.Connection, options:JDBCOptions): Unit = { logger.info(s"Creating JDBC relation '$identifier', this will create JDBC table $tableIdentifier with schema\n${this.schema.map(_.treeString).orNull}") if (this.schema.isEmpty) { throw new UnspecifiedSchemaException(identifier) @@ -569,7 +576,7 @@ case class JdbcRelation( (connection.url,props.toMap) } - private def withConnection[T](fn:(java.sql.Connection,JDBCOptions) => T) : T = { + protected def withConnection[T](fn:(java.sql.Connection,JDBCOptions) => T) : T = { val (url,props) = createConnectionProperties() logger.debug(s"Connecting to jdbc source at $url") @@ -590,16 +597,24 @@ case class JdbcRelation( } } - private def withStatement[T](fn:(Statement,JDBCOptions) => T) : T = { + protected def withTransaction[T](con:java.sql.Connection)(fn: => T) : T = { + JdbcUtils.withTransaction(con)(fn) + } + + protected def withStatement[T](fn:(Statement,JDBCOptions) => T) : T = { withConnection { (con, options) => - val statement = con.createStatement() - try { - statement.setQueryTimeout(JdbcUtils.queryTimeout(options)) - fn(statement, options) - } - finally { - statement.close() - } + withStatement(con,options)(fn) + } + } + + protected def withStatement[T](con:java.sql.Connection,options:JDBCOptions)(fn:(Statement,JDBCOptions) => T) : T = { + val statement = con.createStatement() + try { + statement.setQueryTimeout(JdbcUtils.queryTimeout(options)) + fn(statement, options) + } + finally { + statement.close() } } @@ -651,6 +666,30 @@ case class JdbcRelation( } +case class JdbcRelation( + override val instanceProperties:Relation.Properties, + override val schema:Option[Schema] = None, + override val partitions: Seq[PartitionField] = Seq(), + connection: Reference[Connection], + properties: Map[String,String] = Map(), + database: Option[String] = None, + table: Option[String] = None, + query: Option[String] = None, + mergeKey: Seq[String] = Seq(), + primaryKey: Seq[String] = Seq() +) extends JdbcRelationBase( + instanceProperties, + schema, + partitions, + connection, + properties, + database, + table, + query, + mergeKey, + primaryKey +) { +} class JdbcRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec { From ad1654d2469fce4afce23f9ab5fe4e8ed15afca7 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 19 Feb 2022 14:40:52 +0100 Subject: [PATCH 58/95] Update some dependencies --- examples/weather/schema/measurements.avsc | 2 +- flowman-server-ui/package-lock.json | 74 +-- flowman-studio-ui/package-lock.json | 660 +++------------------- 3 files changed, 112 insertions(+), 624 deletions(-) diff --git a/examples/weather/schema/measurements.avsc b/examples/weather/schema/measurements.avsc index 8e75774b5..ced3fc4fc 100644 --- a/examples/weather/schema/measurements.avsc +++ b/examples/weather/schema/measurements.avsc @@ -25,7 +25,7 @@ }, { "name": "wind_direction", - "type": [ "string", "null" ] + "type": [ "int", "null" ] }, { "name": "wind_direction_qual", diff --git a/flowman-server-ui/package-lock.json b/flowman-server-ui/package-lock.json index 9d60faec1..2de734f9f 100644 --- a/flowman-server-ui/package-lock.json +++ b/flowman-server-ui/package-lock.json @@ -2473,9 +2473,9 @@ } }, "node_modules/@vue/component-compiler-utils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.2.2.tgz", - "integrity": "sha512-rAYMLmgMuqJFWAOb3Awjqqv5X3Q3hVr4jH/kgrFJpiU0j3a90tnNBplqbj+snzrgZhC9W128z+dtgMifOiMfJg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz", + "integrity": "sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ==", "dev": true, "dependencies": { "consolidate": "^0.15.1", @@ -2488,7 +2488,7 @@ "vue-template-es2015-compiler": "^1.9.0" }, "optionalDependencies": { - "prettier": "^1.18.2" + "prettier": "^1.18.2 || ^2.0.0" } }, "node_modules/@vue/component-compiler-utils/node_modules/hash-sum": { @@ -9274,9 +9274,9 @@ "dev": true }, "node_modules/json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "dev": true }, "node_modules/json-schema-traverse": { @@ -9328,18 +9328,18 @@ } }, "node_modules/jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", "dev": true, - "engines": [ - "node >=0.6.0" - ], "dependencies": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" + }, + "engines": { + "node": ">=0.6.0" } }, "node_modules/killable": { @@ -13343,9 +13343,9 @@ "dev": true }, "node_modules/selfsigned": { - "version": "1.10.11", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.11.tgz", - "integrity": "sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA==", + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.14.tgz", + "integrity": "sha512-lkjaiAye+wBZDCBsu5BGi0XiLRxeUlsGod5ZP924CRSEoGuZAw/f7y9RKu28rwTfiHVhdavhB0qH0INV6P1lEA==", "dev": true, "dependencies": { "node-forge": "^0.10.0" @@ -15255,9 +15255,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", + "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", "dev": true, "dependencies": { "querystringify": "^2.1.1", @@ -18607,9 +18607,9 @@ } }, "@vue/component-compiler-utils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.2.2.tgz", - "integrity": "sha512-rAYMLmgMuqJFWAOb3Awjqqv5X3Q3hVr4jH/kgrFJpiU0j3a90tnNBplqbj+snzrgZhC9W128z+dtgMifOiMfJg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz", + "integrity": "sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ==", "dev": true, "requires": { "consolidate": "^0.15.1", @@ -18618,7 +18618,7 @@ "merge-source-map": "^1.1.0", "postcss": "^7.0.36", "postcss-selector-parser": "^6.0.2", - "prettier": "^1.18.2", + "prettier": "^1.18.2 || ^2.0.0", "source-map": "~0.6.1", "vue-template-es2015-compiler": "^1.9.0" }, @@ -24289,9 +24289,9 @@ "dev": true }, "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "dev": true }, "json-schema-traverse": { @@ -24337,14 +24337,14 @@ } }, "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" } }, @@ -27675,9 +27675,9 @@ "dev": true }, "selfsigned": { - "version": "1.10.11", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.11.tgz", - "integrity": "sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA==", + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.14.tgz", + "integrity": "sha512-lkjaiAye+wBZDCBsu5BGi0XiLRxeUlsGod5ZP924CRSEoGuZAw/f7y9RKu28rwTfiHVhdavhB0qH0INV6P1lEA==", "dev": true, "requires": { "node-forge": "^0.10.0" @@ -29298,9 +29298,9 @@ } }, "url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", + "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", "dev": true, "requires": { "querystringify": "^2.1.1", diff --git a/flowman-studio-ui/package-lock.json b/flowman-studio-ui/package-lock.json index da6bb130f..198a8356a 100644 --- a/flowman-studio-ui/package-lock.json +++ b/flowman-studio-ui/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "flowman-studio-ui", "version": "0.1.0", "dependencies": { "axios": "^0.21.4", @@ -2464,9 +2465,9 @@ } }, "node_modules/@vue/component-compiler-utils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.2.2.tgz", - "integrity": "sha512-rAYMLmgMuqJFWAOb3Awjqqv5X3Q3hVr4jH/kgrFJpiU0j3a90tnNBplqbj+snzrgZhC9W128z+dtgMifOiMfJg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz", + "integrity": "sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ==", "dev": true, "dependencies": { "consolidate": "^0.15.1", @@ -2479,7 +2480,7 @@ "vue-template-es2015-compiler": "^1.9.0" }, "optionalDependencies": { - "prettier": "^1.18.2" + "prettier": "^1.18.2 || ^2.0.0" } }, "node_modules/@vue/component-compiler-utils/node_modules/hash-sum": { @@ -6294,13 +6295,13 @@ } }, "node_modules/eslint-module-utils": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz", - "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==", + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz", + "integrity": "sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ==", "dev": true, "dependencies": { "debug": "^3.2.7", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "engines": { "node": ">=4" @@ -6382,18 +6383,6 @@ "node": ">=4" } }, - "node_modules/eslint-module-utils/node_modules/pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "dependencies": { - "find-up": "^2.1.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/eslint-plugin-es": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", @@ -6441,32 +6430,30 @@ } }, "node_modules/eslint-plugin-import": { - "version": "2.24.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz", - "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "dependencies": { - "array-includes": "^3.1.3", - "array.prototype.flat": "^1.2.4", + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.6.2", - "find-up": "^2.0.0", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", - "is-core-module": "^2.6.0", + "is-core-module": "^2.8.0", + "is-glob": "^4.0.3", "minimatch": "^3.0.4", - "object.values": "^1.1.4", - "pkg-up": "^2.0.0", - "read-pkg-up": "^3.0.0", + "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "engines": { "node": ">=4" }, "peerDependencies": { - "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0" + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8" } }, "node_modules/eslint-plugin-import/node_modules/debug": { @@ -6490,79 +6477,12 @@ "node": ">=0.10.0" } }, - "node_modules/eslint-plugin-import/node_modules/find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "dependencies": { - "locate-path": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/eslint-plugin-import/node_modules/locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "dependencies": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/eslint-plugin-import/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, - "node_modules/eslint-plugin-import/node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "dependencies": { - "p-try": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/eslint-plugin-import/node_modules/p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "dependencies": { - "p-limit": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/eslint-plugin-import/node_modules/p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/eslint-plugin-import/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/eslint-plugin-node": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", @@ -9108,9 +9028,9 @@ } }, "node_modules/is-core-module": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.7.0.tgz", - "integrity": "sha512-ByY+tjCciCr+9nLryBYcSD50EOGWt95c7tIsKTG1J2ixKKXPvF7Ej3AVd+UfDydAJom3biBGDBALaO79ktwgEQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", + "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", "dev": true, "dependencies": { "has": "^1.0.3" @@ -9712,43 +9632,6 @@ "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=", "dev": true }, - "node_modules/load-json-file": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", - "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.2", - "parse-json": "^4.0.0", - "pify": "^3.0.0", - "strip-bom": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/load-json-file/node_modules/parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", - "dev": true, - "dependencies": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/load-json-file/node_modules/pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/loader-fs-cache": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz", @@ -11717,85 +11600,6 @@ "node": ">=8" } }, - "node_modules/pkg-up": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz", - "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=", - "dev": true, - "dependencies": { - "find-up": "^2.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "dependencies": { - "locate-path": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "dependencies": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "dependencies": { - "p-try": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "dependencies": { - "p-limit": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/pkg-up/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/pnp-webpack-plugin": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/pnp-webpack-plugin/-/pnp-webpack-plugin-1.7.0.tgz", @@ -12783,100 +12587,6 @@ "node": ">=8" } }, - "node_modules/read-pkg-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", - "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", - "dev": true, - "dependencies": { - "find-up": "^2.0.0", - "read-pkg": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "dependencies": { - "locate-path": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "dependencies": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "dependencies": { - "p-try": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "dependencies": { - "p-limit": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/read-pkg-up/node_modules/read-pkg": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", - "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", - "dev": true, - "dependencies": { - "load-json-file": "^4.0.0", - "normalize-package-data": "^2.3.2", - "path-type": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/readable-stream": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", @@ -13593,9 +13303,9 @@ } }, "node_modules/sass-loader": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-10.2.0.tgz", - "integrity": "sha512-kUceLzC1gIHz0zNJPpqRsJyisWatGYNFRmv2CKZK2/ngMJgLqxTbXwe/hJ85luyvZkgqU3VlJ33UVF2T/0g6mw==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-10.2.1.tgz", + "integrity": "sha512-RRvWl+3K2LSMezIsd008ErK4rk6CulIMSwrcc2aZvjymUgKo/vjXGp1rSWmfTUX7bblEOz8tst4wBwWtCGBqKA==", "dev": true, "dependencies": { "klona": "^2.0.4", @@ -13747,9 +13457,9 @@ "dev": true }, "node_modules/selfsigned": { - "version": "1.10.11", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.11.tgz", - "integrity": "sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA==", + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.14.tgz", + "integrity": "sha512-lkjaiAye+wBZDCBsu5BGi0XiLRxeUlsGod5ZP924CRSEoGuZAw/f7y9RKu28rwTfiHVhdavhB0qH0INV6P1lEA==", "dev": true, "dependencies": { "node-forge": "^0.10.0" @@ -13991,9 +13701,9 @@ "dev": true }, "node_modules/shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "dependencies": { "glob": "^7.0.0", @@ -15256,9 +14966,9 @@ } }, "node_modules/tsconfig-paths": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz", - "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==", + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", "dev": true, "dependencies": { "@types/json5": "^0.0.29", @@ -15618,9 +15328,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", + "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", "dev": true, "dependencies": { "querystringify": "^2.1.1", @@ -19049,9 +18759,9 @@ } }, "@vue/component-compiler-utils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.2.2.tgz", - "integrity": "sha512-rAYMLmgMuqJFWAOb3Awjqqv5X3Q3hVr4jH/kgrFJpiU0j3a90tnNBplqbj+snzrgZhC9W128z+dtgMifOiMfJg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@vue/component-compiler-utils/-/component-compiler-utils-3.3.0.tgz", + "integrity": "sha512-97sfH2mYNU+2PzGrmK2haqffDpVASuib9/w2/noxiFi31Z54hW+q3izKQXXQZSNhtiUpAI36uSuYepeBe4wpHQ==", "dev": true, "requires": { "consolidate": "^0.15.1", @@ -19060,7 +18770,7 @@ "merge-source-map": "^1.1.0", "postcss": "^7.0.36", "postcss-selector-parser": "^6.0.2", - "prettier": "^1.18.2", + "prettier": "^1.18.2 || ^2.0.0", "source-map": "~0.6.1", "vue-template-es2015-compiler": "^1.9.0" }, @@ -22234,13 +21944,13 @@ } }, "eslint-module-utils": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.2.tgz", - "integrity": "sha512-QG8pcgThYOuqxupd06oYTZoNOGaUdTY1PqK+oS6ElF6vs4pBdk/aYxFVQQXzcrAqp9m7cl7lb2ubazX+g16k2Q==", + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.3.tgz", + "integrity": "sha512-088JEC7O3lDZM9xGe0RerkOMd0EjFl+Yvd1jPWIkMT5u3H9+HC34mWWPnqPrN13gieT9pBOO+Qt07Nb/6TresQ==", "dev": true, "requires": { "debug": "^3.2.7", - "pkg-dir": "^2.0.0" + "find-up": "^2.1.0" }, "dependencies": { "debug": { @@ -22300,15 +22010,6 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", "dev": true - }, - "pkg-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", - "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", - "dev": true, - "requires": { - "find-up": "^2.1.0" - } } } }, @@ -22340,26 +22041,24 @@ } }, "eslint-plugin-import": { - "version": "2.24.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz", - "integrity": "sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.25.4.tgz", + "integrity": "sha512-/KJBASVFxpu0xg1kIBn9AUa8hQVnszpwgE7Ld0lKAlx7Ie87yzEzCgSkekt+le/YVhiaosO4Y14GDAOc41nfxA==", "dev": true, "requires": { - "array-includes": "^3.1.3", - "array.prototype.flat": "^1.2.4", + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", "debug": "^2.6.9", "doctrine": "^2.1.0", "eslint-import-resolver-node": "^0.3.6", - "eslint-module-utils": "^2.6.2", - "find-up": "^2.0.0", + "eslint-module-utils": "^2.7.2", "has": "^1.0.3", - "is-core-module": "^2.6.0", + "is-core-module": "^2.8.0", + "is-glob": "^4.0.3", "minimatch": "^3.0.4", - "object.values": "^1.1.4", - "pkg-up": "^2.0.0", - "read-pkg-up": "^3.0.0", + "object.values": "^1.1.5", "resolve": "^1.20.0", - "tsconfig-paths": "^3.11.0" + "tsconfig-paths": "^3.12.0" }, "dependencies": { "debug": { @@ -22380,60 +22079,11 @@ "esutils": "^2.0.2" } }, - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "requires": { - "locate-path": "^2.0.0" - } - }, - "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - } - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true - }, - "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true } } }, @@ -24354,9 +24004,9 @@ } }, "is-core-module": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.7.0.tgz", - "integrity": "sha512-ByY+tjCciCr+9nLryBYcSD50EOGWt95c7tIsKTG1J2ixKKXPvF7Ej3AVd+UfDydAJom3biBGDBALaO79ktwgEQ==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", + "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", "dev": true, "requires": { "has": "^1.0.3" @@ -24811,36 +24461,6 @@ "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=", "dev": true }, - "load-json-file": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", - "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", - "dev": true, - "requires": { - "graceful-fs": "^4.1.2", - "parse-json": "^4.0.0", - "pify": "^3.0.0", - "strip-bom": "^3.0.0" - }, - "dependencies": { - "parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", - "dev": true, - "requires": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - } - }, - "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true - } - } - }, "loader-fs-cache": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz", @@ -26414,66 +26034,6 @@ "find-up": "^4.0.0" } }, - "pkg-up": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-2.0.0.tgz", - "integrity": "sha1-yBmscoBZpGHKscOImivjxJoATX8=", - "dev": true, - "requires": { - "find-up": "^2.1.0" - }, - "dependencies": { - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "requires": { - "locate-path": "^2.0.0" - } - }, - "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - } - }, - "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true - } - } - }, "pnp-webpack-plugin": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/pnp-webpack-plugin/-/pnp-webpack-plugin-1.7.0.tgz", @@ -27325,78 +26885,6 @@ "type-fest": "^0.6.0" } }, - "read-pkg-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", - "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", - "dev": true, - "requires": { - "find-up": "^2.0.0", - "read-pkg": "^3.0.0" - }, - "dependencies": { - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "dev": true, - "requires": { - "locate-path": "^2.0.0" - } - }, - "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "dev": true, - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - } - }, - "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "dev": true, - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", - "dev": true - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", - "dev": true - }, - "read-pkg": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", - "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", - "dev": true, - "requires": { - "load-json-file": "^4.0.0", - "normalize-package-data": "^2.3.2", - "path-type": "^3.0.0" - } - } - } - }, "readable-stream": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", @@ -27975,9 +27463,9 @@ } }, "sass-loader": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-10.2.0.tgz", - "integrity": "sha512-kUceLzC1gIHz0zNJPpqRsJyisWatGYNFRmv2CKZK2/ngMJgLqxTbXwe/hJ85luyvZkgqU3VlJ33UVF2T/0g6mw==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-10.2.1.tgz", + "integrity": "sha512-RRvWl+3K2LSMezIsd008ErK4rk6CulIMSwrcc2aZvjymUgKo/vjXGp1rSWmfTUX7bblEOz8tst4wBwWtCGBqKA==", "dev": true, "requires": { "klona": "^2.0.4", @@ -28080,9 +27568,9 @@ "dev": true }, "selfsigned": { - "version": "1.10.11", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.11.tgz", - "integrity": "sha512-aVmbPOfViZqOZPgRBT0+3u4yZFHpmnIghLMlAcb5/xhp5ZtB/RVnKhz5vl2M32CLXAqR4kha9zfhNg0Lf/sxKA==", + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.14.tgz", + "integrity": "sha512-lkjaiAye+wBZDCBsu5BGi0XiLRxeUlsGod5ZP924CRSEoGuZAw/f7y9RKu28rwTfiHVhdavhB0qH0INV6P1lEA==", "dev": true, "requires": { "node-forge": "^0.10.0" @@ -28296,9 +27784,9 @@ "dev": true }, "shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "requires": { "glob": "^7.0.0", @@ -29338,9 +28826,9 @@ "dev": true }, "tsconfig-paths": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz", - "integrity": "sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA==", + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", + "integrity": "sha512-e5adrnOYT6zqVnWqZu7i/BQ3BnhzvGbjEjejFXO20lKIKpwTaupkCPgEfv4GZK1IBciJUEhYs3J3p75FdaTFVg==", "dev": true, "requires": { "@types/json5": "^0.0.29", @@ -29639,9 +29127,9 @@ } }, "url-parse": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.3.tgz", - "integrity": "sha512-IIORyIQD9rvj0A4CLWsHkBBJuNqWpFQe224b6j9t/ABmquIS0qDU2pY6kl6AuOrL5OkCXHMCFNe1jBcuAggjvQ==", + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", + "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", "dev": true, "requires": { "querystringify": "^2.1.1", From e4eea507ef5e0f853f22041fde30853686f528fd Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sat, 19 Feb 2022 16:44:37 +0100 Subject: [PATCH 59/95] Add all sources to relations in generated documentation --- .../documentation/RelationCollector.scala | 27 ++++ .../flowman/documentation/RelationDoc.scala | 6 +- .../flowman/documentation/velocity.scala | 2 + .../documentation/RelationCollectorTest.scala | 6 +- .../flowman/documentation/html/project.vtl | 122 +++++++++++++++--- 5 files changed, 139 insertions(+), 24 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index a89ce9ec3..54e0b0d3a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -26,9 +26,11 @@ import com.dimajix.common.ExceptionUtils.reasons import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.InputMapping +import com.dimajix.flowman.graph.MappingRef import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.graph.RelationRef import com.dimajix.flowman.graph.WriteRelation +import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.types.FieldValue @@ -75,6 +77,29 @@ class RelationCollector extends Collector { case _ => None } + // Recursively collect all sources from upstream mappings + def collectMappingSources(map:MappingRef) : Seq[ResourceIdentifier] = { + val direct = map.mapping.requires.toSeq + val indirect = map.incoming.flatMap { + case in:InputMapping => + collectMappingSources(in.input) + case _ => Seq.empty + } + (direct ++ indirect).distinct + } + + val sources = node.incoming.flatMap { + case write:WriteRelation => + write.input.incoming.flatMap { + case map: InputMapping => + collectMappingSources(map.input) + case rel: ReadRelation => + rel.input.relation.provides.toSeq + case _ => Seq.empty + } + case _ => Seq.empty + }.distinct + val partitions = (inputPartitions ++ outputPartitions).foldLeft(Map.empty[String,FieldValue])((a,b) => a ++ b) val doc = RelationDoc( @@ -84,6 +109,8 @@ class RelationCollector extends Collector { None, inputs, relation.provides.toSeq, + relation.requires.toSeq, + sources, partitions ) val ref = doc.reference diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala index 7ebdc15b2..331b5c7de 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationDoc.scala @@ -57,6 +57,8 @@ final case class RelationDoc( schema:Option[SchemaDoc] = None, inputs:Seq[Reference] = Seq(), provides:Seq[ResourceIdentifier] = Seq(), + requires:Seq[ResourceIdentifier] = Seq(), + sources:Seq[ResourceIdentifier] = Seq(), partitions:Map[String,FieldValue] = Map() ) extends EntityDoc { override def reference: RelationReference = RelationReference(parent, identifier.name) @@ -88,7 +90,9 @@ final case class RelationDoc( val desc = other.description.orElse(this.description) val schm = schema.map(_.merge(other.schema)).orElse(other.schema) val prov = provides.toSet ++ other.provides.toSet - val result = copy(identifier=id, description=desc, schema=schm, provides=prov.toSeq) + val reqs = requires.toSet ++ other.requires.toSet + val srcs = sources.toSet ++ other.sources.toSet + val result = copy(identifier=id, description=desc, schema=schm, provides=prov.toSeq, requires=reqs.toSeq, sources=srcs.toSeq) parent.orElse(other.parent) .map(result.reparent) .getOrElse(result) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 81bb2786a..670e75c0c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -120,6 +120,8 @@ final case class RelationDocWrapper(relation:RelationDoc) extends FragmentWrappe def getSchema() : SchemaDocWrapper = relation.schema.map(SchemaDocWrapper).orNull def getInputs() : java.util.List[ReferenceWrapper] = relation.inputs.map(ReferenceWrapper).asJava def getResources() : java.util.List[ResourceIdentifierWrapper] = relation.provides.map(ResourceIdentifierWrapper).asJava + def getDependencies() : java.util.List[ResourceIdentifierWrapper] = relation.requires.map(ResourceIdentifierWrapper).asJava + def getSources() : java.util.List[ResourceIdentifierWrapper] = relation.sources.map(ResourceIdentifierWrapper).asJava } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala index e242733d6..e5ce455f0 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala @@ -93,12 +93,14 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { (mapping1.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m1")) //(mapping2.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m2")) + (mapping1.requires _).expects().returns(Set()) + (mapping2.requires _).expects().returns(Set()) (sourceRelation.identifier _).expects().atLeastOnce().returns(RelationIdentifier("project/src")) (sourceRelation.description _).expects().atLeastOnce().returns(Some("source relation")) (sourceRelation.documentation _).expects().returns(None) (sourceRelation.provides _).expects().returns(Set()) - //(sourceRelation.requires _).expects().returns(Set()) + (sourceRelation.requires _).expects().returns(Set()) (sourceRelation.schema _).expects().returns(None) (sourceRelation.describe _).expects(*,Map("pcol"-> SingleValue("part1"))).returns(StructType(Seq())) @@ -106,7 +108,7 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { (targetRelation.description _).expects().atLeastOnce().returns(Some("target relation")) (targetRelation.documentation _).expects().returns(None) (targetRelation.provides _).expects().returns(Set()) - //(targetRelation.requires _).expects().returns(Set()) + (targetRelation.requires _).expects().returns(Set()) (targetRelation.schema _).expects().returns(None) (targetRelation.describe _).expects(*,Map("outcol"-> SingleValue("part1"))).returns(StructType(Seq())) diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 8271f706c..2ef9927e6 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -55,11 +55,20 @@ background: transparent; } + h3 { + margin-top: 4px; + font-size: x-large; + } + h4 { + margin-top: 2px; + font-size: large; + } h2.separator { background-color: #6ab0de; border-radius: 20px; margin: 4px; padding: 20px; + font-size: xx-large; } .identifier { @@ -92,6 +101,12 @@ padding: 20px; border-radius: 20px; } + div.detailsBox { + background-color: #9fbed4; + padding: 20px; + margin: 10px; + border-radius: 10px; + } div.infoBox { background-color: #5592bb; padding: 20px; @@ -101,12 +116,26 @@ padding: 10px 20px; } - div.bubble { - display: inline; + div.multicolumn { + display: flex; + } + div.column_2 { + flex: 50%; + } + div.column_3 { + flex: 33%; + } + div.column_4 { + flex: 25%; + } + + span.bubble { + display: inline-flex; background-color: #0e84b5; - margin: 2px; + margin: 4px; padding: 6px; border-radius: 20px; + font-size: small; } @@ -184,6 +213,17 @@
#end +#macro(resources $res) + + #foreach($source in ${res}) + + + + + #end +
${source.category}${source.name}
+#end +
@@ -230,8 +270,14 @@

Mapping '${mapping.identifier}'

Description: ${mapping.description}
+ + #if(${mapping.inputs}) +

Inputs

#references(${mapping.inputs}) +
+ #end +

Outputs

#foreach($output in ${mapping.outputs}) @@ -252,17 +298,34 @@

Relation '${relation.identifier}'

Description: ${relation.description}
-

Resources

- - #foreach($resource in ${relation.resources}) - - - - + + #if(${relation.resources}) +
+

Physical Resources

+ #resources(${relation.resources}) +
#end -
${resource.category}${resource.name}
-

Inputs

- #references(${relation.inputs}) + +
+ #if(${relation.sources}) +
+
+

Sources

+ #resources(${relation.sources}) +
+
+ #end + + #if(${relation.inputs}) +
+
+

Direct Inputs

+ #references(${relation.inputs}) +
+
+ #end +
+

Schema

#schema($relation.schema)
@@ -277,14 +340,31 @@

Target '${target.identifier}'

Description: ${target.description}
-

Inputs

- #references(${target.inputs}) -

Outputs

- #references(${target.outputs}) -

Phases

- #foreach($phase in ${target.phases}) -
${phase.name}
- #end + +
+
+
+

Inputs

+ #references(${target.inputs}) +
+
+ +
+
+

Outputs

+ #references(${target.outputs}) +
+
+ +
+
+

Phases

+ #foreach($phase in ${target.phases}) + ${phase.name} + #end +
+
+
#end #end From 15eb40b2238a2cad67f8c3515d012d8c2b112014 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sun, 20 Feb 2022 11:03:31 +0100 Subject: [PATCH 60/95] Add unittest for MappingCollector --- .../documentation/MappingCollectorTest.scala | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala new file mode 100644 index 000000000..84e3ab983 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala @@ -0,0 +1,119 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Execution +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.graph.Graph +import com.dimajix.flowman.graph.Linker +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.Target +import com.dimajix.flowman.types.SingleValue +import com.dimajix.flowman.types.StructType + + +class MappingCollectorTest extends AnyFlatSpec with Matchers with MockFactory { + "RelationCollector.collect" should "work" in { + val mapping1 = mock[Mapping] + val mappingTemplate1 = mock[Prototype[Mapping]] + val mapping2 = mock[Mapping] + val mappingTemplate2 = mock[Prototype[Mapping]] + val sourceRelation = mock[Relation] + val sourceRelationTemplate = mock[Prototype[Relation]] + + val project = Project( + name = "project", + mappings = Map( + "m1" -> mappingTemplate1, + "m2" -> mappingTemplate2 + ), + relations = Map( + "src" -> sourceRelationTemplate + ) + ) + val session = Session.builder().disableSpark().build() + val context = session.getContext(project) + val execution = session.execution + + (mappingTemplate1.instantiate _).expects(context).returns(mapping1) + (mapping1.context _).expects().returns(context) + (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) + + (mappingTemplate2.instantiate _).expects(context).returns(mapping2) + (mapping2.context _).expects().returns(context) + (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map("pcol"-> SingleValue("part1"))))) + + (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) + (sourceRelation.context _).expects().returns(context) + (sourceRelation.link _).expects(*).returns(Unit) + + val graph = Graph.ofProject(session, project, Phase.BUILD) + + (mapping1.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m1")) + (mapping1.inputs _).expects().returns(Seq(MappingOutputIdentifier("project/m2"))) + (mapping1.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) + (mapping1.documentation _).expects().returns(None) + (mapping2.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m2")) + (mapping2.inputs _).expects().returns(Seq()) + (mapping2.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) + (mapping2.documentation _).expects().returns(None) + + val collector = new MappingCollector() + val projectDoc = collector.collect(execution, graph, ProjectDoc(project.name)) + + val mapping1Doc = projectDoc.mappings.find(_.identifier == RelationIdentifier("project/m1")) + val mapping2Doc = projectDoc.mappings.find(_.identifier == RelationIdentifier("project/m2")) + + mapping1Doc should be (Some(MappingDoc( + parent = Some(ProjectReference("project")), + identifier = MappingIdentifier("project/m1"), + inputs = Seq(MappingOutputReference(Some(MappingReference(Some(ProjectReference("project")), "m2")), "main")), + outputs = Seq( + MappingOutputDoc( + parent = Some(MappingReference(Some(ProjectReference("project")), "m1")), + identifier = MappingOutputIdentifier("project/m1:main"), + schema = Some(SchemaDoc( + parent = Some(MappingOutputReference(Some(MappingReference(Some(ProjectReference("project")), "m1")), "main")) + )) + )) + ))) + mapping2Doc should be (Some(MappingDoc( + parent = Some(ProjectReference("project")), + identifier = MappingIdentifier("project/m2"), + inputs = Seq(), + outputs = Seq( + MappingOutputDoc( + parent = Some(MappingReference(Some(ProjectReference("project")), "m2")), + identifier = MappingOutputIdentifier("project/m2:main"), + schema = Some(SchemaDoc( + parent = Some(MappingOutputReference(Some(MappingReference(Some(ProjectReference("project")), "m2")), "main")) + )) + )) + ))) + } +} From 0c2ba82fb892428b4633e0b93d2a4623c928be7d Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sun, 20 Feb 2022 14:09:38 +0100 Subject: [PATCH 61/95] Change Mapping.outputs and Mapping.inputs to return Sets instead of Seqs --- flowman-client/pom.xml | 2 +- .../com/dimajix/flowman/history/graph.scala | 11 +++--- .../com/dimajix/flowman/model/Mapping.scala | 6 +-- .../documentation/ColumnTestTest.scala | 4 +- .../documentation/MappingCollectorTest.scala | 4 +- .../flowman/execution/MappingUtilsTest.scala | 4 +- .../flowman/execution/RootExecutionTest.scala | 6 +-- .../flowman/execution/RunnerTestTest.scala | 10 ++--- .../dimajix/flowman/model/MappingTest.scala | 18 ++++----- .../spec/mapping/AggregateMapping.scala | 6 +-- .../flowman/spec/mapping/AliasMapping.scala | 6 +-- .../spec/mapping/AssembleMapping.scala | 6 +-- .../spec/mapping/CoalesceMapping.scala | 6 +-- .../flowman/spec/mapping/ConformMapping.scala | 6 +-- .../spec/mapping/DeduplicateMapping.scala | 6 +-- .../spec/mapping/DistinctMapping.scala | 6 +-- .../flowman/spec/mapping/DropMapping.scala | 6 +-- .../flowman/spec/mapping/ExplodeMapping.scala | 8 ++-- .../flowman/spec/mapping/ExtendMapping.scala | 6 +-- .../spec/mapping/ExtractJsonMapping.scala | 6 +-- .../flowman/spec/mapping/FilterMapping.scala | 4 +- .../flowman/spec/mapping/FlattenMapping.scala | 4 +- .../mapping/GroupedAggregateMapping.scala | 6 +-- .../spec/mapping/HistorizeMapping.scala | 4 +- .../flowman/spec/mapping/JoinMapping.scala | 12 +++--- .../flowman/spec/mapping/MockMapping.scala | 37 +++++++++---------- .../flowman/spec/mapping/NullMapping.scala | 4 +- .../flowman/spec/mapping/ProjectMapping.scala | 6 +-- .../spec/mapping/ProvidedMapping.scala | 6 +-- .../flowman/spec/mapping/RankMapping.scala | 6 +-- .../spec/mapping/ReadHiveMapping.scala | 6 +-- .../spec/mapping/ReadRelationMapping.scala | 6 +-- .../spec/mapping/ReadStreamMapping.scala | 6 +-- .../spec/mapping/RebalanceMapping.scala | 6 +-- .../spec/mapping/RecursiveSqlMapping.scala | 5 +-- .../spec/mapping/RepartitionMapping.scala | 6 +-- .../flowman/spec/mapping/SchemaMapping.scala | 6 +-- .../flowman/spec/mapping/SelectMapping.scala | 4 +- .../flowman/spec/mapping/SortMapping.scala | 4 +- .../flowman/spec/mapping/SqlMapping.scala | 6 +-- .../flowman/spec/mapping/StackMapping.scala | 4 +- .../spec/mapping/TemplateMapping.scala | 4 +- .../mapping/TransitiveChildrenMapping.scala | 4 +- .../flowman/spec/mapping/UnionMapping.scala | 8 ++-- .../flowman/spec/mapping/UnitMapping.scala | 8 ++-- .../spec/mapping/UnpackJsonMapping.scala | 6 +-- .../flowman/spec/mapping/UpsertMapping.scala | 6 +-- .../flowman/spec/mapping/ValuesMapping.scala | 37 +++++++++---------- .../spec/dataset/MappingDatasetTest.scala | 4 +- .../spec/mapping/AggregateMappingTest.scala | 10 ++--- .../spec/mapping/AliasMappingTest.scala | 5 ++- .../spec/mapping/CoalesceMappingTest.scala | 5 ++- .../spec/mapping/ExtendMappingTest.scala | 4 +- .../spec/mapping/ExtractJsonMappingTest.scala | 4 +- .../spec/mapping/FilterMappingTest.scala | 5 ++- .../spec/mapping/HistorizeMappingTest.scala | 10 ++--- .../spec/mapping/JoinMappingTest.scala | 11 ++---- .../spec/mapping/MockMappingTest.scala | 20 +++++----- .../spec/mapping/NullMappingTest.scala | 9 +++-- .../spec/mapping/ProjectMappingTest.scala | 2 +- .../spec/mapping/RankMappingTest.scala | 8 ++-- .../flowman/spec/mapping/ReadHiveTest.scala | 6 +-- .../spec/mapping/RebalanceMappingTest.scala | 2 +- .../spec/mapping/RepartitionMappingTest.scala | 5 ++- .../spec/mapping/SchemaMappingTest.scala | 10 ++--- .../spec/mapping/SortMappingTest.scala | 5 ++- .../flowman/spec/mapping/SqlMappingTest.scala | 6 +-- .../spec/mapping/TemplateMappingTest.scala | 4 +- .../spec/mapping/UnitMappingTest.scala | 18 ++++----- .../spec/mapping/UpsertMappingTest.scala | 2 +- .../spec/mapping/ValuesMappingTest.scala | 13 ++++--- .../flowman/studio/model/Converter.scala | 4 +- 72 files changed, 260 insertions(+), 266 deletions(-) diff --git a/flowman-client/pom.xml b/flowman-client/pom.xml index bb9c286ab..c10464f8a 100644 --- a/flowman-client/pom.xml +++ b/flowman-client/pom.xml @@ -51,7 +51,7 @@ - *:*:sources + *:sources diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala index 86e1236c0..1567594d4 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -128,18 +128,19 @@ object Graph { */ def ofGraph(graph:g.Graph) : Graph = { val builder = Graph.builder() - val nodesById = graph.nodes.map { + val nodesById = graph.nodes.flatMap { case target:g.TargetRef => val provides = target.provides.map(r => Resource(r.category, r.name, r.partition)).toSeq val requires = target.requires.map(r => Resource(r.category, r.name, r.partition)).toSeq - target.id -> builder.newTargetNode(target.name, target.kind, provides, requires) + Some(target.id -> builder.newTargetNode(target.name, target.kind, provides, requires)) case mapping:g.MappingRef => val requires = mapping.requires.map(r => Resource(r.category, r.name, r.partition)).toSeq - mapping.id -> builder.newMappingNode(mapping.name, mapping.kind, requires) + Some(mapping.id -> builder.newMappingNode(mapping.name, mapping.kind, requires)) case relation:g.RelationRef => val provides = relation.provides.map(r => Resource(r.category, r.name, r.partition)).toSeq val requires = relation.requires.map(r => Resource(r.category, r.name, r.partition)).toSeq - relation.id -> builder.newRelationNode(relation.name, relation.kind, provides, requires) + Some(relation.id -> builder.newRelationNode(relation.name, relation.kind, provides, requires)) + case _ => None }.toMap val relationsById = graph.nodes.collect { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala index 3a3ff029c..309045e6e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Mapping.scala @@ -108,7 +108,7 @@ trait Mapping extends Instance { * Returns the dependencies (i.e. names of tables in the Dataflow model) * @return */ - def inputs : Seq[MappingOutputIdentifier] + def inputs : Set[MappingOutputIdentifier] /** * Lists all outputs of this mapping. Every mapping should have one "main" output, which is the default output @@ -116,7 +116,7 @@ trait Mapping extends Instance { * recommended. * @return */ - def outputs : Seq[String] + def outputs : Set[String] /** * Creates an output identifier for the primary output @@ -213,7 +213,7 @@ abstract class BaseMapping extends AbstractInstance with Mapping { * Lists all outputs of this mapping. Every mapping should have one "main" output * @return */ - override def outputs : Seq[String] = Seq("main") + override def outputs : Set[String] = Set("main") /** * Creates an output identifier for the primary output diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala index 4c1d28b07..37a982b14 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala @@ -235,8 +235,8 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc (mappingSpec.instantiate _).expects(*).returns(mapping) (mapping.context _).expects().returns(context) - (mapping.inputs _).expects().returns(Seq()) - (mapping.outputs _).expects().atLeastOnce().returns(Seq("main")) + (mapping.inputs _).expects().returns(Set()) + (mapping.outputs _).expects().atLeastOnce().returns(Set("main")) (mapping.broadcast _).expects().returns(false) (mapping.cache _).expects().returns(StorageLevel.NONE) (mapping.checkpoint _).expects().returns(false) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala index 84e3ab983..77a704475 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala @@ -75,11 +75,11 @@ class MappingCollectorTest extends AnyFlatSpec with Matchers with MockFactory { val graph = Graph.ofProject(session, project, Phase.BUILD) (mapping1.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m1")) - (mapping1.inputs _).expects().returns(Seq(MappingOutputIdentifier("project/m2"))) + (mapping1.inputs _).expects().returns(Set(MappingOutputIdentifier("project/m2"))) (mapping1.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) (mapping1.documentation _).expects().returns(None) (mapping2.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m2")) - (mapping2.inputs _).expects().returns(Seq()) + (mapping2.inputs _).expects().returns(Set()) (mapping2.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) (mapping2.documentation _).expects().returns(None) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/execution/MappingUtilsTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/execution/MappingUtilsTest.scala index 00f6ab993..f8873d0be 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/execution/MappingUtilsTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/execution/MappingUtilsTest.scala @@ -35,7 +35,7 @@ object MappingUtilsTest { case class DummyMapping( override val context: Context, override val name: String, - override val inputs: Seq[MappingOutputIdentifier], + override val inputs: Set[MappingOutputIdentifier], override val requires: Set[ResourceIdentifier] ) extends BaseMapping { protected override def instanceProperties: Mapping.Properties = Mapping.Properties(context, name) @@ -47,7 +47,7 @@ object MappingUtilsTest { inputs: Seq[MappingOutputIdentifier], requires: Set[ResourceIdentifier] ) extends Prototype[Mapping] { - override def instantiate(context: Context): Mapping = DummyMapping(context, name, inputs, requires) + override def instantiate(context: Context): Mapping = DummyMapping(context, name, inputs.toSet, requires) } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootExecutionTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootExecutionTest.scala index f2ceb1127..e0e47a5e9 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootExecutionTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RootExecutionTest.scala @@ -39,7 +39,7 @@ import com.dimajix.spark.testing.LocalSparkSession object RootExecutionTest { case class TestMapping( instanceProperties: Mapping.Properties, - inputs:Seq[MappingOutputIdentifier] + inputs:Set[MappingOutputIdentifier] ) extends BaseMapping { override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = { Map("main" -> input.values.head) @@ -49,7 +49,7 @@ object RootExecutionTest { case class RangeMapping( instanceProperties: Mapping.Properties ) extends BaseMapping { - override def inputs: Seq[MappingOutputIdentifier] = Seq() + override def inputs: Set[MappingOutputIdentifier] = Set.empty override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = { val spark = execution.spark @@ -61,7 +61,7 @@ object RootExecutionTest { override def instantiate(context: Context): Mapping = { TestMapping( Mapping.Properties(context, name), - inputs.map(i => MappingOutputIdentifier(i)) + inputs.map(i => MappingOutputIdentifier(i)).toSet ) } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerTestTest.scala index 27d5a0cc7..f23855045 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerTestTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -193,8 +193,8 @@ class RunnerTestTest extends AnyFlatSpec with MockFactory with Matchers with Loc overrideMappingContext = ctx overrideMapping } - (overrideMapping.inputs _).expects().atLeastOnce().returns(Seq()) - (overrideMapping.outputs _).expects().atLeastOnce().returns(Seq("main")) + (overrideMapping.inputs _).expects().atLeastOnce().returns(Set()) + (overrideMapping.outputs _).expects().atLeastOnce().returns(Set("main")) (overrideMapping.identifier _).expects().atLeastOnce().returns(MappingIdentifier("map")) (overrideMapping.context _).expects().onCall(() => overrideMappingContext) (overrideMapping.broadcast _).expects().returns(false) @@ -312,8 +312,8 @@ class RunnerTestTest extends AnyFlatSpec with MockFactory with Matchers with Loc mappingContext = ctx mapping } - (mapping.inputs _).expects().atLeastOnce().returns(Seq()) - (mapping.outputs _).expects().atLeastOnce().returns(Seq("main")) + (mapping.inputs _).expects().atLeastOnce().returns(Set()) + (mapping.outputs _).expects().atLeastOnce().returns(Set("main")) (mapping.identifier _).expects().atLeastOnce().returns(MappingIdentifier("map")) (mapping.context _).expects().onCall(() => mappingContext) (mapping.broadcast _).expects().returns(false) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala index 32f34ab43..c0a29f722 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,10 +38,10 @@ import com.dimajix.spark.testing.LocalSparkSession object MappingTest { - class DummyMapping(props:Mapping.Properties, ins:Seq[MappingOutputIdentifier]) extends BaseMapping { + class DummyMapping(props:Mapping.Properties, ins:Set[MappingOutputIdentifier]) extends BaseMapping { protected override def instanceProperties: Mapping.Properties = props - override def inputs: Seq[MappingOutputIdentifier] = ins + override def inputs: Set[MappingOutputIdentifier] = ins override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = { val df = input.head._2.groupBy("id").agg(f.sum("val")) @@ -60,7 +60,7 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val mapping = new DummyMapping( Mapping.Properties(context, "m1"), - Seq() + Set() ) mapping.metadata should be (Metadata( @@ -83,7 +83,7 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val mapping = new DummyMapping( Mapping.Properties(context, "m1"), - Seq() + Set() ) mapping.output("main") should be (MappingOutputIdentifier("project/m1:main")) an[NoSuchMappingOutputException] should be thrownBy(mapping.output("no_such_output")) @@ -95,7 +95,7 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val mapping = new DummyMapping( Mapping.Properties(context, "m1"), - Seq() + Set() ) mapping.output("main") should be (MappingOutputIdentifier("m1:main")) an[NoSuchMappingOutputException] should be thrownBy(mapping.output("no_such_output")) @@ -108,7 +108,7 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val mapping = new DummyMapping( Mapping.Properties(context, "m1"), - Seq(MappingOutputIdentifier("input:main")) + Set(MappingOutputIdentifier("input:main")) ) val inputSchema = StructType(Seq( @@ -140,11 +140,11 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val mapping1 = new DummyMapping( Mapping.Properties(context, "m1"), - Seq(MappingOutputIdentifier("m2")) + Set(MappingOutputIdentifier("m2")) ) val mapping2 = new DummyMapping( Mapping.Properties(context, "m2"), - Seq() + Set() ) //(mappingTemplate1.instantiate _).expects(context).returns(mapping1) (mappingTemplate2.instantiate _).expects(context).returns(mapping2) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala index eb3d47047..13e511066 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AggregateMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -68,8 +68,8 @@ case class AggregateMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala index ed070eb5f..7b45c2011 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AliasMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,8 +36,8 @@ case class AliasMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala index d211da657..bd01b91b1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/AssembleMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,8 +126,8 @@ case class AssembleMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala index 6d48096a8..6b1a3346d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/CoalesceMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,8 +54,8 @@ case class CoalesceMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala index 7519888a8..45caf0b83 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ConformMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,8 +47,8 @@ extends BaseMapping { * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs: Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala index d7cc89a2e..311e334e3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DeduplicateMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,8 +38,8 @@ case class DeduplicateMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala index 235885fc1..74487d88a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DistinctMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,8 @@ case class DistinctMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala index 9a3804758..81c055aee 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/DropMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ case class DropMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs: Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala index a3d6a7f5d..d638602d7 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExplodeMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,15 +53,15 @@ case class ExplodeMapping( flatten: Boolean = false, naming: CaseFormat = CaseFormat.SNAKE_CASE ) extends BaseMapping { - override def outputs: Seq[String] = Seq("main", "explode") + override def outputs: Set[String] = Set("main", "explode") /** * Returns the dependencies (i.e. names of tables in the Dataflow model) * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs: Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtendMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtendMapping.scala index 67147dc5c..215b9cb7c 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtendMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtendMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,8 +39,8 @@ case class ExtendMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala index 019c644de..72b2e1a8d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMapping.scala @@ -53,8 +53,8 @@ case class ExtractJsonMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } @@ -63,7 +63,7 @@ case class ExtractJsonMapping( * * @return */ - override def outputs: Seq[String] = Seq("main", "error") + override def outputs: Set[String] = Set("main", "error") /** * Executes this MappingType and returns a corresponding DataFrame diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala index 703ffb392..6ac27e2a0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FilterMapping.scala @@ -37,8 +37,8 @@ case class FilterMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala index 325da2546..625b699d0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/FlattenMapping.scala @@ -40,8 +40,8 @@ case class FlattenMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/GroupedAggregateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/GroupedAggregateMapping.scala index 7607f0646..c75df85b2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/GroupedAggregateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/GroupedAggregateMapping.scala @@ -67,15 +67,15 @@ case class GroupedAggregateMapping( * recommended. * @return */ - override def outputs: Seq[String] = groups.keys.toSeq :+ "cache" + override def outputs: Set[String] = groups.keys.toSet + "cache" /** * Returns the dependencies of this mapping, which is exactly one input table * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala index dbae9f5b4..9f837d37b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/HistorizeMapping.scala @@ -47,8 +47,8 @@ case class HistorizeMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/JoinMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/JoinMapping.scala index 240b77b93..560df3ae1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/JoinMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/JoinMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ case class JoinMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - input + override def inputs : Set[MappingOutputIdentifier] = { + input.toSet } /** @@ -56,10 +56,10 @@ case class JoinMapping( require(tables != null) val result = if (condition.nonEmpty) { - require(inputs.size == 2, "Joining using an condition only supports exactly two inputs") + require(input.size == 2, "Joining using an condition only supports exactly two inputs") - val left = inputs(0) - val right = inputs(1) + val left = input(0) + val right = input(1) val leftDf = tables(left).as(left.name) val rightDf = tables(right).as(right.name) leftDf.join(rightDf, expr(condition), mode) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala index aadc8f14a..913e5ab9a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/MockMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,23 @@ case class MockMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = Seq() + override def inputs: Set[MappingOutputIdentifier] = Set.empty + + /** + * Creates an output identifier for the primary output + * + * @return + */ + override def output: MappingOutputIdentifier = { + MappingOutputIdentifier(identifier, mocked.output.output) + } + + /** + * Lists all outputs of this mapping. Every mapping should have one "main" output + * + * @return + */ + override def outputs: Set[String] = mocked.outputs /** * Executes this Mapping and returns a corresponding map of DataFrames per output @@ -74,23 +90,6 @@ case class MockMapping( } } - - /** - * Creates an output identifier for the primary output - * - * @return - */ - override def output: MappingOutputIdentifier = { - MappingOutputIdentifier(identifier, mocked.output.output) - } - - /** - * Lists all outputs of this mapping. Every mapping should have one "main" output - * - * @return - */ - override def outputs: Seq[String] = mocked.outputs - /** * Returns the schema as produced by this mapping, relative to the given input schema. The map might not contain * schema information for all outputs, if the schema cannot be inferred. diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala index 9a162df5b..7cde7e32c 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/NullMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,7 +55,7 @@ case class NullMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = Seq() + override def inputs: Set[MappingOutputIdentifier] = Set.empty /** * Executes this Mapping and returns a corresponding map of DataFrames per output diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala index 6d3582bb3..0698db051 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProjectMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,8 +50,8 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProvidedMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProvidedMapping.scala index c1c755291..7f54f89b9 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProvidedMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ProvidedMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,9 +36,7 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq() - } + override def inputs : Set[MappingOutputIdentifier] = Set.empty /** * Instantiates the specified table, which must be available in the Spark session diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala index 945507587..1cf05dd50 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RankMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,8 +61,8 @@ case class RankMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala index d3db1b559..c8d73d156 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,9 +61,7 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq() - } + override def inputs : Set[MappingOutputIdentifier] = Set.empty /** * Executes this Transform by reading from the specified source and returns a corresponding DataFrame diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala index 471a94e6f..eb6b3ff43 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,9 +66,7 @@ case class ReadRelationMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq() - } + override def inputs : Set[MappingOutputIdentifier] = Set.empty /** * Executes this Transform by reading from the specified source and returns a corresponding DataFrame diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala index 75ed819a2..852265a36 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -62,9 +62,7 @@ case class ReadStreamMapping ( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq() - } + override def inputs : Set[MappingOutputIdentifier] = Set.empty /** * Executes this Transform by reading from the specified source and returns a corresponding DataFrame diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala index 8a93e8bcf..5460cf469 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RebalanceMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,8 @@ case class RebalanceMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala index af7ba692a..b5b7858a2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RecursiveSqlMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,11 +59,10 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { + override def inputs : Set[MappingOutputIdentifier] = { SqlParser.resolveDependencies(statement) .filter(_.toLowerCase(Locale.ROOT) != "__this__") .map(MappingOutputIdentifier.parse) - .toSeq } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala index 446659433..b13b6d308 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/RepartitionMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ case class RepartitionMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala index f600da62e..b04128eb0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SchemaMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,8 +53,8 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala index 915acd069..e2334d886 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SelectMapping.scala @@ -44,8 +44,8 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala index 10cbb5626..856b6e7c6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SortMapping.scala @@ -38,8 +38,8 @@ case class SortMapping( * Returns the dependencies (i.e. names of tables in the Dataflow model) * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SqlMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SqlMapping.scala index 99ce1bb29..a9f4e82ca 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SqlMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/SqlMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -64,8 +64,8 @@ extends BaseMapping { * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - SqlParser.resolveDependencies(statement).map(MappingOutputIdentifier.parse).toSeq + override def inputs : Set[MappingOutputIdentifier] = { + SqlParser.resolveDependencies(statement).map(MappingOutputIdentifier.parse) } private def statement : String = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala index fb64b58ad..73d5fe4a2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/StackMapping.scala @@ -50,8 +50,8 @@ case class StackMapping( * Returns the dependencies (i.e. names of tables in the Dataflow model) * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala index dacc945d6..31452389b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TemplateMapping.scala @@ -78,7 +78,7 @@ case class TemplateMapping( * * @return */ - override def outputs : Seq[String] = { + override def outputs : Set[String] = { mappingInstance.outputs } @@ -87,7 +87,7 @@ case class TemplateMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = { + override def inputs: Set[MappingOutputIdentifier] = { mappingInstance.inputs } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala index 86c33cb36..35238551f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/TransitiveChildrenMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ case class TransitiveChildrenMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = Seq(input) + override def inputs: Set[MappingOutputIdentifier] = Set(input) /** * Executes this MappingType and returns a corresponding DataFrame diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala index 636564c6c..5d5ceb0a6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnionMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,8 +43,8 @@ case class UnionMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - input + override def inputs : Set[MappingOutputIdentifier] = { + input.toSet } /** @@ -58,7 +58,7 @@ case class UnionMapping( require(execution != null) require(tables != null) - val dfs = inputs.map(tables(_)) + val dfs = input.map(tables(_)) // Now create a union of all tables val union = diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala index f3491b909..f48b0a8f1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnitMapping.scala @@ -62,10 +62,10 @@ case class UnitMapping( * Return all outputs provided by this unit * @return */ - override def outputs: Seq[String] = { + override def outputs: Set[String] = { mappingInstances .filter(_._2.outputs.contains("main")) - .keys.toSeq + .keySet } /** @@ -73,14 +73,14 @@ case class UnitMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = { + override def inputs: Set[MappingOutputIdentifier] = { // For all mappings, find only external dependencies. val ownMappings = mappingInstances.keySet mappingInstances.values .filter(_.outputs.contains("main")) .flatMap(_.inputs) .filter(dep => dep.project.nonEmpty || !ownMappings.contains(dep.name)) - .toSeq + .toSet } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala index 2a1672f38..9663b7a5a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UnpackJsonMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,8 +55,8 @@ case class UnpackJsonMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala index 2066bbba3..12ff8c45f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/UpsertMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ case class UpsertMapping( * * @return */ - override def inputs : Seq[MappingOutputIdentifier] = { - Seq(input, updates) + override def inputs : Set[MappingOutputIdentifier] = { + Set(input, updates) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala index b6b949861..31766aad6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ValuesMapping.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,40 +53,39 @@ case class ValuesMapping( * * @return */ - override def inputs: Seq[MappingOutputIdentifier] = Seq() + override def inputs: Set[MappingOutputIdentifier] = Set.empty /** - * Executes this Mapping and returns a corresponding map of DataFrames per output + * Creates an output identifier for the primary output * - * @param execution - * @param input * @return */ - override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = { - val recordsSchema = StructType(schema.map(_.fields).getOrElse(columns)) - val sparkSchema = recordsSchema.sparkType - - val values = records.map(_.toArray(recordsSchema)) - val df = DataFrameBuilder.ofStringValues(execution.spark, values, sparkSchema) - Map("main" -> df) + override def output: MappingOutputIdentifier = { + MappingOutputIdentifier(identifier, "main") } - /** - * Creates an output identifier for the primary output + * Lists all outputs of this mapping. Every mapping should have one "main" output * * @return */ - override def output: MappingOutputIdentifier = { - MappingOutputIdentifier(identifier, "main") - } + override def outputs: Set[String] = Set("main") /** - * Lists all outputs of this mapping. Every mapping should have one "main" output + * Executes this Mapping and returns a corresponding map of DataFrames per output * + * @param execution + * @param input * @return */ - override def outputs: Seq[String] = Seq("main") + override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = { + val recordsSchema = StructType(schema.map(_.fields).getOrElse(columns)) + val sparkSchema = recordsSchema.sparkType + + val values = records.map(_.toArray(recordsSchema)) + val df = DataFrameBuilder.ofStringValues(execution.spark, values, sparkSchema) + Map("main" -> df) + } /** * Returns the schema as produced by this mapping, relative to the given input schema. The map might not contain diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/MappingDatasetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/MappingDatasetTest.scala index d9f0cb2f5..c4dda06d9 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/MappingDatasetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/MappingDatasetTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ object MappingDatasetTest { ) extends BaseMapping { protected override def instanceProperties: Mapping.Properties = Mapping.Properties(context, name) - override def inputs: Seq[MappingOutputIdentifier] = Seq() + override def inputs: Set[MappingOutputIdentifier] = Set.empty override def execute(execution: Execution, input: Map[MappingOutputIdentifier, DataFrame]): Map[String, DataFrame] = Map("main" -> execution.spark.emptyDataFrame) override def describe(execution: Execution, input: Map[MappingOutputIdentifier, StructType]): Map[String, StructType] = Map("main"-> new StructType()) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AggregateMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AggregateMappingTest.scala index 8008940c8..91c5aa010 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AggregateMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AggregateMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,10 +61,10 @@ class AggregateMappingTest extends AnyFlatSpec with Matchers with LocalSparkSess ) xfs.input should be (MappingOutputIdentifier("myview")) - xfs.outputs should be (Seq("main")) + xfs.outputs should be (Set("main")) xfs.dimensions should be (Array("_1", "_2")) xfs.aggregations should be (Map("agg3" -> "sum(_3)", "agg4" -> "sum(_4)", "agg5" -> "sum(_4)", "agg6" -> "sum(_4)", "agg7" -> "sum(_4)")) - xfs.inputs should be (Seq(MappingOutputIdentifier("myview"))) + xfs.inputs should be (Set(MappingOutputIdentifier("myview"))) val df2 = xfs.execute(executor, Map(MappingOutputIdentifier("myview") -> df))("main") .orderBy("_1", "_2") @@ -107,10 +107,10 @@ class AggregateMappingTest extends AnyFlatSpec with Matchers with LocalSparkSess ) xfs.input should be (MappingOutputIdentifier("myview")) - xfs.outputs should be (Seq("main")) + xfs.outputs should be (Set("main")) xfs.dimensions should be (Seq("_1 AS dim1", "upper(_2) AS dim2")) xfs.aggregations should be (Map("agg3" -> "sum(_3)")) - xfs.inputs should be (Seq(MappingOutputIdentifier("myview"))) + xfs.inputs should be (Set(MappingOutputIdentifier("myview"))) val df2 = xfs.execute(executor, Map(MappingOutputIdentifier("myview") -> df))("main") .orderBy("dim1", "dim2") diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AliasMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AliasMappingTest.scala index 7af32d4fc..45cc474b8 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AliasMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/AliasMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,7 +59,8 @@ class AliasMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession val inputDf = spark.emptyDataFrame mapping.input should be (MappingOutputIdentifier("input_df:output_2")) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set(MappingOutputIdentifier("input_df:output_2"))) + mapping.outputs should be (Set("main")) val result = mapping.execute(executor, Map(MappingOutputIdentifier("input_df:output_2") -> inputDf))("main") result.count() should be (0) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/CoalesceMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/CoalesceMappingTest.scala index 10f12d1ba..55e7ff927 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/CoalesceMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/CoalesceMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,8 @@ class CoalesceMappingTest extends AnyFlatSpec with Matchers with LocalSparkSessi val typedInstance = instance.asInstanceOf[CoalesceMapping] typedInstance.input should be (MappingOutputIdentifier("some_mapping")) - typedInstance.outputs should be (Seq("main")) + typedInstance.inputs should be (Set(MappingOutputIdentifier("some_mapping"))) + typedInstance.outputs should be (Set("main")) typedInstance.partitions should be (1) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtendMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtendMappingTest.scala index 47bd9ec2b..2b66a5d7a 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtendMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtendMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,8 +44,8 @@ class ExtendMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession Map("new_f" -> "2*_2") ) xfs.input should be (MappingOutputIdentifier("myview")) + xfs.inputs should be (Set(MappingOutputIdentifier("myview"))) xfs.columns should be (Map("new_f" -> "2*_2")) - xfs.inputs should be (Seq(MappingOutputIdentifier("myview"))) val result = xfs.execute(executor, Map(MappingOutputIdentifier("myview") -> df))("main") .orderBy("_1").collect() diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMappingTest.scala index 3c32b71d2..c1d41ec7d 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ExtractJsonMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -122,7 +122,7 @@ class ExtractJsonMappingTest extends AnyFlatSpec with Matchers with LocalSparkSe ) val mapping = context.getMapping(MappingIdentifier("m0")) - mapping.outputs should be (Seq("main", "error")) + mapping.outputs should be (Set("main", "error")) val result = mapping.execute(executor, Map(MappingOutputIdentifier("p0") -> input))("main") result.count() should be (2) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/FilterMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/FilterMappingTest.scala index 9ffb2d0a5..c663a07e7 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/FilterMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/FilterMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,8 @@ class FilterMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession val filter = instance.asInstanceOf[FilterMapping] filter.input should be (MappingOutputIdentifier("some_mapping")) - filter.outputs should be (Seq("main")) + filter.inputs should be (Set(MappingOutputIdentifier("some_mapping"))) + filter.outputs should be (Set("main")) filter.condition should be ("value < 50") } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/HistorizeMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/HistorizeMappingTest.scala index 1399f6c9e..b938dc0d6 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/HistorizeMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/HistorizeMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,12 +67,12 @@ class HistorizeMappingTest extends AnyFlatSpec with Matchers with LocalSparkSess "valid_to" ) mapping.input should be (MappingOutputIdentifier("df1")) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"))) + mapping.outputs should be (Set("main")) mapping.keyColumns should be (Seq("id" )) mapping.timeColumn should be ("ts") mapping.validFromColumn should be ("valid_from") mapping.validToColumn should be ("valid_to") - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"))) val expectedSchema = StructType(Seq( StructField("a", ArrayType(LongType)), @@ -123,12 +123,12 @@ class HistorizeMappingTest extends AnyFlatSpec with Matchers with LocalSparkSess InsertPosition.BEGINNING ) mapping.input should be (MappingOutputIdentifier("df1")) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"))) + mapping.outputs should be (Set("main")) mapping.keyColumns should be (Seq("id" )) mapping.timeColumn should be ("ts") mapping.validFromColumn should be ("valid_from") mapping.validToColumn should be ("valid_to") - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"))) val expectedSchema = StructType(Seq( StructField("valid_from", LongType), diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/JoinMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/JoinMappingTest.scala index 73b6bf5ce..ceac86ccb 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/JoinMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/JoinMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,9 +67,8 @@ class JoinMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession{ Seq("key"), mode="left" ) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) mapping.columns should be (Seq("key" )) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) val resultDf = mapping.execute(executor, Map(MappingOutputIdentifier("df1") -> df1, MappingOutputIdentifier("df2") -> df2))("main") .orderBy("key") @@ -120,9 +119,8 @@ class JoinMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession{ condition="df1.key = df2.key", mode="left" ) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) mapping.condition should be ("df1.key = df2.key") - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) val resultDf = mapping.execute(executor, Map(MappingOutputIdentifier("df1") -> df1, MappingOutputIdentifier("df2") -> df2))("main") .orderBy("df1.key") @@ -154,8 +152,7 @@ class JoinMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession{ mapping shouldBe a[JoinMappingSpec] val join = mapping.instantiate(session.context).asInstanceOf[JoinMapping] - join.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) + join.inputs should be (Set(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) join.condition should be ("df1.key = df2.key") - join.inputs should be (Seq(MappingOutputIdentifier("df1"), MappingOutputIdentifier("df2"))) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/MockMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/MockMappingTest.scala index 8a5d18de6..ec4902c96 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/MockMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/MockMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ class MockMappingTest extends AnyFlatSpec with Matchers with MockFactory with Lo mapping.kind should be ("mock") mapping.mapping should be (MappingIdentifier("empty")) mapping.output should be (MappingOutputIdentifier("project/mock:main")) - mapping.outputs should be (Seq("main")) + mapping.outputs should be (Set("main")) mapping.records should be (Seq( ArrayRecord("a","12","3"), ArrayRecord("cat","","7"), @@ -113,14 +113,14 @@ class MockMappingTest extends AnyFlatSpec with Matchers with MockFactory with Lo mapping.category should be (Category.MAPPING) (baseMappingTemplate.instantiate _).expects(context).returns(baseMapping) - (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Seq("other", "error")) - mapping.outputs should be (Seq("other", "error")) + (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Set("other", "error")) + mapping.outputs should be (Set("other", "error")) (baseMapping.output _).expects().returns(MappingOutputIdentifier("base", "other", Some(project.name))) mapping.output should be (MappingOutputIdentifier("my_project/mock:other")) (baseMapping.context _).expects().anyNumberOfTimes().returns(context) - (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Seq()) + (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Set()) (baseMapping.identifier _).expects().anyNumberOfTimes().returns(MappingIdentifier("my_project/base")) (baseMapping.describe:(Execution,Map[MappingOutputIdentifier,StructType],String) => StructType).expects(executor,*,"other") .anyNumberOfTimes().returns(otherSchema) @@ -177,14 +177,14 @@ class MockMappingTest extends AnyFlatSpec with Matchers with MockFactory with Lo val mapping = context.getMapping(MappingIdentifier("mock")) (baseMappingTemplate.instantiate _).expects(context).returns(baseMapping) - (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Seq("main")) - mapping.outputs should be (Seq("main")) + (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Set("main")) + mapping.outputs should be (Set("main")) (baseMapping.output _).expects().returns(MappingOutputIdentifier("mock", "main", Some(project.name))) mapping.output should be (MappingOutputIdentifier("my_project/mock:main")) (baseMapping.context _).expects().anyNumberOfTimes().returns(context) - (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Seq()) + (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Set()) (baseMapping.identifier _).expects().anyNumberOfTimes().returns(MappingIdentifier("my_project/base")) (baseMapping.describe:(Execution,Map[MappingOutputIdentifier,StructType],String) => StructType).expects(executor,*,"main") .anyNumberOfTimes().returns(schema) @@ -233,8 +233,8 @@ class MockMappingTest extends AnyFlatSpec with Matchers with MockFactory with Lo (baseMappingTemplate.instantiate _).expects(context).returns(baseMapping) (baseMapping.context _).expects().anyNumberOfTimes().returns(context) - (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Seq("main")) - (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Seq()) + (baseMapping.outputs _).expects().anyNumberOfTimes().returns(Set("main")) + (baseMapping.inputs _).expects().anyNumberOfTimes().returns(Set()) (baseMapping.identifier _).expects().anyNumberOfTimes().returns(MappingIdentifier("my_project/base")) (baseMapping.describe:(Execution,Map[MappingOutputIdentifier,StructType],String) => StructType).expects(executor,*,"main") .anyNumberOfTimes().returns(schema) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/NullMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/NullMappingTest.scala index ec4b35d32..a65ff3c56 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/NullMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/NullMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,7 +71,8 @@ class NullMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { )) mapping1.schema should be (None) mapping1.output should be (MappingOutputIdentifier("project/empty1:main")) - mapping1.outputs should be (Seq("main")) + mapping1.outputs should be (Set("main")) + mapping1.inputs should be (Set.empty) } it should "create empty DataFrames with specified columns" in { @@ -90,7 +91,7 @@ class NullMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping.category should be (Category.MAPPING) //mapping.kind should be ("null") - mapping.outputs should be (Seq("main")) + mapping.outputs should be (Set("main")) mapping.output should be (MappingOutputIdentifier("empty")) mapping.describe(executor, Map()) should be (Map( @@ -131,7 +132,7 @@ class NullMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping.category should be (Category.MAPPING) //mapping.kind should be ("null") - mapping.outputs should be (Seq("main")) + mapping.outputs should be (Set("main")) mapping.output should be (MappingOutputIdentifier("empty")) mapping.describe(executor, Map()) should be (Map( diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ProjectMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ProjectMappingTest.scala index 31e614136..e1a4286a1 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ProjectMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ProjectMappingTest.scala @@ -52,7 +52,7 @@ class ProjectMappingTest extends AnyFlatSpec with Matchers with LocalSparkSessio mapping.input should be (MappingOutputIdentifier("myview")) mapping.columns should be (Seq(ProjectTransformer.Column(Path("_2")))) - mapping.inputs should be (Seq(MappingOutputIdentifier("myview"))) + mapping.inputs should be (Set(MappingOutputIdentifier("myview"))) val result = mapping.execute(executor, Map(MappingOutputIdentifier("myview") -> df))("main") .orderBy("_2").collect() diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RankMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RankMappingTest.scala index c71dd843a..e8bba2f68 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RankMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RankMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,7 +71,7 @@ class RankMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping.input should be (MappingOutputIdentifier("df1")) mapping.keyColumns should be (Seq("id" )) mapping.versionColumns should be (Seq("ts")) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"))) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"))) val result = mapping.execute(executor, Map(MappingOutputIdentifier("df1") -> df))("main") result.schema should be (df.schema) @@ -112,7 +112,7 @@ class RankMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping.input should be (MappingOutputIdentifier("df1")) mapping.keyColumns should be (Seq("id" )) mapping.versionColumns should be (Seq("ts")) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"))) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"))) val result = mapping.execute(executor, Map(MappingOutputIdentifier("df1") -> df))("main") result.schema should be (df.schema) @@ -177,7 +177,7 @@ class RankMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping.input should be (MappingOutputIdentifier("df1")) mapping.keyColumns should be (Seq("id._1" )) mapping.versionColumns should be (Seq("ts._2")) - mapping.inputs should be (Seq(MappingOutputIdentifier("df1"))) + mapping.inputs should be (Set(MappingOutputIdentifier("df1"))) val result = mapping.execute(executor, Map(MappingOutputIdentifier("df1") -> df))("main") result.schema should be (df.schema) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala index 2da406ebf..5c8393b79 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { ResourceIdentifier.ofHiveTable("lala_0007", Some("default")), ResourceIdentifier.ofHiveDatabase("default") )) - mapping.inputs should be (Seq()) + mapping.inputs should be (Set()) mapping.describe(execution, Map()) should be (Map( "main" -> ftypes.StructType(Seq( Field("str_col", ftypes.StringType), @@ -142,7 +142,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { ResourceIdentifier.ofHiveTable("lala_0007", Some("default")), ResourceIdentifier.ofHiveDatabase("default") )) - mapping.inputs should be (Seq()) + mapping.inputs should be (Set()) mapping.describe(execution, Map()) should be (Map( "main" -> ftypes.StructType(Seq( Field("int_col", ftypes.DoubleType) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RebalanceMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RebalanceMappingTest.scala index 4ccb81b83..87717d6c4 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RebalanceMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RebalanceMappingTest.scala @@ -51,7 +51,7 @@ class RebalanceMappingTest extends AnyFlatSpec with Matchers with LocalSparkSess val typedInstance = instance.asInstanceOf[RebalanceMapping] typedInstance.input should be (MappingOutputIdentifier("some_mapping")) - typedInstance.outputs should be (Seq("main")) + typedInstance.outputs should be (Set("main")) typedInstance.partitions should be (2) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RepartitionMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RepartitionMappingTest.scala index 860e633b9..04702dfaf 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RepartitionMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/RepartitionMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,7 +56,8 @@ class RepartitionMappingTest extends AnyFlatSpec with Matchers with LocalSparkSe val typedInstance = instance.asInstanceOf[RepartitionMapping] typedInstance.input should be (MappingOutputIdentifier("some_mapping")) - typedInstance.outputs should be (Seq("main")) + typedInstance.inputs should be (Set(MappingOutputIdentifier("some_mapping"))) + typedInstance.outputs should be (Set("main")) typedInstance.partitions should be (2) typedInstance.columns should be (Seq("col_1", "col_2")) typedInstance.sort should be (true) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SchemaMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SchemaMappingTest.scala index b493a1098..848a93798 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SchemaMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SchemaMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -59,7 +59,7 @@ class SchemaMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession project.mappings.contains("t1") should be (true) val mapping = context.getMapping(MappingIdentifier("t1")).asInstanceOf[SchemaMapping] - mapping.inputs should be (Seq(MappingOutputIdentifier("t0"))) + mapping.inputs should be (Set(MappingOutputIdentifier("t0"))) mapping.output should be (MappingOutputIdentifier("project/t1:main")) mapping.identifier should be (MappingIdentifier("project/t1")) mapping.schema should be (None) @@ -90,7 +90,7 @@ class SchemaMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession mapping.input should be (MappingOutputIdentifier("myview")) mapping.columns should be (Seq(Field("_2", FieldType.of("int")))) - mapping.inputs should be (Seq(MappingOutputIdentifier("myview"))) + mapping.inputs should be (Set(MappingOutputIdentifier("myview"))) mapping.output should be (MappingOutputIdentifier("map:main")) mapping.identifier should be (MappingIdentifier("map")) @@ -128,8 +128,8 @@ class SchemaMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession ) mapping.input should be (MappingOutputIdentifier("myview")) - mapping.inputs should be (Seq(MappingOutputIdentifier("myview"))) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set(MappingOutputIdentifier("myview"))) + mapping.outputs should be (Set("main")) val result = mapping.execute(executor, Map(MappingOutputIdentifier("myview") -> df))("main") .orderBy("_2") diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SortMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SortMappingTest.scala index b57a93dc6..2a9b95fcf 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SortMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SortMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,7 +54,8 @@ class SortMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val typedInstance = instance.asInstanceOf[SortMapping] typedInstance.input should be (MappingOutputIdentifier("some_mapping")) - typedInstance.outputs should be (Seq("main")) + typedInstance.inputs should be (Set(MappingOutputIdentifier("some_mapping"))) + typedInstance.outputs should be (Set("main")) typedInstance.columns should be (Seq( "c1" -> SortOrder(Ascending, NullsFirst), "c2" -> SortOrder(Descending, NullsFirst) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SqlMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SqlMappingTest.scala index edd10d436..6905ccede 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SqlMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/SqlMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -98,7 +98,7 @@ class SqlMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val session = Session.builder().withSparkSession(spark).build() val context = session.getContext(project) val mapping = context.getMapping(MappingIdentifier("t1")) - mapping.inputs should be (Seq(MappingOutputIdentifier("t0"))) + mapping.inputs should be (Set(MappingOutputIdentifier("t0"))) } it should "also be correct with subqueries" in { @@ -143,7 +143,7 @@ class SqlMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val session = Session.builder().withSparkSession(spark).build() val context = session.getContext(project) val mapping = context.getMapping(MappingIdentifier("t1")) - mapping.inputs.map(_.name).sorted should be (Seq("other_table", "some_table", "some_table_archive")) + mapping.inputs.map(_.name) should be (Set("other_table", "some_table", "some_table_archive")) } it should "execute the SQL query" in { diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/TemplateMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/TemplateMappingTest.scala index d7055577e..6c5a07b77 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/TemplateMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/TemplateMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,6 +56,6 @@ class TemplateMappingTest extends AnyFlatSpec with Matchers { mapping shouldBe a[TemplateMapping] mapping.name should be ("template") - mapping.inputs should be (Seq(MappingOutputIdentifier("lala"))) + mapping.inputs should be (Set(MappingOutputIdentifier("lala"))) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UnitMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UnitMappingTest.scala index 09cdebec7..b3a7cde5c 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UnitMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UnitMappingTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -78,14 +78,14 @@ class UnitMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val executor = session.execution val instance0 = context.getMapping(MappingIdentifier("instance_0")) - instance0.inputs should be (Seq()) - instance0.outputs should be (Seq("input")) + instance0.inputs should be (Set()) + instance0.outputs should be (Set("input")) val df0 = executor.instantiate(instance0, "input") df0.collect() should be (inputDf0.collect()) val instance1 = context.getMapping(MappingIdentifier("instance_1")) - instance1.inputs should be (Seq()) - instance1.outputs should be (Seq("input")) + instance1.inputs should be (Set()) + instance1.outputs should be (Set("input")) val df1 = executor.instantiate(instance1, "input") df1.collect() should be (inputDf1.collect()) } @@ -118,8 +118,8 @@ class UnitMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val executor = session.execution val unit = context.getMapping(MappingIdentifier("macro")) - unit.inputs should be (Seq(MappingOutputIdentifier("outside"))) - unit.outputs.sorted should be (Seq("inside", "output")) + unit.inputs should be (Set(MappingOutputIdentifier("outside"))) + unit.outputs should be (Set("inside", "output")) val df_inside = executor.instantiate(unit, "inside") df_inside.collect() should be (inputDf0.collect()) @@ -151,8 +151,8 @@ class UnitMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession { val executor = session.execution val instance0 = context.getMapping(MappingIdentifier("alias")) - instance0.inputs should be (Seq(MappingOutputIdentifier("macro:input"))) - instance0.outputs should be (Seq("main")) + instance0.inputs should be (Set(MappingOutputIdentifier("macro:input"))) + instance0.outputs should be (Set("main")) val df0 = executor.instantiate(instance0, "main") df0.collect() should be (inputDf0.collect()) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UpsertMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UpsertMappingTest.scala index f108a521b..066acc3fa 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UpsertMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/UpsertMappingTest.scala @@ -201,7 +201,7 @@ class UpsertMappingTest extends AnyFlatSpec with Matchers with LocalSparkSession mapping shouldBe an[UpsertMappingSpec] val updateMapping = mapping.instantiate(session.context).asInstanceOf[UpsertMapping] - updateMapping.inputs should be (Seq(MappingOutputIdentifier("t0"),MappingOutputIdentifier("t1"))) + updateMapping.inputs should be (Set(MappingOutputIdentifier("t0"),MappingOutputIdentifier("t1"))) updateMapping.input should be (MappingOutputIdentifier("t0")) updateMapping.updates should be (MappingOutputIdentifier("t1")) updateMapping.keyColumns should be (Seq("id")) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ValuesMappingTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ValuesMappingTest.scala index 6b3582a87..ee78b1c4f 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ValuesMappingTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ValuesMappingTest.scala @@ -72,7 +72,7 @@ class ValuesMappingTest extends AnyFlatSpec with Matchers with MockFactory with mapping.kind should be ("values") mapping.identifier should be (MappingIdentifier("project/fake")) mapping.output should be (MappingOutputIdentifier("project/fake:main")) - mapping.outputs should be (Seq("main")) + mapping.outputs should be (Set("main")) mapping.records should be (Seq( ArrayRecord("a","12","3"), ArrayRecord("cat","","7"), @@ -108,8 +108,9 @@ class ValuesMappingTest extends AnyFlatSpec with Matchers with MockFactory with mapping.category should be (Category.MAPPING) mapping.kind should be ("values") mapping.identifier should be (MappingIdentifier("project/fake")) + mapping.inputs should be (Set()) mapping.output should be (MappingOutputIdentifier("project/fake:main")) - mapping.outputs should be (Seq("main")) + mapping.outputs should be (Set("main")) mapping.columns should be (Seq( Field("str_col", StringType), Field("int_col", IntegerType), @@ -158,8 +159,8 @@ class ValuesMappingTest extends AnyFlatSpec with Matchers with MockFactory with (mappingTemplate.instantiate _).expects(context).returns(mockMapping) val mapping = context.getMapping(MappingIdentifier("const")) - mapping.inputs should be (Seq()) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set()) + mapping.outputs should be (Set("main")) mapping.describe(executor, Map()) should be (Map("main" -> schema)) mapping.describe(executor, Map(), "main") should be (schema) @@ -203,8 +204,8 @@ class ValuesMappingTest extends AnyFlatSpec with Matchers with MockFactory with (mappingTemplate.instantiate _).expects(context).returns(mockMapping) val mapping = context.getMapping(MappingIdentifier("const")) - mapping.inputs should be (Seq()) - mapping.outputs should be (Seq("main")) + mapping.inputs should be (Set()) + mapping.outputs should be (Set("main")) mapping.describe(executor, Map()) should be (Map("main" -> schema)) mapping.describe(executor, Map(), "main") should be (schema) diff --git a/flowman-studio/src/main/scala/com/dimajix/flowman/studio/model/Converter.scala b/flowman-studio/src/main/scala/com/dimajix/flowman/studio/model/Converter.scala index f0fd83e9f..8f91ed516 100644 --- a/flowman-studio/src/main/scala/com/dimajix/flowman/studio/model/Converter.scala +++ b/flowman-studio/src/main/scala/com/dimajix/flowman/studio/model/Converter.scala @@ -84,8 +84,8 @@ object Converter { mapping.broadcast, mapping.cache.description, mapping.checkpoint, - mapping.inputs.map(_.toString), - mapping.outputs, + mapping.inputs.toSeq.map(_.toString), + mapping.outputs.toSeq, mapping.metadata.labels ) } From b1d28b8486d92210bf5db7946e9a69a27961932c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sun, 20 Feb 2022 20:46:39 +0100 Subject: [PATCH 62/95] Add new MappingOutput node to graph --- .../documentation/RelationCollector.scala | 11 +++-- .../documentation/TargetCollector.scala | 2 +- .../com/dimajix/flowman/graph/Category.scala | 2 + .../com/dimajix/flowman/graph/Graph.scala | 13 ++++- .../dimajix/flowman/graph/GraphBuilder.scala | 5 +- .../com/dimajix/flowman/graph/Linker.scala | 18 +++---- .../com/dimajix/flowman/graph/edges.scala | 6 ++- .../com/dimajix/flowman/graph/nodes.scala | 48 +++++++++++-------- .../com/dimajix/flowman/history/graph.scala | 8 ++-- .../documentation/MappingCollectorTest.scala | 4 +- .../documentation/RelationCollectorTest.scala | 2 + .../flowman/graph/GraphBuilderTest.scala | 10 ++-- .../com/dimajix/flowman/graph/GraphTest.scala | 16 +++++-- .../com/dimajix/flowman/graph/NodeTest.scala | 28 +++++------ .../dimajix/flowman/model/MappingTest.scala | 9 ++-- 15 files changed, 111 insertions(+), 71 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 54e0b0d3a..72ae39102 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -57,7 +57,7 @@ class RelationCollector extends Collector { case write:WriteRelation => write.input.incoming.flatMap { case map: InputMapping => - val mapref = MappingReference.of(parent, map.input.identifier) + val mapref = MappingReference.of(parent, map.mapping.identifier) val outref = MappingOutputReference(Some(mapref), map.pin) Some(outref) case _ => None @@ -81,8 +81,8 @@ class RelationCollector extends Collector { def collectMappingSources(map:MappingRef) : Seq[ResourceIdentifier] = { val direct = map.mapping.requires.toSeq val indirect = map.incoming.flatMap { - case in:InputMapping => - collectMappingSources(in.input) + case map:InputMapping => + collectMappingSources(map.mapping) case _ => Seq.empty } (direct ++ indirect).distinct @@ -92,7 +92,7 @@ class RelationCollector extends Collector { case write:WriteRelation => write.input.incoming.flatMap { case map: InputMapping => - collectMappingSources(map.input) + collectMappingSources(map.mapping) case rel: ReadRelation => rel.input.relation.provides.toSeq case _ => Seq.empty @@ -148,7 +148,8 @@ class RelationCollector extends Collector { write.input.incoming.flatMap { case map: InputMapping => Try { - execution.describe(map.input.mapping, map.pin) + val mapout = map.input + execution.describe(mapout.mapping.mapping, mapout.output) }.toOption case _ => None } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala index 524a0d7a6..9e666c68f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TargetCollector.scala @@ -48,7 +48,7 @@ class TargetCollector extends Collector { val inputs = node.incoming.flatMap { case map: InputMapping => - val mapref = MappingReference.of(parent, map.input.identifier) + val mapref = MappingReference.of(parent, map.mapping.identifier) val outref = MappingOutputReference(Some(mapref), map.pin) Some(outref) case read: ReadRelation => diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Category.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Category.scala index 4c358271c..668c170bf 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Category.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Category.scala @@ -26,6 +26,7 @@ sealed abstract class Category extends Product with Serializable { object Category { case object MAPPING extends Category + case object MAPPING_OUTPUT extends Category case object MAPPING_COLUMN extends Category case object RELATION extends Category case object RELATION_COLUMN extends Category @@ -34,6 +35,7 @@ object Category { def ofString(category:String) : Category = { category.toLowerCase(Locale.ROOT) match { case "mapping" => MAPPING + case "mapping_output" => MAPPING_OUTPUT case "mapping_column" => MAPPING_COLUMN case "relation" => RELATION case "relation_column" => RELATION_COLUMN diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala index f7462c606..20a4dfec7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Graph.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.graph +import scala.annotation.tailrec + import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.NoSuchMappingException import com.dimajix.flowman.execution.NoSuchRelationException @@ -81,7 +83,16 @@ final case class Graph( ) { def project : Option[Project] = context.project - def nodes : Seq[Node] = mappings ++ relations ++ targets + def nodes : Seq[Node] = { + def collectChildren(nodes:Seq[Node]) : Seq[Node] = { + val children = nodes.flatMap(_.children) + val next = if (children.nonEmpty) collectChildren(children) else Seq.empty + nodes ++ next + } + + val roots = mappings ++ relations ++ targets + collectChildren(roots) + } def edges : Seq[Edge] = nodes.flatMap(_.outgoing) /** diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/GraphBuilder.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/GraphBuilder.scala index 9a2186669..e96dba514 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/GraphBuilder.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/GraphBuilder.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -105,7 +105,8 @@ final class GraphBuilder(context:Context, phase:Phase) { } else { // Create new node and *first* put it into map of known mappings - val node = MappingRef(nextNodeId(), mapping) + val outputs = mapping.outputs.toSeq.map(o => MappingOutput(nextNodeId(), null, o)) + val node = MappingRef(nextNodeId(), mapping, outputs) mappings.put(mapping, node) // Now recursively run the linking process on the newly created node val linker = Linker(this, mapping.context, node) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala index bc6fa7d93..e8cfb14c0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/Linker.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,14 +31,14 @@ import com.dimajix.flowman.types.SingleValue final case class Linker private[graph](builder:GraphBuilder, context:Context, node:Node) { def input(mapping: Mapping, output:String) : Linker = { val in = builder.refMapping(mapping) - val edge = InputMapping(in, node, output) + val out = in.outputs.find(_.output == output) + .getOrElse(throw new IllegalArgumentException(s"Mapping '${mapping.identifier}' doesn't provide output '$output'")) + val edge = InputMapping(out, node) link(edge) } def input(mapping: MappingIdentifier, output:String) : Linker = { val instance = context.getMapping(mapping) - val in = builder.refMapping(instance) - val edge = InputMapping(in, node, output) - link(edge) + input(instance, output) } def read(relation: Reference[Relation], partitions:Map[String,FieldValue]) : Linker = { @@ -54,9 +54,7 @@ final case class Linker private[graph](builder:GraphBuilder, context:Context, no } def read(relation: RelationIdentifier, partitions:Map[String,FieldValue]) : Linker = { val instance = context.getRelation(relation) - val in = builder.refRelation(instance) - val edge = ReadRelation(in, node, partitions) - link(edge) + read(instance, partitions) } def write(relation: Reference[Relation], partitions:Map[String,SingleValue]) : Linker = { @@ -72,9 +70,7 @@ final case class Linker private[graph](builder:GraphBuilder, context:Context, no } def write(relation: RelationIdentifier, partition:Map[String,SingleValue]) : Linker = { val instance = context.getRelation(relation) - val out = builder.refRelation(instance) - val edge = WriteRelation(node, out, partition) - link(edge) + write(instance, partition) } /** diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/edges.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/edges.scala index 065192c28..a7b69434d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/edges.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/edges.scala @@ -34,9 +34,11 @@ final case class ReadRelation(override val input:RelationRef, override val outpu def resources : Set[ResourceIdentifier] = input.relation.resources(partitions) } -final case class InputMapping(override val input:MappingRef,override val output:Node,pin:String="main") extends Edge { +final case class InputMapping(override val input:MappingOutput,override val output:Node) extends Edge { override def action: Action = Action.INPUT - override def label: String = s"${action.upper} from ${input.label} output '$pin'" + override def label: String = s"${action.upper} from ${input.mapping.label} output '${input.output}'" + def mapping : MappingRef = input.mapping + def pin : String = input.output } final case class WriteRelation(override val input:Node, override val output:RelationRef, partition:Map[String,SingleValue] = Map()) extends Edge { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala b/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala index d58d347ec..405eb9787 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/graph/nodes.scala @@ -31,8 +31,6 @@ import com.dimajix.flowman.model.TargetIdentifier sealed abstract class Node extends Product with Serializable { private[graph] val inEdges = mutable.Buffer[Edge]() private[graph] val outEdges = mutable.Buffer[Edge]() - private[graph] val _parent : Option[Node] = None - private[graph] val _children = mutable.Seq[Node]() /** Unique node ID, generated by GraphBuilder */ val id : Int @@ -44,9 +42,6 @@ sealed abstract class Node extends Product with Serializable { def name : String def project : Option[String] - def provides : Set[ResourceIdentifier] - def requires : Set[ResourceIdentifier] - /** * List of incoming edges, i.e. the upstream nodes which provide input data * @return @@ -59,18 +54,23 @@ sealed abstract class Node extends Product with Serializable { */ def outgoing : Seq[Edge] = outEdges + /** + * Returns upstream resources + */ + def upstream : Seq[Edge] = incoming + /** * Child nodes providing more detail. For example a "Mapping" node might contain detail information on individual * columns, which would be logical children of the mapping. * @return */ - def children : Seq[Node] = _children + def children : Seq[Node] = Seq.empty /** * Optional parent node. For example a "Column" node might be a child of a "Mapping" node * @return */ - def parent : Option[Node] = _parent + def parent : Option[Node] = None /** * Create a nice string representation of the upstream dependency tree @@ -91,7 +91,9 @@ sealed abstract class Node extends Product with Serializable { Iterator() } } - val trees = incoming.map { child => + + // Do not use incoming edges, but upstream edges instead - this mainly makes sense for MappingOutputs + val trees = upstream.map { child => child.label + "\n" + child.input.upstreamTreeRec } val headChildren = trees.dropRight(1) @@ -103,13 +105,16 @@ sealed abstract class Node extends Product with Serializable { } } -final case class MappingRef(id:Int, mapping:Mapping) extends Node { +final case class MappingRef(id:Int, mapping:Mapping, outputs:Seq[MappingOutput]) extends Node { + require(outputs.forall(_.mapping == null)) + outputs.foreach(_.mapping = this) + override def category: Category = Category.MAPPING override def kind: String = mapping.kind override def name: String = mapping.name override def project: Option[String] = mapping.project.map(_.name) - override def provides : Set[ResourceIdentifier] = Set() - override def requires : Set[ResourceIdentifier] = mapping.requires + override def children: Seq[Node] = outputs + def requires : Set[ResourceIdentifier] = mapping.requires def identifier : MappingIdentifier = mapping.identifier } final case class TargetRef(id:Int, target:Target, phase:Phase) extends Node { @@ -117,8 +122,8 @@ final case class TargetRef(id:Int, target:Target, phase:Phase) extends Node { override def kind: String = target.kind override def name: String = target.name override def project: Option[String] = target.project.map(_.name) - override def provides : Set[ResourceIdentifier] = target.provides(phase) - override def requires : Set[ResourceIdentifier] = target.requires(phase) + def provides : Set[ResourceIdentifier] = target.provides(phase) + def requires : Set[ResourceIdentifier] = target.requires(phase) def identifier : TargetIdentifier = target.identifier } final case class RelationRef(id:Int, relation:Relation) extends Node { @@ -126,24 +131,29 @@ final case class RelationRef(id:Int, relation:Relation) extends Node { override def kind: String = relation.kind override def name: String = relation.name override def project: Option[String] = relation.project.map(_.name) - override def provides : Set[ResourceIdentifier] = relation.provides - override def requires : Set[ResourceIdentifier] = relation.requires + def provides : Set[ResourceIdentifier] = relation.provides + def requires : Set[ResourceIdentifier] = relation.requires def identifier : RelationIdentifier = relation.identifier } +final case class MappingOutput(id:Int, var mapping: MappingRef, output:String) extends Node { + override def toString: String = s"MappingOutput($id, ${mapping.id}, $output)" + override def category: Category = Category.MAPPING_OUTPUT + override def kind: String = "mapping_output" + override def parent: Option[Node] = Some(mapping) + override def name: String = mapping.name + "." + output + override def project: Option[String] = mapping.project + override def upstream : Seq[Edge] = mapping.incoming +} final case class MappingColumn(id:Int, mapping: Mapping, output:String, column:String) extends Node { override def category: Category = Category.MAPPING_COLUMN override def kind: String = "mapping_column" override def name: String = mapping.name + "." + output + "." + column override def project: Option[String] = mapping.project.map(_.name) - override def provides : Set[ResourceIdentifier] = Set() - override def requires : Set[ResourceIdentifier] = Set() } final case class RelationColumn(id:Int, relation: Relation, column:String) extends Node { override def category: Category = Category.RELATION_COLUMN override def kind: String = "relation_column" override def name: String = relation.name + "." + column override def project: Option[String] = relation.project.map(_.name) - override def provides : Set[ResourceIdentifier] = Set() - override def requires : Set[ResourceIdentifier] = Set() } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala index 1567594d4..3503b3d5e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/graph.scala @@ -156,10 +156,10 @@ object Graph { val partitionFields = MapIgnoreCase(relation.partitions.map(p => p.name -> p)) val p = read.partitions.map { case(k,v) => (k -> partitionFields(k).interpolate(v).map(_.toString).toSeq) } builder.addEdge(ReadRelation(in, out, p)) - case input:g.InputMapping => - val in = nodesById(input.input.id).asInstanceOf[MappingNode] - val out = nodesById(input.output.id) - builder.addEdge(InputMapping(in, out, input.pin)) + case map:g.InputMapping => + val in = nodesById(map.mapping.id).asInstanceOf[MappingNode] + val out = nodesById(map.output.id) + builder.addEdge(InputMapping(in, out, map.pin)) case write:g.WriteRelation => val in = nodesById(write.input.id) val out = nodesById(write.output.id).asInstanceOf[RelationNode] diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala index 77a704475..9ece3ea14 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala @@ -38,7 +38,7 @@ import com.dimajix.flowman.types.StructType class MappingCollectorTest extends AnyFlatSpec with Matchers with MockFactory { - "RelationCollector.collect" should "work" in { + "MappingCollector.collect" should "work" in { val mapping1 = mock[Mapping] val mappingTemplate1 = mock[Prototype[Mapping]] val mapping2 = mock[Mapping] @@ -62,10 +62,12 @@ class MappingCollectorTest extends AnyFlatSpec with Matchers with MockFactory { (mappingTemplate1.instantiate _).expects(context).returns(mapping1) (mapping1.context _).expects().returns(context) + (mapping1.outputs _).expects().returns(Set("main")) (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) (mappingTemplate2.instantiate _).expects(context).returns(mapping2) (mapping2.context _).expects().returns(context) + (mapping2.outputs _).expects().returns(Set("main")) (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map("pcol"-> SingleValue("part1"))))) (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala index e5ce455f0..fb7cd4c14 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/RelationCollectorTest.scala @@ -68,10 +68,12 @@ class RelationCollectorTest extends AnyFlatSpec with Matchers with MockFactory { (mappingTemplate1.instantiate _).expects(context).returns(mapping1) (mapping1.context _).expects().returns(context) + (mapping1.outputs _).expects().returns(Set("main")) (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) (mappingTemplate2.instantiate _).expects(context).returns(mapping2) (mapping2.context _).expects().returns(context) + (mapping2.outputs _).expects().returns(Set("main")) (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map("pcol"-> SingleValue("part1"))))) (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphBuilderTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphBuilderTest.scala index 0f4da01b6..d2223ebb1 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphBuilderTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphBuilderTest.scala @@ -49,11 +49,13 @@ class GraphBuilderTest extends AnyFlatSpec with Matchers with MockFactory { (mappingTemplate1.instantiate _).expects(context).returns(mapping1) (mapping1.context _).expects().returns(context) + (mapping1.outputs _).expects().returns(Set("main")) (mapping1.kind _).expects().returns("m1_kind") (mapping1.name _).expects().atLeastOnce().returns("m1") (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) (mappingTemplate2.instantiate _).expects(context).returns(mapping2) (mapping2.context _).expects().returns(context) + (mapping2.outputs _).expects().returns(Set("main")) (mapping2.kind _).expects().returns("m2_kind") (mapping2.name _).expects().atLeastOnce().returns("m2") (mapping2.link _).expects(*).returns(Unit) @@ -66,13 +68,14 @@ class GraphBuilderTest extends AnyFlatSpec with Matchers with MockFactory { val ref1 = nodes.find(_.name == "m1").head.asInstanceOf[MappingRef] val ref2 = nodes.find(_.name == "m2").head.asInstanceOf[MappingRef] + val out2main = ref2.outputs.head ref1.category should be (Category.MAPPING) ref1.kind should be ("m1_kind") ref1.name should be ("m1") ref1.mapping should be (mapping1) ref1.incoming should be (Seq( - InputMapping(ref2, ref1, "main") + InputMapping(out2main, ref1) )) ref1.outgoing should be (Seq()) @@ -81,8 +84,9 @@ class GraphBuilderTest extends AnyFlatSpec with Matchers with MockFactory { ref2.name should be ("m2") ref2.mapping should be (mapping2) ref2.incoming should be (Seq()) - ref2.outgoing should be (Seq( - InputMapping(ref2, ref1, "main") + ref2.outgoing should be (Seq()) + ref2.outputs.head.outgoing should be (Seq( + InputMapping(out2main, ref1) )) } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala index 895d5ff70..f97013888 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/graph/GraphTest.scala @@ -66,11 +66,13 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { (mappingTemplate1.instantiate _).expects(context).returns(mapping1) (mapping1.context _).expects().returns(context) + (mapping1.outputs _).expects().returns(Set("main")) (mapping1.name _).expects().atLeastOnce().returns("m1") (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) (mappingTemplate2.instantiate _).expects(context).returns(mapping2) (mapping2.context _).expects().returns(context) + (mapping2.outputs _).expects().returns(Set("main")) (mapping2.name _).expects().atLeastOnce().returns("m2") (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map.empty[String,FieldValue]))) @@ -95,7 +97,7 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { val graph = Graph.ofProject(session, project, Phase.BUILD) val nodes = graph.nodes - nodes.size should be (5) + nodes.size should be (7) nodes.find(_.name == "m1") should not be (None) nodes.find(_.name == "m1").get shouldBe a[MappingRef] nodes.find(_.name == "m2") should not be (None) @@ -118,6 +120,8 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { maps.find(_.name == "m3") should be (None) val m1 = maps.find(_.name == "m1").get val m2 = maps.find(_.name == "m2").get + val out1main = m1.outputs.head + val out2main = m2.outputs.head val tgts = graph.targets tgts.size should be (1) @@ -128,13 +132,15 @@ class GraphTest extends AnyFlatSpec with Matchers with MockFactory { val src = rels.find(_.name == "src").get val tgt = rels.find(_.name == "tgt").get - m1.incoming should be (Seq(InputMapping(m2, m1, "main"))) - m1.outgoing should be (Seq(InputMapping(m1, t, "main"))) + m1.incoming should be (Seq(InputMapping(out2main, m1))) + m1.outgoing should be (Seq()) + m1.outputs.head.outgoing should be (Seq(InputMapping(out1main, t))) m2.incoming should be (Seq(ReadRelation(src, m2, Map()))) - m2.outgoing should be (Seq(InputMapping(m2, m1, "main"))) + m2.outgoing should be (Seq()) + m2.outputs.head.outgoing should be (Seq(InputMapping(out2main, m1))) src.incoming should be (Seq()) src.outgoing should be (Seq(ReadRelation(src, m2, Map()))) - t.incoming should be (Seq(InputMapping(m1, t, "main"))) + t.incoming should be (Seq(InputMapping(out1main, t))) t.outgoing should be (Seq(WriteRelation(t, tgt, Map()))) tgt.incoming should be (Seq(WriteRelation(t, tgt, Map()))) tgt.outgoing should be (Seq()) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/graph/NodeTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/graph/NodeTest.scala index 0b43cfa03..8f54f1589 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/graph/NodeTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/graph/NodeTest.scala @@ -54,22 +54,22 @@ class NodeTest extends AnyFlatSpec with Matchers with MockFactory { (tgtRelation.name _).expects().atLeastOnce().returns("facts") val srcRelationNode = RelationRef(1, srcRelation) - val readMappingNode = MappingRef(2, readMapping) - val mapping1Node = MappingRef(3, mapping1) - val mapping2Node = MappingRef(4, mapping2) - val mapping3Node = MappingRef(5, mapping3) - val unionMappingNode = MappingRef(6, unionMapping) - val targetNode = TargetRef(7, target, Phase.BUILD) - val tgtRelationNode = RelationRef(8, tgtRelation) + val readMappingNode = MappingRef(2, readMapping, Seq(MappingOutput(3, null, "main"))) + val mapping1Node = MappingRef(4, mapping1, Seq(MappingOutput(5, null, "main"))) + val mapping2Node = MappingRef(6, mapping2, Seq(MappingOutput(7, null, "main"))) + val mapping3Node = MappingRef(8, mapping3, Seq(MappingOutput(9, null, "main"))) + val unionMappingNode = MappingRef(10, unionMapping, Seq(MappingOutput(11, null, "main"))) + val targetNode = TargetRef(12, target, Phase.BUILD) + val tgtRelationNode = RelationRef(13, tgtRelation) tgtRelationNode.inEdges.append(WriteRelation(targetNode, tgtRelationNode)) - targetNode.inEdges.append(InputMapping(unionMappingNode, targetNode)) - unionMappingNode.inEdges.append(InputMapping(mapping1Node, unionMappingNode)) - unionMappingNode.inEdges.append(InputMapping(mapping2Node, unionMappingNode)) - unionMappingNode.inEdges.append(InputMapping(mapping3Node, unionMappingNode)) - mapping1Node.inEdges.append(InputMapping(readMappingNode, mapping1Node)) - mapping2Node.inEdges.append(InputMapping(readMappingNode, mapping2Node)) - mapping3Node.inEdges.append(InputMapping(readMappingNode, mapping3Node)) + targetNode.inEdges.append(InputMapping(unionMappingNode.outputs.head, targetNode)) + unionMappingNode.inEdges.append(InputMapping(mapping1Node.outputs.head, unionMappingNode)) + unionMappingNode.inEdges.append(InputMapping(mapping2Node.outputs.head, unionMappingNode)) + unionMappingNode.inEdges.append(InputMapping(mapping3Node.outputs.head, unionMappingNode)) + mapping1Node.inEdges.append(InputMapping(readMappingNode.outputs.head, mapping1Node)) + mapping2Node.inEdges.append(InputMapping(readMappingNode.outputs.head, mapping2Node)) + mapping3Node.inEdges.append(InputMapping(readMappingNode.outputs.head, mapping3Node)) readMappingNode.inEdges.append(ReadRelation(srcRelationNode, readMappingNode)) println(tgtRelationNode.upstreamDependencyTree) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala index c0a29f722..91843bcd8 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/model/MappingTest.scala @@ -152,17 +152,20 @@ class MappingTest extends AnyFlatSpec with Matchers with MockFactory with LocalS val graphBuilder = new GraphBuilder(context, Phase.BUILD) val ref1 = graphBuilder.refMapping(mapping1) val ref2 = graphBuilder.refMapping(mapping2) + val out11 = ref1.outputs.head + val out21 = ref2.outputs.head ref1.mapping should be (mapping1) ref1.incoming should be (Seq( - InputMapping(ref2, ref1, "main") + InputMapping(out21, ref1) )) ref1.outgoing should be (Seq()) ref2.mapping should be (mapping2) ref2.incoming should be (Seq()) - ref2.outgoing should be (Seq( - InputMapping(ref2, ref1, "main") + ref2.outgoing should be (Seq()) + ref2.outputs.head.outgoing should be (Seq( + InputMapping(out21, ref1) )) } } From 51c46658f3239a4f51233859aa98a64bfd4396a0 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 21 Feb 2022 08:36:15 +0100 Subject: [PATCH 63/95] Add unittest for history graph --- .../dimajix/flowman/history/GraphTest.scala | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/history/GraphTest.scala diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/history/GraphTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/history/GraphTest.scala new file mode 100644 index 000000000..ea2dc8ca2 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/history/GraphTest.scala @@ -0,0 +1,167 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.history + +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.graph.Category +import com.dimajix.flowman.graph.Linker +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.PartitionField +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.model.Relation +import com.dimajix.flowman.model.RelationIdentifier +import com.dimajix.flowman.model.Target +import com.dimajix.flowman.types.SingleValue +import com.dimajix.flowman.types.StringType +import com.dimajix.flowman.types.StructType +import com.dimajix.flowman.{graph => g} + + +class GraphTest extends AnyFlatSpec with Matchers with MockFactory { + "Graph.ofGraph" should "work" in { + val mapping1 = mock[Mapping] + val mappingTemplate1 = mock[Prototype[Mapping]] + val mapping2 = mock[Mapping] + val mappingTemplate2 = mock[Prototype[Mapping]] + val sourceRelation = mock[Relation] + val sourceRelationTemplate = mock[Prototype[Relation]] + val targetRelation = mock[Relation] + val targetRelationTemplate = mock[Prototype[Relation]] + val target = mock[Target] + val targetTemplate = mock[Prototype[Target]] + + val project = Project( + name = "project", + mappings = Map( + "m1" -> mappingTemplate1, + "m2" -> mappingTemplate2 + ), + targets = Map( + "t" -> targetTemplate + ), + relations = Map( + "src" -> sourceRelationTemplate, + "tgt" -> targetRelationTemplate + ) + ) + val session = Session.builder().disableSpark().build() + val context = session.getContext(project) + val execution = session.execution + + (mappingTemplate1.instantiate _).expects(context).returns(mapping1) + (mapping1.context _).expects().returns(context) + (mapping1.outputs _).expects().returns(Set("main")) + (mapping1.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.input(MappingIdentifier("m2"), "main"))) + + (mappingTemplate2.instantiate _).expects(context).returns(mapping2) + (mapping2.context _).expects().returns(context) + (mapping2.outputs _).expects().returns(Set("main")) + (mapping2.link _).expects(*).onCall((l:Linker) => Some(1).foreach(_ => l.read(RelationIdentifier("src"), Map("pcol"-> SingleValue("part1"))))) + + (sourceRelationTemplate.instantiate _).expects(context).returns(sourceRelation) + (sourceRelation.context _).expects().returns(context) + (sourceRelation.link _).expects(*).returns(Unit) + + (targetRelationTemplate.instantiate _).expects(context).returns(targetRelation) + (targetRelation.context _).expects().returns(context) + (targetRelation.link _).expects(*).returns(Unit) + + (targetTemplate.instantiate _).expects(context).returns(target) + (target.context _).expects().returns(context) + (target.link _).expects(*,*).onCall((l:Linker, _:Phase) => Some(1).foreach { _ => + l.input(MappingIdentifier("m1"), "main") + l.write(RelationIdentifier("tgt"), Map("outcol"-> SingleValue("part1"))) + }) + + val graph = g.Graph.ofProject(session, project, Phase.BUILD) + + (mapping1.name _).expects().atLeastOnce().returns("m1") + (mapping1.kind _).expects().atLeastOnce().returns("m1_kind") + (mapping1.requires _).expects().returns(Set()) + (mapping2.name _).expects().atLeastOnce().returns("m2") + (mapping2.kind _).expects().atLeastOnce().returns("m2_kind") + (mapping2.requires _).expects().returns(Set()) + + (sourceRelation.name _).expects().atLeastOnce().returns("src") + (sourceRelation.kind _).expects().atLeastOnce().returns("src_kind") + (sourceRelation.provides _).expects().returns(Set()) + (sourceRelation.requires _).expects().returns(Set()) + (sourceRelation.partitions _).expects().returns(Seq(PartitionField("pcol", StringType))) + + (targetRelation.name _).expects().atLeastOnce().returns("tgt") + (targetRelation.kind _).expects().atLeastOnce().returns("tgt_kind") + (targetRelation.provides _).expects().returns(Set()) + (targetRelation.requires _).expects().returns(Set()) + + (target.provides _).expects(*).returns(Set.empty) + (target.requires _).expects(*).returns(Set.empty) + (target.name _).expects().returns("tgt1") + (target.kind _).expects().returns("tgt1_kind") + + val hgraph = Graph.ofGraph(graph) + val srcRelNode = hgraph.nodes.find(_.name == "src").get + val tgtRelNode = hgraph.nodes.find(_.name == "tgt").get + val m1Node = hgraph.nodes.find(_.name == "m1").get + val m2Node = hgraph.nodes.find(_.name == "m2").get + val tgtNode = hgraph.nodes.find(_.name == "tgt1").get + + srcRelNode.name should be ("src") + srcRelNode.category should be (Category.RELATION) + srcRelNode.kind should be ("src_kind") + srcRelNode.incoming should be (Seq.empty) + srcRelNode.outgoing.head.input should be (srcRelNode) + srcRelNode.outgoing.head.output should be (m2Node) + + m2Node.name should be ("m2") + m2Node.category should be (Category.MAPPING) + m2Node.kind should be ("m2_kind") + m2Node.incoming.head.input should be (srcRelNode) + m2Node.incoming.head.output should be (m2Node) + m2Node.outgoing.head.input should be (m2Node) + m2Node.outgoing.head.output should be (m1Node) + + m1Node.name should be ("m1") + m1Node.category should be (Category.MAPPING) + m1Node.kind should be ("m1_kind") + m1Node.incoming.head.input should be (m2Node) + m1Node.incoming.head.output should be (m1Node) + m1Node.outgoing.head.input should be (m1Node) + m1Node.outgoing.head.output should be (tgtNode) + + tgtNode.name should be ("tgt1") + tgtNode.category should be (Category.TARGET) + tgtNode.kind should be ("tgt1_kind") + tgtNode.incoming.head.input should be (m1Node) + tgtNode.incoming.head.output should be (tgtNode) + tgtNode.outgoing.head.input should be (tgtNode) + tgtNode.outgoing.head.output should be (tgtRelNode) + + tgtRelNode.name should be ("tgt") + tgtRelNode.category should be (Category.RELATION) + tgtRelNode.kind should be ("tgt_kind") + tgtRelNode.outgoing should be (Seq.empty) + tgtRelNode.incoming.head.input should be (tgtNode) + tgtRelNode.incoming.head.output should be (tgtRelNode) + } +} From e71c1573a04573ac4ff3075b99fcaac6af36edd0 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 21 Feb 2022 15:05:01 +0100 Subject: [PATCH 64/95] Allow regular expressions in metric selectors --- docs/cookbook/metrics.md | 7 ++++ docs/spec/hooks/report.md | 32 ++++++++++++++- docs/spec/target/measure.md | 10 +++++ examples/weather/.gitignore | 1 + examples/weather/job/main.yml | 3 ++ examples/weather/target/metrics.yml | 24 +++++++++++ .../flowman/execution/AbstractExecution.scala | 9 ----- .../dimajix/flowman/metric/MetricBoard.scala | 25 +++++++++--- .../dimajix/flowman/metric/MetricSystem.scala | 22 +++++----- .../flowman/metric/MultiMetricBundle.scala | 8 ++-- .../com/dimajix/flowman/metric/package.scala | 6 +-- .../com/dimajix/flowman/model/Target.scala | 2 +- .../flowman/metric/MetricBoardTest.scala | 6 +-- .../flowman/metric/MetricSystemTest.scala | 27 ++++++++----- .../conf/default-namespace.yml.template | 40 +++++++++++++++++++ .../flowman/spec/hook/ReportHook.scala | 6 +-- .../flowman/spec/metric/MetricSpec.scala | 8 ++-- .../flowman/spec/target/MeasureTarget.scala | 4 +- .../spec/target/MeasureTargetTest.scala | 14 +++---- .../flowman/spec/target/MergeTargetTest.scala | 4 +- .../spec/target/RelationTargetTest.scala | 4 +- 21 files changed, 194 insertions(+), 68 deletions(-) create mode 100644 examples/weather/target/metrics.yml diff --git a/docs/cookbook/metrics.md b/docs/cookbook/metrics.md index 2246ea5bc..764c81bc1 100644 --- a/docs/cookbook/metrics.md +++ b/docs/cookbook/metrics.md @@ -31,6 +31,13 @@ jobs: phase: ${phase} datetime: ${processing_datetime} metrics: + # Collect everything + - selector: + name: .* + labels: + category: ${category} + kind: ${kind} + name: ${name} # This metric contains the number of records per output. It will search all metrics called # `target_records` and export them as `flowman_output_records`. It will also label each metric with # the name of each Flowman build target (in case you have multiple targets) diff --git a/docs/spec/hooks/report.md b/docs/spec/hooks/report.md index c9c26db71..bd3dd2215 100644 --- a/docs/spec/hooks/report.md +++ b/docs/spec/hooks/report.md @@ -1,10 +1,40 @@ # Report Hook +The `report` will create a textual report file containing information on the execution. As with all hooks, it can be +either added on the namespace level or on the job leve. + ## Example ```yaml job: main: hooks: - kind: report - location: file:///tmp/my-report.txt + location: ${project.basedir}/generated-report.txt + metrics: + # Define common labels for all metrics + labels: + project: ${project.name} + metrics: + # This metric contains the number of records per output + - name: output_records + selector: + name: target_records + labels: + category: target + labels: + target: ${name} + # This metric contains the processing time per output + - name: output_time + selector: + name: target_runtime + labels: + category: target + labels: + target: ${name} + # This metric contains the overall processing time + - name: processing_time + selector: + name: job_runtime + labels: + category: job ``` diff --git a/docs/spec/target/measure.md b/docs/spec/target/measure.md index 67793ad5b..3ad049359 100644 --- a/docs/spec/target/measure.md +++ b/docs/spec/target/measure.md @@ -23,6 +23,16 @@ This example will provide two metrics, `record_count` and `column_sum`, which th [metric sink](../metric) configured in the [namespace](../namespace.md). +## Provided Metrics +All metrics defined as named columns are exported with the following labels: + - `name` - The name of the measure (i.e. `record_stats` above) + - `category` - Always set to `measure` + - `kind` - Always set to `sql` + - `namespace` - Name of the namespace (typically `default`) + - `project` - Name of the project + - `version` - Version of the project + + ## Supported Execution Phases * `VERIFY` - The evaluation of all measures will only be performed in the `VERIFY` phase diff --git a/examples/weather/.gitignore b/examples/weather/.gitignore index 0cdba66e7..86cf85d0b 100644 --- a/examples/weather/.gitignore +++ b/examples/weather/.gitignore @@ -1 +1,2 @@ generated-documentation +generated-report.txt diff --git a/examples/weather/job/main.yml b/examples/weather/job/main.yml index b3e8a034d..8a050f797 100644 --- a/examples/weather/job/main.yml +++ b/examples/weather/job/main.yml @@ -13,4 +13,7 @@ jobs: - stations - aggregates - validate_stations_raw + # Collect some measures which will be published as metrics + - metrics + # Generate documentation - documentation diff --git a/examples/weather/target/metrics.yml b/examples/weather/target/metrics.yml new file mode 100644 index 000000000..48350b6f5 --- /dev/null +++ b/examples/weather/target/metrics.yml @@ -0,0 +1,24 @@ +targets: + metrics: + kind: measure + description: "Collect relevant metrics from measurements, to be published to a metrics collector" + measures: + measurement_metrics: + kind: sql + # The following SQL will provide the following metrics: + # - valid_wind_direction + # - invalid_wind_direction + # - valid_wind_speed + # - invalid_wind_speed + # - valid_air_temperature + # - invalid_air_temperature + query: " + SELECT + SUM(IF(wind_direction_qual = '1', 1, 0)) AS valid_wind_direction, + SUM(IF(wind_direction_qual <> '1', 1, 0)) AS invalid_wind_direction, + SUM(IF(wind_speed_qual = '1', 1, 0)) AS valid_wind_speed, + SUM(IF(wind_speed_qual <> '1', 1, 0)) AS invalid_wind_speed, + SUM(IF(air_temperature_qual = '1', 1, 0)) AS valid_air_temperature, + SUM(IF(air_temperature_qual <> '1', 1, 0)) AS invalid_air_temperature + FROM measurements + " diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractExecution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractExecution.scala index 1649940bb..aed2fc310 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractExecution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/AbstractExecution.scala @@ -20,15 +20,9 @@ import java.time.Instant import scala.util.control.NonFatal -import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.SparkSession import org.slf4j.LoggerFactory -import com.dimajix.flowman.catalog.HiveCatalog -import com.dimajix.flowman.config.FlowmanConf -import com.dimajix.flowman.hadoop.FileSystem import com.dimajix.flowman.metric.MetricBoard -import com.dimajix.flowman.metric.MetricSystem import com.dimajix.flowman.metric.withWallTime import com.dimajix.flowman.model.Assertion import com.dimajix.flowman.model.AssertionResult @@ -37,13 +31,10 @@ import com.dimajix.flowman.model.JobDigest import com.dimajix.flowman.model.JobLifecycle import com.dimajix.flowman.model.JobResult import com.dimajix.flowman.model.LifecycleResult -import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Measure import com.dimajix.flowman.model.MeasureResult -import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetResult -import com.dimajix.flowman.types.StructType import com.dimajix.flowman.util.withShutdownHook diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala index 4670a523f..aad40ccd3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package com.dimajix.flowman.metric +import scala.util.matching.Regex + import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Status @@ -54,7 +56,8 @@ final case class MetricBoard( selections.flatMap { sel => // Relabeling should happen has late as possible, since some values might be dynamic def relabel(metric:Metric) : Metric = metric match { - case gauge:GaugeMetric => FixedGaugeMetric(sel.name, env.evaluate(labels ++ sel.labels, gauge.labels + ("status" -> status)), gauge.value) + // Remove "project" from gauge.labels + case gauge:GaugeMetric => FixedGaugeMetric(sel.name.getOrElse(gauge.name), env.evaluate(labels ++ sel.labels, gauge.labels - "project" + ("status" -> status)), gauge.value) case _ => throw new IllegalArgumentException(s"Metric of type ${metric.getClass} not supported") } @@ -67,7 +70,7 @@ final case class MetricBoard( /** * A MetricSelection represents a possibly dynamic set of Metrics to be published inside a MetricBoard */ -final case class MetricSelection(name:String, selector:Selector, labels:Map[String,String]) { +final case class MetricSelection(name:Option[String], selector:Selector, labels:Map[String,String]) { /** * Returns all metrics identified by this selection. This operation may be expensive, since the set of metrics may be * dynamic and change over time @@ -83,8 +86,18 @@ final case class MetricSelection(name:String, selector:Selector, labels:Map[Stri def bundles(implicit catalog:MetricCatalog) : Seq[MetricBundle] = catalog.findBundle(selector) } - +object Selector { + def apply(labels:Map[String,String]) : Selector = { + new Selector(None, labels.map { case(k,v) => k -> v.r } ) + } + def apply(name:String) : Selector = { + new Selector(Some(name.r), Map.empty ) + } + def apply(name:String, labels:Map[String,String]) : Selector = { + new Selector(Some(name.r), labels.map { case(k,v) => k -> v.r } ) + } +} final case class Selector( - name:Option[String] = None, - labels:Map[String,String] = Map() + name:Option[Regex] = None, + labels:Map[String,Regex] = Map() ) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala index 0a150c7f1..55847fe68 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package com.dimajix.flowman.metric +import scala.util.matching.Regex + import com.dimajix.common.IdentityHashSet import com.dimajix.common.SynchronizedSet import com.dimajix.flowman.execution.Status @@ -81,12 +83,12 @@ class MetricSystem extends MetricCatalog { metricBundles.remove(bundle) } - def getOrCreateBundle[T <: MetricBundle](query:Selector)(creator: => T) : T = { - metricBundles.find(bundle => query.name.forall(_ == bundle.name) && bundle.labels == query.labels) + def getOrCreateBundle[T <: MetricBundle](name:String, labels:Map[String,String])(creator: => T) : T = { + metricBundles.find(bundle => name == bundle.name && bundle.labels == labels) .map(_.asInstanceOf[T]) .getOrElse{ val bundle = creator - if (!query.name.forall(_ == bundle.name) || query.labels != bundle.labels) + if (name != bundle.name || labels != bundle.labels) throw new IllegalArgumentException("Newly created bundle needs to match query") addBundle(bundle) bundle @@ -181,13 +183,13 @@ class MetricSystem extends MetricCatalog { // Matches bundle labels to query. Only existing labels need to match def matchBundle(bundle:MetricBundle) : Boolean = { val labels = bundle.labels - selector.name.forall(_ == bundle.name) && - labels.keySet.intersect(selector.labels.keySet).forall(key => selector.labels(key) == labels(key)) + selector.name.forall(_.unapplySeq(bundle.name).nonEmpty) && + labels.keySet.intersect(selector.labels.keySet).forall(key => selector.labels(key).unapplySeq(labels(key)).nonEmpty) } // Matches metric labels to query. All labels need to match - def matchMetric(metric:Metric, query:Map[String,String]) : Boolean = { + def matchMetric(metric:Metric, query:Map[String,Regex]) : Boolean = { val labels = metric.labels - query.forall(kv => labels.get(kv._1).contains(kv._2)) + query.forall(kv => labels.get(kv._1).exists(v => kv._2.unapplySeq(v).nonEmpty)) } // Query a bundle and return all matching metrics within that bundle def queryBundle(bundle:MetricBundle) : Seq[Metric] = { @@ -214,8 +216,8 @@ class MetricSystem extends MetricCatalog { def matchBundle(bundle:MetricBundle) : Boolean = { val labels = bundle.labels - selector.name.forall(_ == bundle.name) && - selector.labels.forall(kv => labels.get(kv._1).contains(kv._2)) + selector.name.forall(_.unapplySeq(bundle.name).nonEmpty) && + selector.labels.forall(kv => labels.get(kv._1).exists(v => kv._2.unapplySeq(v).nonEmpty)) } metricBundles.toSeq diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MultiMetricBundle.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MultiMetricBundle.scala index 47069ad55..1a286d9ee 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MultiMetricBundle.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MultiMetricBundle.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,12 +31,12 @@ final case class MultiMetricBundle(override val name:String, override val labels bundleMetrics.remove(metric) } - def getOrCreateMetric[T <: Metric](query:Selector)(creator: => T) : T = { - bundleMetrics.find(metric => query.name.forall(_ == metric.name) && metric.labels == query.labels) + def getOrCreateMetric[T <: Metric](name:String, labels:Map[String,String])(creator: => T) : T = { + bundleMetrics.find(metric => name == metric.name && metric.labels == labels) .map(_.asInstanceOf[T]) .getOrElse{ val metric = creator - if (!query.name.forall(_ == metric.name) || query.labels != metric.labels) + if (name != metric.name || labels != metric.labels) throw new IllegalArgumentException("Newly created metric needs to match query") addMetric(metric) metric diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/package.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/package.scala index e5a022f42..f0b300aab 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/package.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/package.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019-2021 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,11 +63,11 @@ package object metric { // Create and register bundle val metricName = metadata.category + "_runtime" val bundleLabels = metadata.asMap + ("phase" -> phase.toString) - val bundle = registry.getOrCreateBundle(Selector(Some(metricName), bundleLabels))(MultiMetricBundle(metricName, bundleLabels)) + val bundle = registry.getOrCreateBundle(metricName, bundleLabels)(MultiMetricBundle(metricName, bundleLabels)) // Create and register metric val metricLabels = bundleLabels ++ Map("name" -> metadata.name) ++ metadata.labels - val metric = bundle.getOrCreateMetric(Selector(Some(metricName), metricLabels))(WallTimeMetric(metricName, metricLabels)) + val metric = bundle.getOrCreateMetric(metricName, metricLabels)(WallTimeMetric(metricName, metricLabels)) metric.reset() // Execute function itself, and catch any exception diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala index 89de0bf4f..0afe1c299 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala @@ -390,7 +390,7 @@ abstract class BaseTarget extends AbstractInstance with Target { protected def countRecords(execution:Execution, df:DataFrame, phase:Phase=Phase.BUILD) : DataFrame = { val labels = metadata.asMap + ("phase" -> phase.upper) - val counter = execution.metricSystem.findMetric(Selector(Some("target_records"), labels)) + val counter = execution.metricSystem.findMetric(Selector("target_records", labels)) .headOption .map(_.asInstanceOf[LongAccumulatorMetric].counter) .getOrElse { diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricBoardTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricBoardTest.scala index ef13b061d..60dc6eb94 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricBoardTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricBoardTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,8 +39,8 @@ class MetricBoardTest extends AnyFlatSpec with Matchers { registry.addBundle(CounterAccumulatorMetricBundle("some_metric", Map("raw_label" -> "raw_value"), accumulator1, "sublabel")) val selections = Seq( MetricSelection( - "m1", - Selector(Some("some_metric"), + Some("m1"), + Selector("some_metric", Map("raw_label" -> "raw_value", "sublabel" -> "a") ), Map("rl" -> "$raw_label", "sl" -> "$sublabel", "ev" -> "$env_var") diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricSystemTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricSystemTest.scala index 110d1015f..8c3c589df 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricSystemTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/metric/MetricSystemTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,16 +81,16 @@ class MetricSystemTest extends AnyFlatSpec with Matchers { r7.size should be (1) r7.forall(m => m.labels("label") == "acc1" && m.labels("sublabel") == "a") should be (true) - val r8 = registry.findMetric(Selector(name=Some("no_such_metric"))) + val r8 = registry.findMetric(Selector(name="no_such_metric")) r8.size should be (0) - val r9 = registry.findMetric(Selector(name=Some("some_metric_1"))) + val r9 = registry.findMetric(Selector(name="some_metric_1")) r9.size should be (2) - val r10 = registry.findMetric(Selector(name=Some("some_metric_1"), labels=Map("label" -> "acc1"))) + val r10 = registry.findMetric(Selector(name="some_metric_1", labels=Map("label" -> "acc1"))) r10.size should be (2) - val r11 = registry.findMetric(Selector(name=Some("some_metric_1"), labels=Map("label" -> "acc2"))) + val r11 = registry.findMetric(Selector(name="some_metric_1", labels=Map("label" -> "acc2"))) r11.size should be (0) } @@ -98,17 +98,22 @@ class MetricSystemTest extends AnyFlatSpec with Matchers { val registry = new MetricSystem val accumulator1 = new CounterAccumulator() - registry.addBundle(new CounterAccumulatorMetricBundle("some_metric_1", Map("label" -> "acc1"), accumulator1, "sublabel")) + registry.addBundle(CounterAccumulatorMetricBundle("some_metric_1", Map("label" -> "acc1"), accumulator1, "sublabel")) val accumulator2 = new CounterAccumulator() - registry.addBundle(new CounterAccumulatorMetricBundle("some_metric_2", Map("label" -> "acc2"), accumulator2, "sublabel")) + registry.addBundle(CounterAccumulatorMetricBundle("some_metric_2", Map("label" -> "acc2"), accumulator2, "sublabel")) registry.findBundle(Selector()).size should be (2) registry.findBundle(Selector(labels=Map("label" -> "acc2"))).size should be (1) registry.findBundle(Selector(labels=Map("label" -> "acc3"))).size should be (0) - registry.findBundle(Selector(name=Some("no_such_metric"))).size should be (0) - registry.findBundle(Selector(name=Some("some_metric_1"))).size should be (1) - registry.findBundle(Selector(name=Some("some_metric_1"), labels=Map("label" -> "acc1"))).size should be (1) - registry.findBundle(Selector(name=Some("some_metric_1"), labels=Map("label" -> "acc2"))).size should be (0) + registry.findBundle(Selector(name="no_such_metric")).size should be (0) + registry.findBundle(Selector(name="some_metric_1")).size should be (1) + registry.findBundle(Selector(name="some_metric_1", labels=Map("label" -> "acc1"))).size should be (1) + registry.findBundle(Selector(name="some_metric_1", labels=Map("label" -> "acc2"))).size should be (0) + + registry.findBundle(Selector(labels=Map("label" -> ".*2"))).size should be (1) + registry.findBundle(Selector(labels=Map("label" -> ".*3"))).size should be (0) + registry.findBundle(Selector(name="no_such_.*")).size should be (0) + registry.findBundle(Selector(name="some_metric_.*")).size should be (2) } } diff --git a/flowman-dist/conf/default-namespace.yml.template b/flowman-dist/conf/default-namespace.yml.template index 0b67358c8..2d6cfc82f 100644 --- a/flowman-dist/conf/default-namespace.yml.template +++ b/flowman-dist/conf/default-namespace.yml.template @@ -18,6 +18,46 @@ connections: password: $System.getenv('FLOWMAN_HISTORY_PASSWORD', '') +# This adds a hook for creating an execution log in a file +hooks: + kind: report + location: ${project.basedir}/generated-report.txt + metrics: + # Define common labels for all metrics + labels: + project: ${project.name} + metrics: + # Collect everything + - selector: + name: .* + labels: + category: ${category} + kind: ${kind} + name: ${name} + # This metric contains the number of records per output + - name: output_records + selector: + name: target_records + labels: + category: target + labels: + target: ${name} + # This metric contains the processing time per output + - name: output_time + selector: + name: target_runtime + labels: + category: target + labels: + target: ${name} + # This metric contains the overall processing time + - name: processing_time + selector: + name: job_runtime + labels: + category: job + + # This configures where metrics should be written to. Since we cannot assume a working Prometheus push gateway, we # simply print them onto the console metrics: diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/ReportHook.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/ReportHook.scala index eac55bbc0..55364da17 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/ReportHook.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/hook/ReportHook.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -192,7 +192,7 @@ case class ReportHook( None } - // Register custom metrics board + // Reset metrics of custom metrics board without adding it metrics.foreach { board => board.reset(execution.metricSystem) } @@ -235,7 +235,7 @@ case class ReportHook( "phase" -> result.instance.phase.toString, "status" -> result.status.toString, "result" -> JobResultWrapper(result), - "metrics" -> (boardMetrics ++ sinkMetrics).asJava + "metrics" -> (boardMetrics ++ sinkMetrics).sortBy(_.getName()).asJava ) val text = context.evaluate(jobFinishVtl, vars) p.print(text) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSpec.scala index 10d6f91bc..4cbe762d1 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019-2020 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import com.dimajix.flowman.spec.Spec class MetricSpec extends Spec[MetricSelection] { - @JsonProperty(value = "name", required = true) var name:String = _ + @JsonProperty(value = "name", required = true) var name:Option[String] = None @JsonProperty(value = "labels", required = false) var labels:Map[String,String] = Map() @JsonProperty(value = "selector", required = true) var selector:SelectorSpec = _ @@ -46,8 +46,8 @@ class SelectorSpec extends Spec[Selector] { def instantiate(context: Context): Selector = { Selector( - name.map(context.evaluate), - context.evaluate(labels) + name.map(context.evaluate).map(_.r), + context.evaluate(labels).map { case(k,v) => k -> v.r } ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MeasureTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MeasureTarget.scala index ff17cd869..cb47c86fe 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MeasureTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/MeasureTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -132,7 +132,7 @@ case class MeasureTarget( // Publish result as metrics val metrics = execution.metricSystem result.flatMap(_.measurements).foreach { measurement => - val gauge = metrics.findMetric(Selector(Some(measurement.name), measurement.labels)) + val gauge = metrics.findMetric(Selector(measurement.name, measurement.labels)) .headOption .map(_.asInstanceOf[SettableGaugeMetric]) .getOrElse { diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MeasureTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MeasureTargetTest.scala index 08e0221d9..fc2f919c0 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MeasureTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MeasureTargetTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -102,12 +102,12 @@ class MeasureTargetTest extends AnyFlatSpec with Matchers with MockFactory { measureResult.measurements should be (Seq(Measurement("m1", Map("name" -> "a1", "category" -> "measure", "kind" -> "sql", "phase" -> "VERIFY"), 23))) val metrics = execution.metricSystem - metrics.findMetric(Selector(Some("m1"), Map("name" -> "a1", "category" -> "measure", "kind" -> "sql", "phase" -> "VERIFY"))).size should be (1) - metrics.findMetric(Selector(Some("m1"), Map("name" -> "a1", "category" -> "measure", "kind" -> "sql" ))).size should be (1) - metrics.findMetric(Selector(Some("m1"), Map("name" -> "a1", "category" -> "measure"))).size should be (1) - metrics.findMetric(Selector(Some("m1"), Map("name" -> "a1"))).size should be (1) - metrics.findMetric(Selector(Some("m1"), Map())).size should be (1) - val gauges = metrics.findMetric(Selector(Some("m1"), Map("name" -> "a1", "category" -> "measure", "kind" -> "sql", "phase" -> "VERIFY"))) + metrics.findMetric(Selector("m1", Map("name" -> "a1", "category" -> "measure", "kind" -> "sql", "phase" -> "VERIFY"))).size should be (1) + metrics.findMetric(Selector("m1", Map("name" -> "a1", "category" -> "measure", "kind" -> "sql" ))).size should be (1) + metrics.findMetric(Selector("m1", Map("name" -> "a1", "category" -> "measure"))).size should be (1) + metrics.findMetric(Selector("m1", Map("name" -> "a1"))).size should be (1) + metrics.findMetric(Selector("m1")).size should be (1) + val gauges = metrics.findMetric(Selector("m1", Map("name" -> "a1", "category" -> "measure", "kind" -> "sql", "phase" -> "VERIFY"))) gauges.head.asInstanceOf[GaugeMetric].value should be (23.0) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MergeTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MergeTargetTest.scala index d4493382c..4f4d9ef27 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MergeTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/MergeTargetTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -180,7 +180,7 @@ class MergeTargetTest extends AnyFlatSpec with Matchers with LocalSparkSession { target.execute(executor, Phase.BUILD) val metric = executor.metricSystem - .findMetric(Selector(Some("target_records"), target.metadata.asMap)) + .findMetric(Selector("target_records", target.metadata.asMap)) .head .asInstanceOf[GaugeMetric] diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala index a6a463e78..7ab4415ce 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -305,7 +305,7 @@ class RelationTargetTest extends AnyFlatSpec with Matchers with LocalSparkSessio target.execute(executor, Phase.BUILD) val metric = executor.metricSystem - .findMetric(Selector(Some("target_records"), target.metadata.asMap)) + .findMetric(Selector("target_records", target.metadata.asMap)) .head .asInstanceOf[GaugeMetric] From e322dfc6562a7bca60c14669ad25b9c7fd4b75d5 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 21 Feb 2022 19:55:55 +0100 Subject: [PATCH 65/95] Fix linking of ReadRelationMapping with embedded relation --- .../spec/mapping/ReadRelationMapping.scala | 2 +- .../spec/mapping/ReadRelationTest.scala | 62 ++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala index eb6b3ff43..8b7c3193b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala @@ -122,7 +122,7 @@ case class ReadRelationMapping( * Params: linker - The linker object to use for creating new edges */ override def link(linker: Linker): Unit = { - linker.read(relation.identifier, partitions) + linker.read(relation, partitions) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadRelationTest.scala index 243f4be32..48dd3c9e2 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadRelationTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,12 +19,15 @@ package com.dimajix.flowman.spec.mapping import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.model.IdentifierRelationReference import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.Module import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.ValueRelationReference +import com.dimajix.flowman.spec.relation.ValuesRelation import com.dimajix.flowman.types.SingleValue @@ -83,5 +86,62 @@ class ReadRelationTest extends AnyFlatSpec with Matchers { rrm.relation shouldBe a[ValueRelationReference] rrm.relation.identifier should be (RelationIdentifier("embedded", "project")) rrm.relation.name should be ("embedded") + + // Check execution graph + val graph = Graph.ofProject(context, project, Phase.BUILD) + graph.relations.size should be (1) + graph.mappings.size should be (1) + + val relNode = graph.relations.head + relNode.relation should be (rrm.relation.value) + relNode.parent should be (None) + + val mapNode = graph.mappings.head + mapNode.mapping should be (rrm) + mapNode.parent should be (None) + } + + it should "support create an appropriate graph" in { + val spec = + """ + |mappings: + | t0: + | kind: readRelation + | relation: + | name: embedded + | kind: values + | records: + | - ["key",12] + | schema: + | kind: embedded + | fields: + | - name: key_column + | type: string + | - name: value_column + | type: integer + """.stripMargin + + val project = Module.read.string(spec).toProject("project") + val session = Session.builder().withProject(project).disableSpark().build() + val context = session.getContext(project) + val mapping = context.getMapping(MappingIdentifier("t0")) + + mapping shouldBe a[ReadRelationMapping] + val relation = mapping.asInstanceOf[ReadRelationMapping].relation.value + relation shouldBe a[ValuesRelation] + + // Check execution graph + val graph = Graph.ofProject(context, project, Phase.BUILD) + graph.nodes.size should be (3) // 1 mapping + 1 mapping output + 1 relation + graph.relations.size should be (1) + graph.mappings.size should be (1) + + val relNode = graph.relations.head + relNode.relation should be (relation) + relNode.parent should be (None) + + val mapNode = graph.mappings.head + mapNode.mapping should be (mapping) + mapNode.parent should be (None) } } From 759ce99175032c2bedf3e864d43a3b5202aa85d4 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 22 Feb 2022 07:57:54 +0100 Subject: [PATCH 66/95] Move Prometheus sink to flowman-spec --- .../spec}/metric/PrometheusMetricSink.scala | 25 +++++++++++-- .../metric/PrometheusMetricSinkSpec.scala | 36 ------------------- 2 files changed, 22 insertions(+), 39 deletions(-) rename {flowman-core/src/main/scala/com/dimajix/flowman => flowman-spec/src/main/scala/com/dimajix/flowman/spec}/metric/PrometheusMetricSink.scala (82%) delete mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSinkSpec.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/PrometheusMetricSink.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala similarity index 82% rename from flowman-core/src/main/scala/com/dimajix/flowman/metric/PrometheusMetricSink.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala index 1709c333e..44e3e6e9c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/PrometheusMetricSink.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,14 @@ * limitations under the License. */ -package com.dimajix.flowman.metric +package com.dimajix.flowman.spec.metric import java.io.IOException import java.net.URI import scala.util.control.NonFatal +import com.fasterxml.jackson.annotation.JsonProperty import org.apache.http.HttpResponse import org.apache.http.client.HttpResponseException import org.apache.http.client.ResponseHandler @@ -29,14 +30,19 @@ import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.slf4j.LoggerFactory +import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.metric.AbstractMetricSink +import com.dimajix.flowman.metric.GaugeMetric +import com.dimajix.flowman.metric.MetricBoard +import com.dimajix.flowman.metric.MetricSink class PrometheusMetricSink( url:String, labels:Map[String,String] ) -extends AbstractMetricSink { + extends AbstractMetricSink { private val logger = LoggerFactory.getLogger(classOf[PrometheusMetricSink]) override def commit(board:MetricBoard, status:Status) : Unit = { @@ -102,3 +108,16 @@ extends AbstractMetricSink { str.replace("\"","\\\"").replace("\n","").trim } } + + +class PrometheusMetricSinkSpec extends MetricSinkSpec { + @JsonProperty(value = "url", required = true) private var url:String = "" + @JsonProperty(value = "labels", required = false) private var labels:Map[String,String] = Map() + + override def instantiate(context: Context): MetricSink = { + new PrometheusMetricSink( + context.evaluate(url), + labels + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSinkSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSinkSpec.scala deleted file mode 100644 index 6865ad221..000000000 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSinkSpec.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 Kaya Kupferschmidt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dimajix.flowman.spec.metric - -import com.fasterxml.jackson.annotation.JsonProperty - -import com.dimajix.flowman.execution.Context -import com.dimajix.flowman.metric.MetricSink -import com.dimajix.flowman.metric.PrometheusMetricSink - - -class PrometheusMetricSinkSpec extends MetricSinkSpec { - @JsonProperty(value = "url", required = true) private var url:String = "" - @JsonProperty(value = "labels", required = false) private var labels:Map[String,String] = Map() - - override def instantiate(context: Context): MetricSink = { - new PrometheusMetricSink( - context.evaluate(url), - labels - ) - } -} From 748fce6d95d56029b7669d6b18cce9d6582c5c70 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 22 Feb 2022 07:58:28 +0100 Subject: [PATCH 67/95] Improve entitiy lookup in MappingCollector and DeltaVacuumTarget --- .../documentation/MappingCollector.scala | 17 +++++++++-------- .../flowman/documentation/velocity.scala | 18 +++++++++++++++--- .../documentation/MappingCollectorTest.scala | 1 + .../spec/target/DeltaVacuumTarget.scala | 2 +- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala index fb146d000..ef244994b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/MappingCollector.scala @@ -16,18 +16,17 @@ package com.dimajix.flowman.documentation -import scala.collection.mutable import scala.util.control.NonFatal import org.slf4j.LoggerFactory import com.dimajix.common.ExceptionUtils.reasons +import com.dimajix.common.IdentityHashMap import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph import com.dimajix.flowman.graph.MappingRef import com.dimajix.flowman.graph.ReadRelation import com.dimajix.flowman.model.Mapping -import com.dimajix.flowman.model.MappingIdentifier import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.types.StructType @@ -36,17 +35,15 @@ class MappingCollector extends Collector { private val logger = LoggerFactory.getLogger(getClass) override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { - val mappings = mutable.Map[MappingIdentifier, MappingDoc]() + val mappings = IdentityHashMap[Mapping, MappingDoc]() val parent = documentation.reference def getMappingDoc(node:MappingRef) : MappingDoc = { val mapping = node.mapping - mappings.getOrElseUpdate(mapping.identifier, genDoc(node)) + mappings.getOrElseUpdate(mapping, genDoc(node)) } - def getOutputDoc(mappingOutput:MappingOutputIdentifier) : Option[MappingOutputDoc] = { - val mapping = mappingOutput.mapping + def getOutputDoc(mapping:Mapping, output:String) : Option[MappingOutputDoc] = { val doc = mappings.getOrElseUpdate(mapping, genDoc(graph.mapping(mapping))) - val output = mappingOutput.output doc.outputs.find(_.identifier.output == output) } def genDoc(node:MappingRef) : MappingDoc = { @@ -54,7 +51,10 @@ class MappingCollector extends Collector { logger.info(s"Collecting documentation for mapping '${mapping.identifier}'") // Collect fundamental basis information - val inputs = mapping.inputs.flatMap(in => getOutputDoc(in).map(in -> _)).toMap + val inputs = mapping.inputs.flatMap { in => + val inmap = mapping.context.getMapping(in.mapping) + getOutputDoc(inmap, in.output).map(in -> _) + }.toMap val doc = document(execution, parent, mapping, inputs) // Add additional inputs from non-mapping entities @@ -88,6 +88,7 @@ class MappingCollector extends Collector { val ref = doc.reference val outputs = try { + // Do not use Execution.describe because that wouldn't use our hand-crafted input documentation val schemas = mapping.describe(execution, inputSchemas) schemas.map { case(output,schema) => val doc = MappingOutputDoc( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 670e75c0c..1c3e4421f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -166,7 +166,19 @@ final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(p }.orNull } - def getMappings() : java.util.List[MappingDocWrapper] = project.mappings.map(MappingDocWrapper).asJava - def getRelations() : java.util.List[RelationDocWrapper] = project.relations.map(RelationDocWrapper).asJava - def getTargets() : java.util.List[TargetDocWrapper] = project.targets.map(TargetDocWrapper).asJava + def getMappings() : java.util.List[MappingDocWrapper] = + project.mappings + .sortBy(_.identifier.toString) + .map(MappingDocWrapper) + .asJava + def getRelations() : java.util.List[RelationDocWrapper] = + project.relations + .sortBy(_.identifier.toString) + .map(RelationDocWrapper) + .asJava + def getTargets() : java.util.List[TargetDocWrapper] = + project.targets + .sortBy(_.identifier.toString) + .map(TargetDocWrapper) + .asJava } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala index 9ece3ea14..014f2df21 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/MappingCollectorTest.scala @@ -80,6 +80,7 @@ class MappingCollectorTest extends AnyFlatSpec with Matchers with MockFactory { (mapping1.inputs _).expects().returns(Set(MappingOutputIdentifier("project/m2"))) (mapping1.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) (mapping1.documentation _).expects().returns(None) + (mapping1.context _).expects().returns(context) (mapping2.identifier _).expects().atLeastOnce().returns(MappingIdentifier("project/m2")) (mapping2.inputs _).expects().returns(Set()) (mapping2.describe: (Execution,Map[MappingOutputIdentifier,StructType]) => Map[String,StructType] ).expects(*,*).returns(Map("main" -> StructType(Seq()))) diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala index 5c26fdc30..fb7409572 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala @@ -120,7 +120,7 @@ case class DeltaVacuumTarget( */ override def link(linker: Linker, phase:Phase): Unit = { if (phase == Phase.BUILD) { - linker.write(relation.identifier, Map.empty[String,SingleValue]) + linker.write(relation, Map.empty[String,SingleValue]) } } From 7cf5331691cfe91f31fbab7ae2225ec2a1ec6699 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 22 Feb 2022 08:19:30 +0100 Subject: [PATCH 68/95] Provide color to test status in documentation --- .../flowman/documentation/TestResult.scala | 35 ++++++++++++++++--- .../flowman/documentation/velocity.scala | 6 ++++ .../flowman/documentation/html/project.vtl | 16 +++++++-- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala index dbe08e4cc..29babf3ee 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala @@ -16,13 +16,34 @@ package com.dimajix.flowman.documentation -sealed abstract class TestStatus extends Product with Serializable + +sealed abstract class TestStatus extends Product with Serializable { + def success : Boolean + def failure : Boolean + def run : Boolean +} object TestStatus { - final case object FAILED extends TestStatus - final case object SUCCESS extends TestStatus - final case object ERROR extends TestStatus - final case object NOT_RUN extends TestStatus + final case object FAILED extends TestStatus { + def success : Boolean = false + def failure : Boolean = true + def run : Boolean = true + } + final case object SUCCESS extends TestStatus { + def success : Boolean = true + def failure : Boolean = false + def run : Boolean = true + } + final case object ERROR extends TestStatus { + def success : Boolean = false + def failure : Boolean = true + def run : Boolean = true + } + final case object NOT_RUN extends TestStatus { + def success : Boolean = false + def failure : Boolean = false + def run : Boolean = false + } } @@ -55,4 +76,8 @@ final case class TestResult( details = details.map(_.reparent(ref)) ) } + + def success : Boolean = status.success + def failure : Boolean = status.failure + def run : Boolean = status.run } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index 1c3e4421f..ad44d3e57 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -47,6 +47,8 @@ final case class TestResultWrapper(result:TestResult) extends FragmentWrapper(re override def toString: String = result.status.toString def getStatus() : String = result.status.toString + def getSuccess() : Boolean = result.success + def getFailure() : Boolean = result.failure } @@ -56,6 +58,8 @@ final case class ColumnTestWrapper(test:ColumnTest) extends FragmentWrapper(test def getName() : String = test.name def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") + def getSuccess() : Boolean = test.result.exists(_.success) + def getFailure() : Boolean = test.result.exists(_.failure) } @@ -79,6 +83,8 @@ final case class SchemaTestWrapper(test:SchemaTest) extends FragmentWrapper(test def getName() : String = test.name def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") + def getSuccess() : Boolean = test.result.exists(_.success) + def getFailure() : Boolean = test.result.exists(_.failure) } diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 2ef9927e6..95cc3417e 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -129,6 +129,13 @@ flex: 25%; } + span.success { + color: forestgreen; + } + span.failure { + color: orangered; + } + span.bubble { display: inline-flex; background-color: #0e84b5; @@ -140,6 +147,11 @@ + +#macro(testStatus $test) +#if(${test.success})#elseif(${test.failure})#else#end${test.status} +#end + #macro(schema $schema) @@ -164,7 +176,7 @@ #foreach($test in ${column.tests}) - + #end @@ -188,7 +200,7 @@ #foreach($test in ${schema.tests}) - + #end From 9fe917120d5c59895734019ca33ce4fc60177bcf Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 22 Feb 2022 09:02:09 +0100 Subject: [PATCH 69/95] Add unittest for SchemaTest --- .../flowman/documentation/SchemaTest.scala | 8 +- .../documentation/SchemaTestTest.scala | 129 ++++++++++++++++++ 2 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala index fa4c013f5..f84fb416b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -103,7 +103,7 @@ class DefaultSchemaTestExecutor extends SchemaTestExecutor { test match { case p:PrimaryKeySchemaTest => val cols = p.columns.map(df(_)) - val agg = df.filter(cols.map(_.isNotNull).reduce(_ && _)).groupBy(cols:_*).count() + val agg = df.filter(cols.map(_.isNotNull).reduce(_ || _)).groupBy(cols:_*).count() val result = agg.groupBy(agg(agg.columns(cols.length)) > 1).count().collect() val numSuccess = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) @@ -121,7 +121,11 @@ class DefaultSchemaTestExecutor extends SchemaTestExecutor { execution.instantiate(mapping, map.output) }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test ${test.reference.toString}")) val cols = f.columns.map(df(_)) - val otherCols = f.references.map(otherDf(_)) + val otherCols = + if (f.references.nonEmpty) + f.references.map(otherDf(_)) + else + f.columns.map(otherDf(_)) val joined = df.join(otherDf, cols.zip(otherCols).map(lr => lr._1 === lr._2).reduce(_ && _), "left") executePredicateTest(joined, test, otherCols.map(_.isNotNull).reduce(_ || _)) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala new file mode 100644 index 000000000..8b6c1b3b2 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala @@ -0,0 +1,129 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.documentation + +import org.apache.spark.storage.StorageLevel +import org.scalamock.scalatest.MockFactory +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.MappingIdentifier +import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.spark.testing.LocalSparkSession + + +class SchemaTestTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { + "A PrimaryKeySchemaTest" should "be executable" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val context = session.context + val testExecutor = new DefaultSchemaTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,3), + (None,3,4), + (None,3,5) + )) + + val test1 = PrimaryKeySchemaTest(None, columns=Seq("_1","_3")) + val result1 = testExecutor.execute(execution, context, df, test1) + result1 should be (Some(TestResult(Some(test1.reference), TestStatus.SUCCESS, description=Some("3 keys are unique, 0 keys are non-unique")))) + + val test2 = PrimaryKeySchemaTest(None, columns=Seq("_1","_2")) + val result2 = testExecutor.execute(execution, context, df, test2) + result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 keys are unique, 1 keys are non-unique")))) + } + + "An ExpressionSchemaTest" should "work" in { + val session = Session.builder() + .withSparkSession(spark) + .build() + val execution = session.execution + val context = session.context + val testExecutor = new DefaultSchemaTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),2,1), + (None,3,2) + )) + + val test1 = ExpressionSchemaTest(None, expression="_2 > _3") + val result1 = testExecutor.execute(execution, context, df, test1) + result1 should be (Some(TestResult(Some(test1.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + + val test2 = ExpressionSchemaTest(None, expression="_2 < _3") + val result2 = testExecutor.execute(execution, context, df, test2) + result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + } + + "A ForeignKeySchemaTest" should "work" in { + val mappingSpec = mock[Prototype[Mapping]] + val mapping = mock[Mapping] + + val session = Session.builder() + .withSparkSession(spark) + .build() + val project = Project( + name = "project", + mappings = Map("mapping" -> mappingSpec) + ) + val context = session.getContext(project) + val execution = session.execution + + val testExecutor = new DefaultSchemaTestExecutor + + val df = spark.createDataFrame(Seq( + (Some(1),1,1), + (None,2,3) + )) + val otherDf = spark.createDataFrame(Seq( + (1,1), + (2,2) + )) + + (mappingSpec.instantiate _).expects(*).returns(mapping) + (mapping.context _).expects().returns(context) + (mapping.inputs _).expects().returns(Set()) + (mapping.outputs _).expects().atLeastOnce().returns(Set("main")) + (mapping.broadcast _).expects().returns(false) + (mapping.cache _).expects().returns(StorageLevel.NONE) + (mapping.checkpoint _).expects().returns(false) + (mapping.identifier _).expects().returns(MappingIdentifier("project/mapping")) + (mapping.execute _).expects(*,*).returns(Map("main" -> otherDf)) + + val test1 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_1")) + val result1 = testExecutor.execute(execution, context, df, test1) + result1 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + + val test2 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_3"), references=Seq("_2")) + val result2 = testExecutor.execute(execution, context, df, test2) + result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + + val test3 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2")) + val result3 = testExecutor.execute(execution, context, df, test3) + result3 should be (Some(TestResult(Some(test3.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + + val test4 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2"), references=Seq("_3")) + an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, test4)) + } +} From 245511c7e96b3f71330893a6237dc5b8fb12e96e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 22 Feb 2022 16:12:11 +0100 Subject: [PATCH 70/95] Add new SPARK_JARS environment variable --- docs/cookbook/override-jars.md | 24 +++++++++++++++++++++++ docs/plugins/aws.md | 12 ++++++++++++ docs/plugins/azure.md | 9 +++++++++ docs/plugins/delta.md | 9 +++++++++ docs/plugins/json.md | 9 +++++++++ docs/plugins/kafka.md | 9 +++++++++ docs/plugins/mariadb.md | 11 +++++++++++ docs/plugins/mssql.md | 4 ---- docs/plugins/mssqlserver.md | 19 ++++++++++++++++++ docs/plugins/mysql.md | 11 +++++++++++ docs/plugins/openapi.md | 12 ++++++++++++ docs/plugins/swagger.md | 12 ++++++++++++ flowman-dist/conf/flowman-env.sh.template | 3 +++ flowman-dist/libexec/flowman-common.sh | 9 ++++++++- 14 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 docs/cookbook/override-jars.md delete mode 100644 docs/plugins/mssql.md create mode 100644 docs/plugins/mssqlserver.md diff --git a/docs/cookbook/override-jars.md b/docs/cookbook/override-jars.md new file mode 100644 index 000000000..4399f5c5a --- /dev/null +++ b/docs/cookbook/override-jars.md @@ -0,0 +1,24 @@ +# Force Spark to specific jar version + +A common problem with Spark and specifically with many Hadoop environments (like Cloudera) are mismatches between +application jar versions and jars provided by the runtime environment. Flowman is built with carefully set dependency +version to match those of each supported runtime environment. But sometimes this might not be enough. + +For example Cloudera ships with a rather old JDBC driver for MS SQL Server / Aure SQL Server which is not compatible +with the `sqlserver` relation type provided by the [MS SQL Server plugin](../plugins/mssqlserver.md). But it is still +possible to force Spark to use the newer JDBC driver by changing some config options + + +## Configuration + +You need to add the following lines to your custom `flowman-env.sh` file which is stored in the `conf` subdirectory: + +```shell +# Add MS SQL JDBC Driver. Normally this is handled by the plugin mechanism, but Cloudera already provides some +# old version of the JDBC driver, and this is the only place where we can force to use our JDBC driver +SPARK_JARS="$FLOWMAN_HOME/plugins/flowman-mssqlserver/mssql-jdbc-9.2.1.jre8.jar" +SPARK_OPTS="--conf spark.executor.extraClassPath=mssql-jdbc-9.2.1.jre8.jar" +``` +The first line will explicitly add the plugin jar to the list of jars as passed to `spark-submit`. But this is still +not enough, we also have to set `spark.executor.extraClassPath` which will *prepend* the specified jars to the +classpath of the executor. diff --git a/docs/plugins/aws.md b/docs/plugins/aws.md index 65a8385ee..24de19a17 100644 --- a/docs/plugins/aws.md +++ b/docs/plugins/aws.md @@ -1 +1,13 @@ # AWS Plugin + +The AWS plugin does not provide new entity types to Flowman, but will provide compatibility with the S3 object +store to be usable as a data source or sink via the `s3a` file system. + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-aws +``` diff --git a/docs/plugins/azure.md b/docs/plugins/azure.md index f1da18c0c..b4b0c23f2 100644 --- a/docs/plugins/azure.md +++ b/docs/plugins/azure.md @@ -2,3 +2,12 @@ The Azure plugin mainly provides the ADLS (Azure DataLake Filesystem) and ABS (Azure Blob Filesystem) to be used as the storage layer. + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-azure +``` diff --git a/docs/plugins/delta.md b/docs/plugins/delta.md index a60e251a5..795c69afd 100644 --- a/docs/plugins/delta.md +++ b/docs/plugins/delta.md @@ -12,3 +12,12 @@ move to Spark 3.0+. * [`deltaTable` relation](../spec/relation/deltaTable.md) * [`deltaFile` relation](../spec/relation/deltaFile.md) * ['deltaVacuum' target](../spec/target/deltaVacuum.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-delta +``` diff --git a/docs/plugins/json.md b/docs/plugins/json.md index 878164a32..652fccf91 100644 --- a/docs/plugins/json.md +++ b/docs/plugins/json.md @@ -2,3 +2,12 @@ ## Provided Entities * [`json` schema](../spec/schema/json.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-json +``` diff --git a/docs/plugins/kafka.md b/docs/plugins/kafka.md index 94cf05d3d..b8d72849b 100644 --- a/docs/plugins/kafka.md +++ b/docs/plugins/kafka.md @@ -2,3 +2,12 @@ ## Provided Entities * [`kafka` relation](../spec/relation/kafka.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-kafka +``` diff --git a/docs/plugins/mariadb.md b/docs/plugins/mariadb.md index 28b2a6e43..346c4c5de 100644 --- a/docs/plugins/mariadb.md +++ b/docs/plugins/mariadb.md @@ -1 +1,12 @@ # MariaDB Plugin + +The MariaDB plugin mainly provides a JDBC driver to access MariaDB databases via the [JDBC relation](../spec/relation/jdbc.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-mariadb +``` diff --git a/docs/plugins/mssql.md b/docs/plugins/mssql.md deleted file mode 100644 index c2b2b5eb4..000000000 --- a/docs/plugins/mssql.md +++ /dev/null @@ -1,4 +0,0 @@ -# MS SQL Server Plugin - -## Provided Entities -* [`sqlserver` relation](../spec/relation/sqlserver.md) diff --git a/docs/plugins/mssqlserver.md b/docs/plugins/mssqlserver.md new file mode 100644 index 000000000..9237d7a57 --- /dev/null +++ b/docs/plugins/mssqlserver.md @@ -0,0 +1,19 @@ +# MS SQL Server Plugin + +The MS SQL Server plugin provides a JDBC driver to access MS SQL Server and Azure SQL Server databases via +the [JDBC relation](../spec/relation/jdbc.md). Moreover, it also provides a specialized +[`sqlserver` relation](../spec/relation/sqlserver.md) which uses bulk copy to speed up writing process and it +also uses temp tables to encapsulate the whole data upload within a transaction. + + +## Provided Entities +* [`sqlserver` relation](../spec/relation/sqlserver.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-mssqlserver +``` diff --git a/docs/plugins/mysql.md b/docs/plugins/mysql.md index 17a308daa..802afa606 100644 --- a/docs/plugins/mysql.md +++ b/docs/plugins/mysql.md @@ -1 +1,12 @@ # MySQL Plugin + +The MySQL plugin mainly provides a JDBC driver to access MySQL databases via the [JDBC relation](../spec/relation/jdbc.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-mysql +``` diff --git a/docs/plugins/openapi.md b/docs/plugins/openapi.md index 5e8a8ba28..ca5222993 100644 --- a/docs/plugins/openapi.md +++ b/docs/plugins/openapi.md @@ -1,4 +1,16 @@ # OpenAPI Plugin +The OpenAPI plugin provides compatibility with OpenAPI schema definition files. + + ## Provided Entities * [`openApi` schema](../spec/schema/open-api.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-openapi +``` diff --git a/docs/plugins/swagger.md b/docs/plugins/swagger.md index 623194122..a2b390371 100644 --- a/docs/plugins/swagger.md +++ b/docs/plugins/swagger.md @@ -1,4 +1,16 @@ # Swagger Plugin +The Swagger plugin provides compatibility with Swagger schema definition files. + + ## Provided Entities * [`swagger` schema](../spec/schema/swagger.md) + + +## Activation + +The plugin can be easily activated by adding the following section to the [default-namespace.yml](../spec/namespace.md) +```yaml +plugins: + - flowman-swagger +``` diff --git a/flowman-dist/conf/flowman-env.sh.template b/flowman-dist/conf/flowman-env.sh.template index ca28cace3..904a49360 100644 --- a/flowman-dist/conf/flowman-env.sh.template +++ b/flowman-dist/conf/flowman-env.sh.template @@ -53,6 +53,9 @@ SPARK_DRIVER_MEMORY="3G" # #SPARK_SUBMIT= +# Add some more jars to spark-submit. This can be a comma-separated list of jars. +# +#SPARK_JARS= # Apply any proxy settings from the system environment # diff --git a/flowman-dist/libexec/flowman-common.sh b/flowman-dist/libexec/flowman-common.sh index c57dcb969..cd73013cb 100644 --- a/flowman-dist/libexec/flowman-common.sh +++ b/flowman-dist/libexec/flowman-common.sh @@ -19,6 +19,7 @@ fi # Set basic Spark options : ${SPARK_SUBMIT:="$SPARK_HOME"/bin/spark-submit} : ${SPARK_OPTS:=""} +: ${SPARK_JARS:=""} : ${SPARK_DRIVER_JAVA_OPTS:="-server"} : ${SPARK_EXECUTOR_JAVA_OPTS:="-server"} @@ -84,11 +85,17 @@ flowman_lib() { spark_submit() { + if [ "$SPARK_JARS" != "" ]; then + extra_jars=",$SPARK_JARS" + else + extra_jars="" + fi + $SPARK_SUBMIT \ --driver-java-options "$SPARK_DRIVER_JAVA_OPTS" \ --conf spark.execution.extraJavaOptions="$SPARK_EXECUTOR_JAVA_OPTS" \ --class $3 \ $SPARK_OPTS \ - --jars "$(flowman_lib $2)" \ + --jars "$(flowman_lib $2)$extra_jars" \ $FLOWMAN_HOME/lib/$1 "${@:4}" } From 5e48cfe9e8ded29cc11f64350030f9f749367dc0 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 23 Feb 2022 10:33:38 +0100 Subject: [PATCH 71/95] Implement new JDBC metric sink --- CHANGELOG.md | 3 +- docker/conf/default-namespace.yml | 24 ++- docs/cookbook/override-jars.md | 7 +- docs/plugins/json.md | 3 + docs/spec/metric/jdbc.md | 46 +++++ docs/spec/metric/prometheus.md | 11 ++ docs/spec/relation/sqlserver.md | 5 +- examples/weather/job/main.yml | 12 ++ .../flowman/history/JdbcStateStore.scala | 67 +++---- .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 27 +++ .../dimajix/flowman/metric/MetricBoard.scala | 2 +- .../conf/default-namespace.yml.template | 13 +- .../spec/metric/JdbcMetricRepository.scala | 185 ++++++++++++++++++ .../flowman/spec/metric/JdbcMetricSink.scala | 136 +++++++++++++ .../flowman/spec/metric/MetricSinkSpec.scala | 3 +- .../spec/metric/PrometheusMetricSink.scala | 6 +- .../spec/history/JdbcStateStoreTest.scala | 15 +- .../spec/metric/JdbcMetricSinkTest.scala | 133 +++++++++++++ 18 files changed, 641 insertions(+), 57 deletions(-) create mode 100644 docs/spec/metric/jdbc.md create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala create mode 100644 flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index e416a291c..888d1680e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Version 0.22.0 * Add new `sqlserver` relation -* Work on new documentation subsystem +* Implement new documentation subsystem * Change default build to Spark 3.2.1 and Hadoop 3.3.1 * Add new `drop` target for removing tables * Speed up project loading by reusing Jackson mapper +* Implement new `jdbc` metric sink # Version 0.21.1 - 2022-01-28 diff --git a/docker/conf/default-namespace.yml b/docker/conf/default-namespace.yml index 06363ec75..0f2cfcf0d 100644 --- a/docker/conf/default-namespace.yml +++ b/docker/conf/default-namespace.yml @@ -13,6 +13,28 @@ connections: username: $System.getenv('FLOWMAN_LOGDB_USER', '') password: $System.getenv('FLOWMAN_LOGDB_PASSWORD', '') +# This adds a hook for creating an execution log in a file +hooks: + kind: report + location: ${project.basedir}/generated-report.txt + metrics: + # Define common labels for all metrics + labels: + project: ${project.name} + metrics: + # Collect everything + - selector: + name: .* + labels: + category: ${category} + kind: ${kind} + name: ${name} + +# This configures where metrics should be written to. Since we cannot assume a working Prometheus push gateway, we +# simply print them onto the console +metrics: + - kind: console + config: - spark.sql.warehouse.dir=/opt/flowman/hive/warehouse - spark.hadoop.hive.metastore.uris= @@ -21,7 +43,7 @@ config: store: kind: file - location: /opt/flowman/examples + location: $System.getenv('FLOWMAN_HOME')/examples plugins: - flowman-aws diff --git a/docs/cookbook/override-jars.md b/docs/cookbook/override-jars.md index 4399f5c5a..9a0201ab3 100644 --- a/docs/cookbook/override-jars.md +++ b/docs/cookbook/override-jars.md @@ -4,9 +4,10 @@ A common problem with Spark and specifically with many Hadoop environments (like application jar versions and jars provided by the runtime environment. Flowman is built with carefully set dependency version to match those of each supported runtime environment. But sometimes this might not be enough. -For example Cloudera ships with a rather old JDBC driver for MS SQL Server / Aure SQL Server which is not compatible -with the `sqlserver` relation type provided by the [MS SQL Server plugin](../plugins/mssqlserver.md). But it is still -possible to force Spark to use the newer JDBC driver by changing some config options +For example Cloudera ships with a rather old JDBC driver for MS SQL Server / Azure SQL Server which is not compatible +with the `sqlserver` relation type provided by the [MS SQL Server plugin](../plugins/mssqlserver.md). This will result +in `ClassNotFound` or `MethodNotFound` exceptions during execution. But it is still +possible to force Spark to use the newer JDBC driver by changing some config options. ## Configuration diff --git a/docs/plugins/json.md b/docs/plugins/json.md index 652fccf91..0aff7043a 100644 --- a/docs/plugins/json.md +++ b/docs/plugins/json.md @@ -1,5 +1,8 @@ # JSON Plugin +The OpenAPI plugin provides compatibility with JSON schema definition files. + + ## Provided Entities * [`json` schema](../spec/schema/json.md) diff --git a/docs/spec/metric/jdbc.md b/docs/spec/metric/jdbc.md new file mode 100644 index 000000000..11be79bbf --- /dev/null +++ b/docs/spec/metric/jdbc.md @@ -0,0 +1,46 @@ +# JDBC Metric Sink + +The `jdbc` metric sink is a very simple sink, which simply stores all collected metrics in a relational database. +Actually it is highly recommended setting up a proper monitoring using Prometheus or other supported and established +monitoring system instead of relying on a relational database. + + +## Example + +```yaml +metrics: + # Also add console metric sink (this is optional, but recommended) + - kind: console + # Now configure the Prometheus metric sink + - kind: jdbc + # Specify labels on a commit level + labels: + project: ${project.name} + version: ${project.version} + connection: + kind: jdbc + driver: "com.mysql.cj.jdbc.Driver" + url: "jdbc:mysql://mysql-01.ffm.dimajix.net/dimajix_flowman" + username: "flowman-metrics" + password: "my-secret-password" +``` + + +## Fields + +* `kind` **(mandatory)** *(string)*: `prometheus` + +* `connection` **(mandatory)** *(string/connection)*: Either the name of a [`jdbc` connection](../connection/jdbc.md) +or an directly embedded JDBC connection (like in the example). + +* `commitTable` **(optional)** *(string)* *(default: flowman_metric_commits)*: The name of the table which will +get one entry ("commit") per publication of metrics. + +* `commitLabelTable` **(optional)** *(string)* *(default: flowman_metric_commit_labels)*: The name of the table which will + contain the labels of each commit. + +* `metricTable` **(optional)** *(string)* *(default: flowman_metrics)*: The name of the table which will contain +the metrics. + +* `metricLabelTable` **(optional)** *(string)* *(default: flowman_metric_labels)*: The name of the table which will +contain the labels of each metric. diff --git a/docs/spec/metric/prometheus.md b/docs/spec/metric/prometheus.md index 094ae51ab..1b81d8270 100644 --- a/docs/spec/metric/prometheus.md +++ b/docs/spec/metric/prometheus.md @@ -1,5 +1,9 @@ # Prometheus Metric Sink +The `prometheus` metric sink allows you to publish collected metrics to a Prometheus push gateway. This then can +be scraped by a Prometheus server. + + ## Example The following example configures a prometheus sink in a namespace. You would need to include this snippet for example in the `default-namespace.yml` in the Flowman configuration directory @@ -18,3 +22,10 @@ metrics: ``` ## Fields + +* `kind` **(mandatory)** *(string)*: `prometheus` + +* `url` **(mandatory)** *(string)*: Specifies the URL of the prometheus push gateway + +* `labels` **(optional)** *(map)*: Specifies an additional set of labels to be pushed to prometheus. This set +of labels will determine the path in Prometheus push gateway, under which all metrics will be atomically published. diff --git a/docs/spec/relation/sqlserver.md b/docs/spec/relation/sqlserver.md index 7988e4a52..77fb96fce 100644 --- a/docs/spec/relation/sqlserver.md +++ b/docs/spec/relation/sqlserver.md @@ -1,7 +1,10 @@ # SQL Server Relations The SQL Server relation allows you to access MS SQL Server and Azure SQL databases using a JDBC driver. It uses the -`spark-sql-connector` from Microsoft to speed up processing. +`spark-sql-connector` from Microsoft to speed up processing. The `sqlserver` relation will also make use of a +global temporary table as an intermediate staging target and then atomically replace the contents of the target +table with the contents of the temp table within a single transaction. + ## Plugin diff --git a/examples/weather/job/main.yml b/examples/weather/job/main.yml index 8a050f797..8a16cb298 100644 --- a/examples/weather/job/main.yml +++ b/examples/weather/job/main.yml @@ -17,3 +17,15 @@ jobs: - metrics # Generate documentation - documentation + + # Define metrics to be published while running this job + metrics: + labels: + project: "${project.name}" + metrics: + - selector: + name: ".*" + labels: + category: "$category" + kind: "$kind" + name: "$name" diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala index 4dd2eb835..b063d8518 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,6 +39,7 @@ import com.dimajix.flowman.history.JdbcStateRepository.JobRun import com.dimajix.flowman.history.JdbcStateRepository.TargetRun import com.dimajix.flowman.history.JdbcStateStore.JdbcJobToken import com.dimajix.flowman.history.JdbcStateStore.JdbcTargetToken +import com.dimajix.flowman.jdbc.JdbcUtils import com.dimajix.flowman.metric.GaugeMetric import com.dimajix.flowman.metric.Metric import com.dimajix.flowman.model.Job @@ -98,7 +99,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t None ) logger.debug(s"Checking last state of '${run.phase}' job '${run.name}' in history database") - withSession { repository => + withRepository { repository => repository.getJobState(run) } } @@ -109,7 +110,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @return */ override def getJobMetrics(jobId:String) : Seq[Measurement] = { - withSession { repository => + withRepository { repository => repository.getJobMetrics(jobId.toLong) } } @@ -121,7 +122,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @return */ override def getJobGraph(jobId: String): Option[Graph] = { - withSession { repository => + withRepository { repository => repository.getJobGraph(jobId.toLong) } } @@ -133,7 +134,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @return */ override def getJobEnvironment(jobId: String): Map[String, String] = { - withSession { repository => + withRepository { repository => repository.getJobEnvironment(jobId.toLong) } } @@ -167,7 +168,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t } logger.debug(s"Start '${digest.phase}' job '${run.name}' in history database") - val run2 = withSession { repository => + val run2 = withRepository { repository => repository.insertJobRun(run, digest.args, env) } @@ -187,7 +188,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t val now = new Timestamp(Clock.systemDefaultZone().instant().toEpochMilli) val graph = Graph.ofGraph(jdbcToken.graph.build()) - withSession{ repository => + withRepository{ repository => repository.setJobStatus(run.copy(end_ts = Some(now), status=status.upper, error=result.exception.map(_.toString))) repository.insertJobMetrics(run.id, metrics) repository.insertJobGraph(run.id, graph) @@ -215,13 +216,13 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t None ) logger.debug(s"Checking state of target '${run.name}' in history database") - withSession { repository => + withRepository { repository => repository.getTargetState(run, target.partitions) } } def getTargetState(targetId: String): TargetState = { - withSession { repository => + withRepository { repository => repository.getTargetState(targetId.toLong) } } @@ -251,7 +252,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t ) logger.debug(s"Start '${digest.phase}' target '${run.name}' in history database") - val run2 = withSession { repository => + val run2 = withRepository { repository => repository.insertTargetRun(run, digest.partitions) } JdbcTargetToken(run2, parentRun) @@ -269,7 +270,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t logger.info(s"Mark '${run.phase}' target '${run.name}' as $status in history database") val now = new Timestamp(Clock.systemDefaultZone().instant().toEpochMilli) - withSession{ repository => + withRepository{ repository => repository.setTargetStatus(run.copy(end_ts = Some(now), status=status.upper, error=result.exception.map(_.toString))) } @@ -289,21 +290,21 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @return */ override def findJobs(query:JobQuery, order:Seq[JobOrder]=Seq(), limit:Int=10000, offset:Int=0) : Seq[JobState] = { - withSession { repository => + withRepository { repository => repository.findJobs(query, order, limit, offset) } } override def countJobs(query: JobQuery): Int = { - withSession { repository => + withRepository { repository => repository.countJobs(query) } } override def countJobs(query: JobQuery, grouping: JobColumn): Map[String, Int] = { - withSession { repository => + withRepository { repository => repository.countJobs(query, grouping).toMap } } @@ -317,25 +318,25 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @return */ override def findTargets(query:TargetQuery, order:Seq[TargetOrder]=Seq(), limit:Int=10000, offset:Int=0) : Seq[TargetState] = { - withSession { repository => + withRepository { repository => repository.findTargets(query, order, limit, offset) } } override def countTargets(query: TargetQuery): Int = { - withSession { repository => + withRepository { repository => repository.countTargets(query) } } override def countTargets(query: TargetQuery, grouping: TargetColumn): Map[String, Int] = { - withSession { repository => + withRepository { repository => repository.countTargets(query, grouping).toMap } } override def findJobMetrics(jobQuery: JobQuery, groupings: Seq[String]): Seq[MetricSeries] = { - withSession { repository => + withRepository { repository => repository.findMetrics(jobQuery, groupings) } } @@ -362,7 +363,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t * @tparam T * @return */ - private def withSession[T](query: JdbcStateRepository => T) : T = { + private def withRepository[T](query: JdbcStateRepository => T) : T = { def retry[T](n:Int)(fn: => T) : T = { try { fn @@ -376,42 +377,20 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t } retry(retries) { - val repository = newRepository() + ensureTables() query(repository) } } private var tablesCreated:Boolean = false + private lazy val repository = new JdbcStateRepository(connection, JdbcUtils.getProfile(connection.driver)) - private def newRepository() : JdbcStateRepository = { - // Get Connection - val derbyPattern = """.*\.derby\..*""".r - val sqlitePattern = """.*\.sqlite\..*""".r - val h2Pattern = """.*\.h2\..*""".r - val mariadbPattern = """.*\.mariadb\..*""".r - val mysqlPattern = """.*\.mysql\..*""".r - val postgresqlPattern = """.*\.postgresql\..*""".r - val sqlserverPattern = """.*\.sqlserver\..*""".r - val profile = connection.driver match { - case derbyPattern() => DerbyProfile - case sqlitePattern() => SQLiteProfile - case h2Pattern() => H2Profile - case mysqlPattern() => MySQLProfile - case mariadbPattern() => MySQLProfile - case postgresqlPattern() => PostgresProfile - case sqlserverPattern() => SQLServerProfile - case _ => throw new UnsupportedOperationException(s"Database with driver ${connection.driver} is not supported") - } - - val repository = new JdbcStateRepository(connection, profile) - + private def ensureTables() : Unit = { // Create Database if not exists if (!tablesCreated) { repository.create() tablesCreated = true } - - repository } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index 3edc9fbbd..b46134aa3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -35,6 +35,13 @@ import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils.createConnectio import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils.savePartition import org.apache.spark.sql.jdbc.JdbcDialects import org.slf4j.LoggerFactory +import slick.jdbc.DerbyProfile +import slick.jdbc.H2Profile +import slick.jdbc.JdbcProfile +import slick.jdbc.MySQLProfile +import slick.jdbc.PostgresProfile +import slick.jdbc.SQLServerProfile +import slick.jdbc.SQLiteProfile import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableChange.AddColumn @@ -341,4 +348,24 @@ object JdbcUtils { getConnection, quotedTarget, iterator, sourceSchema, insertStmt, batchSize, sparkDialect, isolationLevel, options) } } + + def getProfile(driver:String) : JdbcProfile = { + val derbyPattern = """.*\.derby\..*""".r + val sqlitePattern = """.*\.sqlite\..*""".r + val h2Pattern = """.*\.h2\..*""".r + val mariadbPattern = """.*\.mariadb\..*""".r + val mysqlPattern = """.*\.mysql\..*""".r + val postgresqlPattern = """.*\.postgresql\..*""".r + val sqlserverPattern = """.*\.sqlserver\..*""".r + driver match { + case derbyPattern() => DerbyProfile + case sqlitePattern() => SQLiteProfile + case h2Pattern() => H2Profile + case mysqlPattern() => MySQLProfile + case mariadbPattern() => MySQLProfile + case postgresqlPattern() => PostgresProfile + case sqlserverPattern() => SQLServerProfile + case _ => throw new UnsupportedOperationException(s"Database with driver ${driver} is not supported") + } + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala index aad40ccd3..860e7cae5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricBoard.scala @@ -70,7 +70,7 @@ final case class MetricBoard( /** * A MetricSelection represents a possibly dynamic set of Metrics to be published inside a MetricBoard */ -final case class MetricSelection(name:Option[String], selector:Selector, labels:Map[String,String]) { +final case class MetricSelection(name:Option[String] = None, selector:Selector, labels:Map[String,String] = Map()) { /** * Returns all metrics identified by this selection. This operation may be expensive, since the set of metrics may be * dynamic and change over time diff --git a/flowman-dist/conf/default-namespace.yml.template b/flowman-dist/conf/default-namespace.yml.template index 2d6cfc82f..a71a36954 100644 --- a/flowman-dist/conf/default-namespace.yml.template +++ b/flowman-dist/conf/default-namespace.yml.template @@ -61,7 +61,18 @@ hooks: # This configures where metrics should be written to. Since we cannot assume a working Prometheus push gateway, we # simply print them onto the console metrics: - kind: console + - kind: console + # Optionally add a JDBC metric sink + #- kind: jdbc + # labels: + # project: ${project.name} + # version: ${project.version} + # connection: + # kind: jdbc + # url: jdbc:sqlserver://localhost:1433;databaseName=flowman_metrics + # driver: "com.microsoft.sqlserver.jdbc.SQLServerDriver" + # username: "sa" + # password: "yourStrong(!)Password" # This section contains global configuration properties. These still can be overwritten within projects or profiles diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala new file mode 100644 index 000000000..e9ba5dec2 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala @@ -0,0 +1,185 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.metric + +import java.sql.Timestamp +import java.time.Instant +import java.util.Locale +import java.util.Properties + +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.language.higherKinds +import scala.util.control.NonFatal + +import org.slf4j.LoggerFactory +import slick.jdbc.JdbcProfile + +import com.dimajix.flowman.metric.GaugeMetric +import com.dimajix.flowman.spec.connection.JdbcConnection +import com.dimajix.flowman.spec.metric.JdbcMetricRepository.Commit +import com.dimajix.flowman.spec.metric.JdbcMetricRepository.CommitLabel +import com.dimajix.flowman.spec.metric.JdbcMetricRepository.Measurement +import com.dimajix.flowman.spec.metric.JdbcMetricRepository.MetricLabel + + + +private[metric] object JdbcMetricRepository { + case class Commit( + id:Long, + ts:Timestamp + ) + case class CommitLabel( + commit_id:Long, + name:String, + value:String + ) + case class Measurement( + id:Long, + commit_id:Long, + name:String, + ts:Timestamp, + value:Double + ) + case class MetricLabel( + metric_id:Long, + name:String, + value:String + ) +} + + +private[metric] class JdbcMetricRepository( + connection: JdbcConnection, + val profile: JdbcProfile, + commitTable: String = "flowman_metric_commits", + commitLabelTable: String = "flowman_metric_commit_labels", + metricTable: String = "flowman_metrics", + metricLabelTable: String = "flowman_metric_labels" +) { + private val logger = LoggerFactory.getLogger(getClass) + + import profile.api._ + + private lazy val db = { + val url = connection.url + val driver = connection.driver + val user = connection.username + val password = connection.password + val props = new Properties() + connection.properties.foreach(kv => props.setProperty(kv._1, kv._2)) + logger.debug(s"Connecting via JDBC to $url with driver $driver") + val executor = slick.util.AsyncExecutor( + name="Flowman.jdbc_metric_sink", + minThreads = 20, + maxThreads = 20, + queueSize = 1000, + maxConnections = 20) + Database.forURL(url, driver=driver, user=user.orNull, password=password.orNull, prop=props, executor=executor) + } + + class Commits(tag: Tag) extends Table[Commit](tag, commitTable) { + def id = column[Long]("id", O.PrimaryKey, O.AutoInc) + def ts = column[Timestamp]("ts") + + def * = (id, ts) <> (Commit.tupled, Commit.unapply) + } + class CommitLabels(tag: Tag) extends Table[CommitLabel](tag, commitLabelTable) { + def commit_id = column[Long]("commit_id") + def name = column[String]("name", O.Length(64)) + def value = column[String]("value", O.Length(64)) + + def pk = primaryKey(commitLabelTable + "_pk", (commit_id, name)) + def commit = foreignKey(commitLabelTable + "_fk", commit_id, commits)(_.id, onUpdate=ForeignKeyAction.Restrict, onDelete=ForeignKeyAction.Cascade) + def idx = index(commitLabelTable + "_idx", (name, value), unique = false) + + def * = (commit_id, name, value) <> (CommitLabel.tupled, CommitLabel.unapply) + } + class Metrics(tag: Tag) extends Table[Measurement](tag, metricTable) { + def id = column[Long]("id", O.PrimaryKey, O.AutoInc) + def commit_id = column[Long]("commit_id") + def name = column[String]("name", O.Length(64)) + def ts = column[Timestamp]("ts") + def value = column[Double]("value") + + def commit = foreignKey(metricTable + "_fk", commit_id, commits)(_.id, onUpdate=ForeignKeyAction.Restrict, onDelete=ForeignKeyAction.Cascade) + + def * = (id, commit_id, name, ts, value) <> (Measurement.tupled, Measurement.unapply) + } + class MetricLabels(tag: Tag) extends Table[MetricLabel](tag, metricLabelTable) { + def metric_id = column[Long]("metric_id") + def name = column[String]("name", O.Length(64)) + def value = column[String]("value", O.Length(64)) + + def pk = primaryKey(metricLabelTable + "_pk", (metric_id, name)) + def metric = foreignKey(metricLabelTable + "_fk", metric_id, metrics)(_.id, onUpdate=ForeignKeyAction.Restrict, onDelete=ForeignKeyAction.Cascade) + def idx = index(metricLabelTable + "_idx", (name, value), unique = false) + + def * = (metric_id, name, value) <> (MetricLabel.tupled, MetricLabel.unapply) + } + + val commits = TableQuery[Commits] + val commitLabels = TableQuery[CommitLabels] + val metrics = TableQuery[Metrics] + val metricLabels = TableQuery[MetricLabels] + + + def create() : Unit = { + import scala.concurrent.ExecutionContext.Implicits.global + val tables = Seq( + commits, + commitLabels, + metrics, + metricLabels + ) + + try { + val existing = db.run(profile.defaultTables) + val query = existing.flatMap(v => { + val names = v.map(mt => mt.name.name.toLowerCase(Locale.ROOT)) + val createIfNotExist = tables + .filter(table => !names.contains(table.baseTableRow.tableName.toLowerCase(Locale.ROOT))) + .map(_.schema.create) + db.run(DBIO.sequence(createIfNotExist)) + }) + Await.result(query, Duration.Inf) + } + catch { + case NonFatal(ex) => logger.error("Cannot connect to JDBC metric database to create tables", ex) + } + } + + def commit(metrics:Seq[GaugeMetric], labels:Map[String,String]) : Unit = { + val ts = Timestamp.from(Instant.now()) + + val cmQuery = (commits returning commits.map(_.id) into((jm, id) => jm.copy(id=id))) += Commit(0, ts) + val commit = Await.result(db.run(cmQuery), Duration.Inf) + val lbls = labels.map(l => CommitLabel(commit.id, l._1, l._2)) + val clQuery = commitLabels ++= lbls + Await.result(db.run(clQuery), Duration.Inf) + + metrics.foreach { m => + val metrics = this.metrics + val mtQuery = (metrics returning metrics.map(_.id) into((jm, id) => jm.copy(id=id))) += Measurement(0, commit.id, m.name, ts, m.value) + val metric = Await.result(db.run(mtQuery), Duration.Inf) + + val lbls = m.labels.map(l => MetricLabel(metric.id, l._1, l._2)) + val mlQuery = metricLabels ++= lbls + Await.result(db.run(mlQuery), Duration.Inf) + } + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala new file mode 100644 index 000000000..3f4b83cad --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala @@ -0,0 +1,136 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.metric + +import java.sql.SQLRecoverableException +import java.sql.SQLTransientException + +import scala.util.control.NonFatal + +import com.fasterxml.jackson.annotation.JsonProperty +import org.slf4j.LoggerFactory + +import com.dimajix.flowman.execution.Context +import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.jdbc.JdbcUtils +import com.dimajix.flowman.metric.AbstractMetricSink +import com.dimajix.flowman.metric.GaugeMetric +import com.dimajix.flowman.metric.MetricBoard +import com.dimajix.flowman.metric.MetricSink +import com.dimajix.flowman.model.Connection +import com.dimajix.flowman.model.Reference +import com.dimajix.flowman.spec.connection.ConnectionReferenceSpec +import com.dimajix.flowman.spec.connection.JdbcConnection + + +class JdbcMetricSink( + connection: Reference[Connection], + labels: Map[String,String] = Map(), + commitTable: String = "flowman_metric_commits", + commitLabelTable: String = "flowman_metric_commit_labels", + metricTable: String = "flowman_metrics", + metricLabelTable: String = "flowman_metric_labels" +) extends AbstractMetricSink { + private val logger = LoggerFactory.getLogger(getClass) + private val retries:Int = 3 + private val timeout:Int = 1000 + + override def commit(board:MetricBoard, status:Status): Unit = { + logger.info(s"Committing execution metrics to JDBC at '${jdbcConnection.url}'") + val rawLabels = this.labels + val labels = rawLabels.map { case(k,v) => k -> board.context.evaluate(v, Map("status" -> status.toString)) } + + val metrics = board.metrics(catalog(board), status).collect { + case metric:GaugeMetric => metric + } + try { + withRepository { session => + session.commit(metrics, labels) + } + } + catch { + case NonFatal(ex) => + logger.warn(s"Cannot publishing metrics to JDBC sink at ${jdbcConnection.url}: ${ex.getMessage}") + } + } + + /** + * Performs some a task with a JDBC session, also automatically performing retries and timeouts + * + * @param query + * @tparam T + * @return + */ + private def withRepository[T](query: JdbcMetricRepository => T) : T = { + def retry[T](n:Int)(fn: => T) : T = { + try { + fn + } catch { + case e @(_:SQLRecoverableException|_:SQLTransientException) if n > 1 => { + logger.error("Retrying after error while executing SQL: {}", e.getMessage) + Thread.sleep(timeout) + retry(n - 1)(fn) + } + } + } + + retry(retries) { + ensureTables() + query(repository) + } + } + + private lazy val jdbcConnection = connection.value.asInstanceOf[JdbcConnection] + private lazy val repository = new JdbcMetricRepository( + jdbcConnection, + JdbcUtils.getProfile(jdbcConnection.driver), + commitTable, + commitLabelTable, + metricTable, + metricLabelTable + ) + + private var tablesCreated:Boolean = false + private def ensureTables() : Unit = { + // Create Database if not exists + if (!tablesCreated) { + repository.create() + tablesCreated = true + } + } +} + + +class JdbcMetricSinkSpec extends MetricSinkSpec { + @JsonProperty(value = "connection", required = true) private var connection:ConnectionReferenceSpec = _ + @JsonProperty(value = "labels", required = false) private var labels:Map[String,String] = Map.empty + @JsonProperty(value = "commitTable", required = false) private var commitTable:String = "flowman_metric_commits" + @JsonProperty(value = "commitLabelTable", required = false) private var commitLabelTable:String = "flowman_metric_commit_labels" + @JsonProperty(value = "metricTable", required = false) private var metricTable:String = "flowman_metrics" + @JsonProperty(value = "metricLabelTable", required = false) private var metricLabelTable:String = "flowman_metric_labels" + + override def instantiate(context: Context): MetricSink = { + new JdbcMetricSink( + connection.instantiate(context), + labels, + context.evaluate(commitTable), + context.evaluate(commitLabelTable), + context.evaluate(metricTable), + context.evaluate(metricLabelTable) + ) + } +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSinkSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSinkSpec.scala index 3735cf67d..73772f272 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSinkSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/MetricSinkSpec.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ object MetricSinkSpec extends TypeRegistry[MetricSinkSpec] { @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") @JsonSubTypes(value = Array( new JsonSubTypes.Type(name = "console", value = classOf[ConsoleMetricSinkSpec]), + new JsonSubTypes.Type(name = "jdbc", value = classOf[JdbcMetricSinkSpec]), new JsonSubTypes.Type(name = "null", value = classOf[NullMetricSinkSpec]), new JsonSubTypes.Type(name = "prometheus", value = classOf[PrometheusMetricSinkSpec]) )) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala index 44e3e6e9c..da7587455 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala @@ -53,7 +53,7 @@ class PrometheusMetricSink( val labels = rawLabels.map(l => l._1 -> board.context.evaluate(l._2, Map("status" -> status.toString))) val path = labels.map(kv => kv._1 + "/" + kv._2).mkString("/") val url = new URI(this.url).resolve("/metrics/" + path) - logger.info(s"Publishing all metrics to Prometheus at $url") + logger.info(s"Publishing all metrics to Prometheus at '$url'") /* # TYPE some_metric counter @@ -95,9 +95,9 @@ class PrometheusMetricSink( } catch { case ex:HttpResponseException => - logger.warn(s"Got error response ${ex.getStatusCode} from Prometheus at $url: ${ex.toString}. Payload was:\n$payload") + logger.warn(s"Got error response ${ex.getStatusCode} from Prometheus at '$url': ${ex.getMessage}. Payload was:\n$payload") case NonFatal(ex) => - logger.warn(s"Cannot publishing metrics to Prometheus at $url: ${ex.toString}") + logger.warn(s"Cannot publishing metrics to Prometheus at '$url': ${ex.getMessage}") } finally { httpClient.close() diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/history/JdbcStateStoreTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/history/JdbcStateStoreTest.scala index 1226e4ba6..21fe5b63b 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/history/JdbcStateStoreTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/history/JdbcStateStoreTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,4 +61,17 @@ class JdbcStateStoreTest extends AnyFlatSpec with Matchers with BeforeAndAfter { val monitor = ObjectMapper.parse[HistorySpec](spec) monitor shouldBe a[JdbcHistorySpec] } + + it should "be parseable with embedded connection" in { + val spec = + """ + |kind: jdbc + |connection: + | kind: jdbc + | url: some_url + """.stripMargin + + val monitor = ObjectMapper.parse[HistorySpec](spec) + monitor shouldBe a[JdbcHistorySpec] + } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala new file mode 100644 index 000000000..28067bf5b --- /dev/null +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala @@ -0,0 +1,133 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.metric + +import java.nio.file.Files +import java.nio.file.Path + +import org.scalamock.scalatest.MockFactory +import org.scalatest.BeforeAndAfter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import com.dimajix.flowman.execution.RootContext +import com.dimajix.flowman.execution.Status +import com.dimajix.flowman.metric.FixedGaugeMetric +import com.dimajix.flowman.metric.MetricBoard +import com.dimajix.flowman.metric.MetricSelection +import com.dimajix.flowman.metric.MetricSystem +import com.dimajix.flowman.metric.Selector +import com.dimajix.flowman.model.Connection +import com.dimajix.flowman.model.ConnectionReference +import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype +import com.dimajix.flowman.spec.ObjectMapper +import com.dimajix.flowman.spec.connection.JdbcConnection + + +class JdbcMetricSinkTest extends AnyFlatSpec with Matchers with MockFactory with BeforeAndAfter { + var tempDir:Path = _ + + before { + tempDir = Files.createTempDirectory("jdbc_metric_test") + } + after { + tempDir.toFile.listFiles().foreach(_.delete()) + tempDir.toFile.delete() + } + + "The JdbcMetricSink" should "be parsable" in { + val spec = + """ + |kind: jdbc + |connection: metrics + """.stripMargin + + val monitor = ObjectMapper.parse[MetricSinkSpec](spec) + monitor shouldBe a[JdbcMetricSinkSpec] + } + + it should "be parsable with an embedded connection" in { + val spec = + """ + |kind: jdbc + |connection: + | kind: jdbc + | url: some_url + """.stripMargin + + val monitor = ObjectMapper.parse[MetricSinkSpec](spec) + monitor shouldBe a[JdbcMetricSinkSpec] + } + + it should "work" in { + val db = tempDir.resolve("mydb") + val project = Project("prj1") + val context = RootContext.builder().build().getProjectContext(project) + + val connection = JdbcConnection( + Connection.Properties(context), + url = "jdbc:derby:" + db + ";create=true", + driver = "org.apache.derby.jdbc.EmbeddedDriver" + ) + val connectionPrototype = mock[Prototype[Connection]] + (connectionPrototype.instantiate _).expects(context).returns(connection) + + val sink = new JdbcMetricSink( + ConnectionReference.apply(context, connectionPrototype), + Map("project" -> s"${project.name}") + ) + + val metricSystem = new MetricSystem + val metricBoard = MetricBoard(context, + Map("board_label" -> "v1"), + Seq(MetricSelection(selector=Selector(".*"), labels=Map("target" -> "$target", "status" -> "$status"))) + ) + + metricSystem.addMetric(FixedGaugeMetric("metric1", labels=Map("target" -> "p1", "metric_label" -> "v2"), 23.0)) + + sink.addBoard(metricBoard, metricSystem) + sink.commit(metricBoard, Status.SUCCESS) + sink.commit(metricBoard, Status.SUCCESS) + } + + it should "not throw on non-existing database" in { + val db = tempDir.resolve("mydb2") + val context = RootContext.builder().build() + + val connection = JdbcConnection( + Connection.Properties(context), + url = "jdbc:derby:" + db + ";create=false", + driver = "org.apache.derby.jdbc.EmbeddedDriver" + ) + val connectionPrototype = mock[Prototype[Connection]] + (connectionPrototype.instantiate _).expects(context).returns(connection) + + val sink = new JdbcMetricSink( + ConnectionReference.apply(context, connectionPrototype) + ) + + val metricSystem = new MetricSystem + val metricBoard = MetricBoard(context, + Map("board_label" -> "v1"), + Seq(MetricSelection(selector=Selector(".*"), labels=Map("target" -> "$target", "status" -> "$status"))) + ) + + sink.addBoard(metricBoard, metricSystem) + noException should be thrownBy(sink.commit(metricBoard, Status.SUCCESS)) + } +} From 59014831ed602a724968f166be5b909ae36742f4 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 23 Feb 2022 13:42:40 +0100 Subject: [PATCH 72/95] Fix propagation of JDBC connection properties to Slick --- .../com/dimajix/flowman/history/JdbcStateRepository.scala | 7 ++++--- .../dimajix/flowman/spec/metric/JdbcMetricRepository.scala | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala index 2e2a017f7..cea5e17b0 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala @@ -154,10 +154,10 @@ private[history] class JdbcStateRepository(connection: JdbcStateStore.Connection private lazy val db = { val url = connection.url val driver = connection.driver - val user = connection.user - val password = connection.password val props = new Properties() connection.properties.foreach(kv => props.setProperty(kv._1, kv._2)) + connection.user.foreach(props.setProperty("user", _)) + connection.password.foreach(props.setProperty("password", _)) logger.debug(s"Connecting via JDBC to $url with driver $driver") val executor = slick.util.AsyncExecutor( name="Flowman.default", @@ -165,7 +165,8 @@ private[history] class JdbcStateRepository(connection: JdbcStateStore.Connection maxThreads = 20, queueSize = 1000, maxConnections = 20) - Database.forURL(url, driver=driver, user=user.orNull, password=password.orNull, prop=props, executor=executor) + // Do not set username and password, since a bug in Slick would discard all other connection properties + Database.forURL(url, driver=driver, prop=props, executor=executor) } val jobRuns = TableQuery[JobRuns] diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala index e9ba5dec2..c8e5d1e4b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala @@ -78,10 +78,10 @@ private[metric] class JdbcMetricRepository( private lazy val db = { val url = connection.url val driver = connection.driver - val user = connection.username - val password = connection.password val props = new Properties() connection.properties.foreach(kv => props.setProperty(kv._1, kv._2)) + connection.username.foreach(props.setProperty("user", _)) + connection.password.foreach(props.setProperty("password", _)) logger.debug(s"Connecting via JDBC to $url with driver $driver") val executor = slick.util.AsyncExecutor( name="Flowman.jdbc_metric_sink", @@ -89,7 +89,8 @@ private[metric] class JdbcMetricRepository( maxThreads = 20, queueSize = 1000, maxConnections = 20) - Database.forURL(url, driver=driver, user=user.orNull, password=password.orNull, prop=props, executor=executor) + // Do not set username and password, since a bug in Slick would discard all other connection properties + Database.forURL(url, driver=driver, prop=props, executor=executor) } class Commits(tag: Tag) extends Table[Commit](tag, commitTable) { From 45614a8bd429997146039c16dbb28ed278391e58 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 23 Feb 2022 17:21:27 +0100 Subject: [PATCH 73/95] Minor improvements --- .../scala/com/dimajix/flowman/documentation/ColumnTest.scala | 2 +- .../scala/com/dimajix/flowman/documentation/SchemaTest.scala | 2 +- .../scala/com/dimajix/flowman/metric/ConsoleMetricSink.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 84535c84f..284184f6c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -118,7 +118,7 @@ final case class ForeignKeyColumnTest( column: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { - override def name : String = s"FOREIGN KEY (${column.getOrElse("")}) REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")}" + override def name : String = s"FOREIGN KEY REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")} (${column.getOrElse("")})" override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): ForeignKeyColumnTest = { val ref = ColumnTestReference(Some(parent)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala index f84fb416b..74183b6a7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -58,7 +58,7 @@ final case class PrimaryKeySchemaTest( columns:Seq[String] = Seq.empty, result:Option[TestResult] = None ) extends SchemaTest { - override def name : String = s"PRIMARY KEY(${columns.mkString(",")})" + override def name : String = s"PRIMARY KEY (${columns.mkString(",")})" override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) override def reparent(parent: Reference): PrimaryKeySchemaTest = { val ref = SchemaTestReference(Some(parent)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/ConsoleMetricSink.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/ConsoleMetricSink.scala index f90819a21..43fc30c25 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/ConsoleMetricSink.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/ConsoleMetricSink.scala @@ -1,5 +1,5 @@ /* - * Copyright 2019 Kaya Kupferschmidt + * Copyright 2019-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import com.dimajix.flowman.execution.Status class ConsoleMetricSink extends AbstractMetricSink { override def commit(board:MetricBoard, status:Status): Unit = { println("Collected metrics") - board.metrics(catalog(board), status).foreach{ metric => + board.metrics(catalog(board), status).sortBy(_.name).foreach{ metric => val name = metric.name val labels = metric.labels.map(kv => kv._1 + "=" + kv._2) metric match { From 879c0e0254a4da843ca2db49e6d9506f42885b5b Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 23 Feb 2022 21:03:46 +0100 Subject: [PATCH 74/95] Implement schema cache for relations --- .../com/dimajix/common/SynchronizedMap.scala | 10 ++++ .../documentation/RelationCollector.scala | 2 +- .../flowman/execution/CachingExecution.scala | 50 ++++++++++++++++--- .../dimajix/flowman/execution/Execution.scala | 10 ++++ .../flowman/execution/MonitorExecution.scala | 10 ++++ .../flowman/execution/exceptions.scala | 2 + .../spec/relation/DeltaFileRelation.scala | 3 ++ .../flowman/spec/relation/DeltaRelation.scala | 1 + .../spec/relation/DeltaTableRelation.scala | 4 +- .../spec/dataset/RelationDataset.scala | 4 +- .../spec/mapping/ReadRelationMapping.scala | 2 +- .../spec/mapping/ReadStreamMapping.scala | 2 +- .../spec/metric/JdbcMetricRepository.scala | 2 +- .../flowman/spec/relation/FileRelation.scala | 6 ++- .../spec/relation/HiveTableRelation.scala | 6 +++ .../spec/relation/HiveViewRelation.scala | 3 ++ .../flowman/spec/relation/JdbcRelation.scala | 3 ++ .../flowman/spec/relation/LocalRelation.scala | 4 ++ .../flowman/spec/schema/RelationSchema.scala | 2 +- .../tools/exec/model/DescribeCommand.scala | 2 +- 20 files changed, 112 insertions(+), 16 deletions(-) diff --git a/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala b/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala index b63510137..d7ec4e51a 100644 --- a/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala +++ b/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala @@ -91,6 +91,16 @@ case class SynchronizedMap[K,V](impl:mutable.Map[K,V]) { } } + /** + * Remove a value from the map + * @param key + */ + def remove(key: K) : Unit = { + synchronized { + impl.remove(key) + } + } + /** Retrieves the value which is associated with the given key. This * method invokes the `default` method of the map if there is no mapping * from the given key to a value. Unless overridden, the `default` method throws a diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala index 72ae39102..72a81d982 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/RelationCollector.scala @@ -128,7 +128,7 @@ class RelationCollector extends Collector { } val mergedSchema = { Try { - SchemaDoc.ofStruct(ref, relation.describe(execution, partitions)) + SchemaDoc.ofStruct(ref, execution.describe(relation, partitions)) } match { case Success(desc) => Some(desc.merge(schema)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala index 1e6bf0c73..d5269a83f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala @@ -39,7 +39,9 @@ import com.dimajix.flowman.common.ThreadUtils import com.dimajix.flowman.config.FlowmanConf import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingOutputIdentifier +import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.StructType @@ -74,15 +76,24 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte } } - private val schemaCache:SynchronizedMap[Mapping,TrieMap[String,StructType]] = { + private val mappingSchemaCache:SynchronizedMap[Mapping,TrieMap[String,StructType]] = { parent match { case Some(ce:CachingExecution) if !isolated => - ce.schemaCache + ce.mappingSchemaCache case _ => SynchronizedMap(IdentityHashMap[Mapping,TrieMap[String,StructType]]()) } } + private val relationSchemaCache:SynchronizedMap[Relation,StructType] = { + parent match { + case Some(ce:CachingExecution) if !isolated => + ce.relationSchemaCache + case _ => + SynchronizedMap(IdentityHashMap[Relation,StructType]()) + } + } + private val resources:mutable.ListBuffer[(ResourceIdentifier,() => Unit)] = { parent match { case Some(ce: CachingExecution) if !isolated => @@ -129,11 +140,11 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte * @return */ override def describe(mapping:Mapping, output:String) : StructType = { - schemaCache.getOrElseUpdate(mapping, TrieMap()) - .getOrElseUpdate(output, createSchema(mapping, output)) + mappingSchemaCache.getOrElseUpdate(mapping, TrieMap()) + .getOrElseUpdate(output, describeMapping(mapping, output)) } - private def createSchema(mapping:Mapping, output:String) : StructType = { + private def describeMapping(mapping:Mapping, output:String) : StructType = { if (!mapping.outputs.contains(output)) throw new NoSuchMappingOutputException(mapping.identifier, output) val context = mapping.context @@ -166,6 +177,26 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte } } + /** + * Returns the schema for a specific relation + * @param relation + * @param partitions + * @return + */ + override def describe(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = { + relationSchemaCache.getOrElseUpdate(relation, describeRelation(relation, partitions)) + } + + private def describeRelation(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = { + try { + logger.info(s"Describing relation '${relation.identifier}'") + relation.describe(this, partitions) + } + catch { + case NonFatal(e) => throw new DescribeRelationFailedException(relation.identifier, e) + } + } + /** * Registers a refresh function associated with a [[ResourceIdentifier]] * @param key @@ -186,6 +217,12 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte resources.filter(kv => kv._1.contains(key) || key.contains(kv._1)).foreach(_._2()) } parent.foreach(_.refreshResource(key)) + + // Invalidate schema caches + relationSchemaCache.toSeq + .map(_._1) + .filter(_.provides.exists(_.contains(key))) + .foreach(relationSchemaCache.impl.remove) } /** @@ -201,7 +238,8 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte if (!sharedCache) { frameCache.values.foreach(_.values.foreach(_.unpersist(true))) frameCache.clear() - schemaCache.clear() + mappingSchemaCache.clear() + relationSchemaCache.clear() resources.clear() } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Execution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Execution.scala index 75189efa9..079bb9041 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/Execution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/Execution.scala @@ -35,9 +35,11 @@ import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.Measure import com.dimajix.flowman.model.MeasureResult +import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetResult +import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.StructType @@ -186,6 +188,14 @@ abstract class Execution { mapping.describe(this, deps) } + /** + * Returns the schema for a specific relation + * @param relation + * @param partitions + * @return + */ + def describe(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType + /** * Registers a refresh function associated with a [[ResourceIdentifier]] * @param key diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/MonitorExecution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/MonitorExecution.scala index 5a7dcfb5f..736281f33 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/MonitorExecution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/MonitorExecution.scala @@ -25,7 +25,9 @@ import com.dimajix.flowman.hadoop.FileSystem import com.dimajix.flowman.metric.MetricBoard import com.dimajix.flowman.metric.MetricSystem import com.dimajix.flowman.model.Mapping +import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.ResourceIdentifier +import com.dimajix.flowman.types.FieldValue import com.dimajix.flowman.types.StructType @@ -98,6 +100,14 @@ final class MonitorExecution(parent:Execution, override val listeners:Seq[(Execu */ override def describe(mapping: Mapping, output: String): StructType = parent.describe(mapping, output) + /** + * Returns the schema for a specific relation + * @param relation + * @param partitions + * @return + */ + override def describe(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = parent.describe(relation, partitions) + override def addResource(key:ResourceIdentifier)(refresh: => Unit) : Unit = parent.addResource(key)(refresh) override def refreshResource(key:ResourceIdentifier) : Unit = parent.refreshResource(key) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/exceptions.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/exceptions.scala index be67e6b43..39a132a6d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/exceptions.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/exceptions.scala @@ -65,6 +65,8 @@ class DescribeMappingFailedException(val mapping:MappingIdentifier, cause:Throwa extends ExecutionException(s"Describing mapping $mapping failed", cause) class InstantiateMappingFailedException(val mapping:MappingIdentifier, cause:Throwable = None.orNull) extends ExecutionException(s"Instantiating mapping $mapping failed", cause) +class DescribeRelationFailedException(val relation:RelationIdentifier, cause:Throwable = None.orNull) + extends ExecutionException(s"Describing relation $relation failed", cause) class ValidationFailedException(val target:TargetIdentifier, cause:Throwable = None.orNull) extends ExecutionException(s"Validation of target $target failed", cause) diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala index 6778570e7..ab6af4da7 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala @@ -288,6 +288,8 @@ case class DeltaFileRelation( properties, description ) + + provides.foreach(execution.refreshResource) } } @@ -344,6 +346,7 @@ case class DeltaFileRelation( else { logger.info(s"Destroying Delta file relation '$identifier' by deleting directory '$location'") fs.delete(location, true) + provides.foreach(execution.refreshResource) } } diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala index 238d498f6..f80c834cf 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala @@ -129,6 +129,7 @@ abstract class DeltaRelation(options: Map[String,String], mergeKey: Seq[String]) if (requiresMigration) { doMigration(execution, table, sourceSchema, targetSchema, migrationPolicy, migrationStrategy) + provides.foreach(execution.refreshResource) } } private def doMigration(execution: Execution, table:DeltaTableV2, currentSchema:com.dimajix.flowman.types.StructType, targetSchema:com.dimajix.flowman.types.StructType, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala index 6ab742ba7..47bfa6f7d 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala @@ -296,6 +296,8 @@ case class DeltaTableRelation( properties, description ) + + provides.foreach(execution.refreshResource) } } @@ -336,6 +338,7 @@ case class DeltaTableRelation( if (!ifExists || catalog.tableExists(tableIdentifier)) { logger.info(s"Destroying Delta table relation '$identifier' by dropping table $tableIdentifier") catalog.dropTable(tableIdentifier) + provides.foreach(execution.refreshResource) } } @@ -387,7 +390,6 @@ case class DeltaTableRelation( } - @RelationType(kind="deltaTable") class DeltaTableRelationSpec extends RelationSpec with SchemaRelationSpec with PartitionedRelationSpec { @JsonProperty(value = "database", required = false) private var database: String = "default" diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala index 786ffc11f..211a93eee 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/dataset/RelationDataset.scala @@ -119,8 +119,8 @@ case class RelationDataset( * @return */ override def describe(execution:Execution) : Option[StructType] = { - val instance = relation.value - Some(instance.describe(execution, partition)) + val schema = execution.describe(relation.value, partition) + Some(schema) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala index 8b7c3193b..5cf5e3dce 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadRelationMapping.scala @@ -109,7 +109,7 @@ case class ReadRelationMapping( } else { val relation = this.relation.value - relation.describe(execution, partitions) + execution.describe(relation, partitions) } // Apply documentation diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala index 852265a36..9068ed7f3 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadStreamMapping.scala @@ -102,7 +102,7 @@ case class ReadStreamMapping ( } else { val relation = this.relation.value - relation.describe(execution) + execution.describe(relation) } // Apply documentation diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala index c8e5d1e4b..3e2508829 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala @@ -160,7 +160,7 @@ private[metric] class JdbcMetricRepository( Await.result(query, Duration.Inf) } catch { - case NonFatal(ex) => logger.error("Cannot connect to JDBC metric database to create tables", ex) + case NonFatal(ex) => logger.error(s"Cannot connect to JDBC metric database to create tables: ${ex.getMessage}") } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala index c4e2a812c..c68b818f6 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/FileRelation.scala @@ -225,7 +225,7 @@ case class FileRelation( else doWriteStaticPartitions(execution, df, partition, mode) - execution.refreshResource(ResourceIdentifier.ofFile(qualifiedLocation)) + provides.foreach(execution.refreshResource) } private def doWriteDynamicPartitions(execution:Execution, df:DataFrame, mode:OutputMode) : Unit = { val outputPath = qualifiedLocation @@ -412,6 +412,8 @@ case class FileRelation( throw new FileSystemException(qualifiedLocation.toString, "", "Cannot create directory.") } } + + provides.foreach(execution.refreshResource) } /** @@ -469,6 +471,8 @@ case class FileRelation( val fs = collector.fs fs.delete(qualifiedLocation, true) } + + provides.foreach(execution.refreshResource) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala index fc9bd6cf1..c955d65c2 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala @@ -168,6 +168,8 @@ case class HiveTableRelation( writeSpark(execution, df, partitionSpec, mode) else throw new IllegalArgumentException("Hive relations only support write modes 'hive' and 'spark'") + + provides.foreach(execution.refreshResource) } /** @@ -486,6 +488,7 @@ case class HiveTableRelation( // Create table val catalog = execution.catalog catalog.createTable(catalogTable, false) + provides.foreach(execution.refreshResource) } } @@ -501,6 +504,7 @@ case class HiveTableRelation( if (!ifExists || catalog.tableExists(tableIdentifier)) { logger.info(s"Destroying Hive table relation '$identifier' by dropping table $tableIdentifier") catalog.dropTable(tableIdentifier) + provides.foreach(execution.refreshResource) } } @@ -525,6 +529,7 @@ case class HiveTableRelation( logger.warn(s"TABLE target $tableIdentifier is currently a VIEW, dropping...") catalog.dropView(tableIdentifier, false) create(execution, false) + provides.foreach(execution.refreshResource) } } else { @@ -540,6 +545,7 @@ case class HiveTableRelation( val requiresMigration = TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) if (requiresMigration) { doMigration(execution, sourceSchema, targetSchema, migrationPolicy, migrationStrategy) + provides.foreach(execution.refreshResource) } } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala index d96ef6c17..b023815b5 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala @@ -164,6 +164,7 @@ case class HiveViewRelation( if (!ifNotExists || !catalog.tableExists(tableIdentifier)) { logger.info(s"Creating Hive view relation '$identifier' with VIEW $tableIdentifier") catalog.createView(tableIdentifier, select, ifNotExists) + provides.foreach(execution.refreshResource) } } @@ -184,6 +185,7 @@ case class HiveViewRelation( else { migrateFromTable(catalog, newSelect, migrationStrategy) } + provides.foreach(execution.refreshResource) } } @@ -228,6 +230,7 @@ case class HiveViewRelation( if (!ifExists || catalog.tableExists(tableIdentifier)) { logger.info(s"Destroying Hive view relation '$identifier' with VIEW $tableIdentifier") catalog.dropView(tableIdentifier) + provides.foreach(execution.refreshResource) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index bb192ca27..97e707d42 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -420,6 +420,7 @@ class JdbcRelationBase( withConnection{ (con,options) => if (!ifNotExists || !JdbcUtils.tableExists(con, tableIdentifier, options)) { doCreate(con, options) + provides.foreach(execution.refreshResource) } } } @@ -453,6 +454,7 @@ class JdbcRelationBase( withConnection{ (con,options) => if (!ifExists || JdbcUtils.tableExists(con, tableIdentifier, options)) { JdbcUtils.dropTable(con, tableIdentifier, options) + provides.foreach(execution.refreshResource) } } } @@ -469,6 +471,7 @@ class JdbcRelationBase( val currentSchema = JdbcUtils.getSchema(con, tableIdentifier, options) if (TableChange.requiresMigration(currentSchema, targetSchema, migrationPolicy)) { doMigration(currentSchema, targetSchema, migrationPolicy, migrationStrategy) + provides.foreach(execution.refreshResource) } } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/LocalRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/LocalRelation.scala index 135c72ac5..24421c9b7 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/LocalRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/LocalRelation.scala @@ -176,6 +176,8 @@ extends BaseRelation with SchemaRelation with PartitionedRelation { writer.format(format) .mode(mode.batchMode) .save(outputFile) + + provides.foreach(execution.refreshResource) } /** @@ -276,6 +278,7 @@ extends BaseRelation with SchemaRelation with PartitionedRelation { else { logger.info(s"Creating local directory '$localDirectory' for local file relation") path.mkdirs() + provides.foreach(execution.refreshResource) } } @@ -312,6 +315,7 @@ extends BaseRelation with SchemaRelation with PartitionedRelation { } delete(root) + provides.foreach(execution.refreshResource) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/RelationSchema.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/RelationSchema.scala index 3d04da6d3..77f7536b5 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/RelationSchema.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/schema/RelationSchema.scala @@ -45,7 +45,7 @@ case class RelationSchema( case Some(schema) => schema.fields ++ rel.partitions.map(_.field) case None => val execution = context.execution - rel.describe(execution).fields + execution.describe(rel).fields } } private lazy val cachedDescription = { diff --git a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala index ac5119892..35c3e3445 100644 --- a/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala +++ b/flowman-tools/src/main/scala/com/dimajix/flowman/tools/exec/model/DescribeCommand.scala @@ -55,7 +55,7 @@ class DescribeCommand extends Command { } else { val execution = session.execution - val schema = relation.describe(execution, partition) + val schema = execution.describe(relation, partition) schema.printTree() } Status.SUCCESS From e73cf26ea526f8e4a9aebc6365060ec0cb37bd2c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Wed, 23 Feb 2022 21:55:21 +0100 Subject: [PATCH 75/95] Minor improvements --- CHANGELOG.md | 1 + .../dimajix/flowman/execution/CachingExecution.scala | 5 +++++ .../dimajix/flowman/execution/ExecutionListener.scala | 10 ++++++++++ .../dimajix/flowman/history/JdbcStateRepository.scala | 2 +- .../main/scala/com/dimajix/flowman/model/Hook.scala | 1 + .../flowman/spec/dataset/RelationDatasetTest.scala | 1 + 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 888d1680e..fd64da640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Add new `drop` target for removing tables * Speed up project loading by reusing Jackson mapper * Implement new `jdbc` metric sink +* Implement schema cache in Executor to speed up documentation and similar tasks # Version 0.21.1 - 2022-01-28 diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala index d5269a83f..b80d404a3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala @@ -190,6 +190,11 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte private def describeRelation(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = { try { logger.info(s"Describing relation '${relation.identifier}'") + listeners.foreach { l => + Try { + l._1.describeRelation(this, relation, l._2) + } + } relation.describe(this, partitions) } catch { diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ExecutionListener.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ExecutionListener.scala index f9c90d907..2c3f6ba78 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/ExecutionListener.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/ExecutionListener.scala @@ -26,6 +26,7 @@ import com.dimajix.flowman.model.LifecycleResult import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Measure import com.dimajix.flowman.model.MeasureResult +import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetDigest import com.dimajix.flowman.model.TargetResult @@ -126,6 +127,14 @@ trait ExecutionListener { * @param parent */ def describeMapping(execution: Execution, mapping:Mapping, parent:Option[Token]) : Unit + + /** + * Informs the listener that a specific relation is about to be described + * @param execution + * @param relation + * @param parent + */ + def describeRelation(execution: Execution, relation:Relation, parent:Option[Token]) : Unit } @@ -142,4 +151,5 @@ abstract class AbstractExecutionListener extends ExecutionListener { override def finishMeasure(execution:Execution, token: MeasureToken, result: MeasureResult): Unit = {} override def instantiateMapping(execution: Execution, mapping:Mapping, parent:Option[Token]) : Unit = {} override def describeMapping(execution: Execution, mapping:Mapping, parent:Option[Token]) : Unit = {} + override def describeRelation(execution: Execution, relation:Relation, parent:Option[Token]) : Unit = {} } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala index cea5e17b0..c30d0c42f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala @@ -382,7 +382,7 @@ private[history] class JdbcStateRepository(connection: JdbcStateStore.Connection Await.result(query, Duration.Inf) } catch { - case NonFatal(ex) => logger.error("Cannot connect to JDBC history database", ex) + case NonFatal(ex) => logger.error(s"Cannot create tables of JDBC history database: ${ex.getMessage}") } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Hook.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Hook.scala index de254697d..3ad1a2855 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Hook.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Hook.scala @@ -147,4 +147,5 @@ abstract class BaseHook extends AbstractInstance with Hook { override def finishMeasure(execution:Execution, token:MeasureToken, result:MeasureResult) : Unit = {} override def instantiateMapping(execution: Execution, mapping:Mapping, parent:Option[Token]) : Unit = {} override def describeMapping(execution: Execution, mapping:Mapping, parent:Option[Token]) : Unit = {} + override def describeRelation(execution: Execution, relation:Relation, parent:Option[Token]) : Unit = {} } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala index 757129f16..734aeb7b3 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/dataset/RelationDatasetTest.scala @@ -108,6 +108,7 @@ class RelationDatasetTest extends AnyFlatSpec with Matchers with MockFactory wit (relation.write _).expects(executor,spark.emptyDataFrame,*,OutputMode.APPEND).returns(Unit) dataset.write(executor, spark.emptyDataFrame, OutputMode.APPEND) + (relation.identifier _).expects().returns(RelationIdentifier("relation")) (relation.describe _).expects(executor, *).returns(new StructType()) dataset.describe(executor) should be (Some(new StructType())) } From c06481d53ccd396146972e56f8878e0bb3c576aa Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 08:41:44 +0100 Subject: [PATCH 76/95] Add config variables to disable schema cache --- .../com/dimajix/common/SynchronizedMap.scala | 14 ++++++++++++-- .../dimajix/flowman/config/FlowmanConf.scala | 8 ++++++++ .../flowman/execution/CachingExecution.scala | 18 +++++++++++++++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala b/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala index d7ec4e51a..733e0b31c 100644 --- a/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala +++ b/flowman-common/src/main/scala/com/dimajix/common/SynchronizedMap.scala @@ -157,11 +157,21 @@ case class SynchronizedMap[K,V](impl:mutable.Map[K,V]) { toSeq.iterator } + /** Collects all keys of this map in an Set. + * + * @return the keys of this map as a Set. + */ + def keys : Set[K] = { + synchronized { + impl.keySet.toSet + } + } + /** Collects all values of this map in an iterable collection. * - * @return the values of this map as an iterable. + * @return the values of this map as a Sequence. */ - def values: Iterable[V] = { + def values: Seq[V] = { synchronized { Seq(impl.values.toSeq:_*) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala b/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala index b0c49a130..ba2ad3dd3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala @@ -101,6 +101,14 @@ object FlowmanConf { .doc("Parallelism of mapping instantiation") .intConf .createWithDefault(1) + val EXECUTION_MAPPING_SCHEMA_CACHE = buildConf("flowman.execution.mapping.schemaCache") + .doc("Cache schema information of mapping instances") + .booleanConf + .createWithDefault(true) + val EXECUTION_RELATION_SCHEMA_CACHE = buildConf("flowman.execution.relation.schemaCache") + .doc("Cache schema information of relation instances") + .booleanConf + .createWithDefault(true) val DEFAULT_RELATION_MIGRATION_POLICY = buildConf("flowman.default.relation.migrationPolicy") .doc("Default migration policy. Allowed values are 'relaxed' and 'strict'") diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala index b80d404a3..85e3154e8 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/CachingExecution.scala @@ -57,6 +57,8 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte } } private lazy val parallelism = flowmanConf.getConf(FlowmanConf.EXECUTION_MAPPING_PARALLELISM) + private lazy val useMappingSchemaCache = flowmanConf.getConf(FlowmanConf.EXECUTION_MAPPING_SCHEMA_CACHE) + private lazy val useRelationSchemaCache = flowmanConf.getConf(FlowmanConf.EXECUTION_RELATION_SCHEMA_CACHE) private val frameCache:SynchronizedMap[Mapping,Map[String,DataFrame]] = { parent match { @@ -140,8 +142,13 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte * @return */ override def describe(mapping:Mapping, output:String) : StructType = { - mappingSchemaCache.getOrElseUpdate(mapping, TrieMap()) - .getOrElseUpdate(output, describeMapping(mapping, output)) + if (useMappingSchemaCache) { + mappingSchemaCache.getOrElseUpdate(mapping, TrieMap()) + .getOrElseUpdate(output, describeMapping(mapping, output)) + } + else { + describeMapping(mapping, output) + } } private def describeMapping(mapping:Mapping, output:String) : StructType = { @@ -184,7 +191,12 @@ abstract class CachingExecution(parent:Option[Execution], isolated:Boolean) exte * @return */ override def describe(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = { - relationSchemaCache.getOrElseUpdate(relation, describeRelation(relation, partitions)) + if (useRelationSchemaCache) { + relationSchemaCache.getOrElseUpdate(relation, describeRelation(relation, partitions)) + } + else { + describeRelation(relation, partitions) + } } private def describeRelation(relation:Relation, partitions:Map[String,FieldValue] = Map()) : StructType = { From 2cb301517f59f7261bfb4386ded7422ed00fb740 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 08:41:50 +0100 Subject: [PATCH 77/95] Minor code improvements --- docs/config.md | 10 ++++++++++ .../flowman/documentation/ColumnTest.scala | 6 +++++- .../flowman/documentation/SchemaTest.scala | 6 +++++- .../com/dimajix/flowman/execution/migration.scala | 4 +--- .../dimajix/flowman/history/JdbcStateStore.scala | 14 ++------------ .../com/dimajix/flowman/history/StateStore.scala | 1 + .../com/dimajix/flowman/metric/MetricSystem.scala | 14 +++++++++++++- .../flowman/transforms/SchemaEnforcer.scala | 6 +++--- .../flowman/spec/metric/JdbcMetricSink.scala | 15 ++++----------- .../spec/metric/PrometheusMetricSink.scala | 7 ++++--- .../flowman/spec/metric/JdbcMetricSinkTest.scala | 5 +++-- 11 files changed, 51 insertions(+), 37 deletions(-) diff --git a/docs/config.md b/docs/config.md index 671aba603..40eb6a09c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -52,6 +52,16 @@ The number of mappings to be processed in parallel. Increasing this number may h relations are read from and their initial setup is slow (for example due to slow directory listings). With the default value of 1, the parallelism is completely disabled and a non-threaded code path is used instead. +- `flowman.execution.mapping.schemaCache` *(type: boolean)* *(default: true)* (since Flowman 0.22.0) +Turn on/off caching of schema information of mappings. Caching this information (which is enabled per default) can + speed up schema inference, which is used for `mapping` schemas and when creating the documentation of mappings. Turning + off the cache is mainly for debugging purposes. + +- `flowman.execution.relation.schemaCache` *(type: boolean)* *(default: true)* (since Flowman 0.22.0) +Turn on/off caching of schema information of relations. Caching this information (which is enabled per default) can + speed up schema inference, which is used for `relation` schemas and when creating the documentation of relations and. + mappings. Turning off the cache is mainly for debugging purposes. + - `flowman.execution.scheduler.class` *(type: class)* *(default: `com.dimajix.flowman.execution.DependencyScheduler`)* (since Flowman 0.16.0) Configure the scheduler to use, which essentially decides which target to build next. - The default `DependencyScheduler` will sort all targets according to their dependency. diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala index 284184f6c..441e1870a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala @@ -118,7 +118,11 @@ final case class ForeignKeyColumnTest( column: Option[String] = None, result:Option[TestResult] = None ) extends ColumnTest { - override def name : String = s"FOREIGN KEY REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")} (${column.getOrElse("")})" + override def name : String = { + val otherEntity = relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("") + val otherColumn = column.getOrElse("") + s"FOREIGN KEY REFERENCES ${otherEntity} (${otherColumn})" + } override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) override def reparent(parent: Reference): ForeignKeyColumnTest = { val ref = ColumnTestReference(Some(parent)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala index 74183b6a7..89d0af15b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala @@ -75,7 +75,11 @@ final case class ForeignKeySchemaTest( references: Seq[String] = Seq.empty, result:Option[TestResult] = None ) extends SchemaTest { - override def name : String = s"FOREIGN KEY (${columns.mkString(",")}) REFERENCES ${relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("")}(${references.mkString(",")})" + override def name : String = { + val otherEntity = relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("") + val otherColumns = if (references.isEmpty) columns else references + s"FOREIGN KEY (${columns.mkString(",")}) REFERENCES ${otherEntity}(${otherColumns.mkString(",")})" + } override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) override def reparent(parent: Reference): ForeignKeySchemaTest = { val ref = SchemaTestReference(Some(parent)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/migration.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/migration.scala index 4e8d94c96..78fc459c9 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/migration.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/migration.scala @@ -20,7 +20,6 @@ import java.util.Locale sealed abstract class MigrationPolicy extends Product with Serializable - object MigrationPolicy { case object RELAXED extends MigrationPolicy case object STRICT extends MigrationPolicy @@ -37,7 +36,6 @@ object MigrationPolicy { sealed abstract class MigrationStrategy extends Product with Serializable - object MigrationStrategy { case object NEVER extends MigrationStrategy case object FAIL extends MigrationStrategy @@ -50,7 +48,7 @@ object MigrationStrategy { case "never" => MigrationStrategy.NEVER case "fail" => MigrationStrategy.FAIL case "alter" => MigrationStrategy.ALTER - case "alter_replace" => MigrationStrategy.ALTER_REPLACE + case "alter_replace"|"alterreplace" => MigrationStrategy.ALTER_REPLACE case "replace" => MigrationStrategy.REPLACE case _ => throw new IllegalArgumentException(s"Unknown migration strategy: '$mode'. " + "Accepted migration strategy are 'NEVER', 'FAIL', 'ALTER', 'ALTER_REPLACE' and 'REPLACE'.") diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala index b063d8518..8a37b8f31 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateStore.scala @@ -21,18 +21,10 @@ import java.sql.SQLRecoverableException import java.sql.SQLTransientException import java.sql.Timestamp import java.time.Clock -import java.time.ZoneId import javax.xml.bind.DatatypeConverter import org.slf4j.LoggerFactory -import slick.jdbc.DerbyProfile -import slick.jdbc.H2Profile -import slick.jdbc.MySQLProfile -import slick.jdbc.PostgresProfile -import slick.jdbc.SQLServerProfile -import slick.jdbc.SQLiteProfile - -import com.dimajix.flowman.execution.Phase + import com.dimajix.flowman.execution.Status import com.dimajix.flowman.graph.GraphBuilder import com.dimajix.flowman.history.JdbcStateRepository.JobRun @@ -40,8 +32,6 @@ import com.dimajix.flowman.history.JdbcStateRepository.TargetRun import com.dimajix.flowman.history.JdbcStateStore.JdbcJobToken import com.dimajix.flowman.history.JdbcStateStore.JdbcTargetToken import com.dimajix.flowman.jdbc.JdbcUtils -import com.dimajix.flowman.metric.GaugeMetric -import com.dimajix.flowman.metric.Metric import com.dimajix.flowman.model.Job import com.dimajix.flowman.model.JobDigest import com.dimajix.flowman.model.JobResult @@ -369,7 +359,7 @@ case class JdbcStateStore(connection:JdbcStateStore.Connection, retries:Int=3, t fn } catch { case e @(_:SQLRecoverableException|_:SQLTransientException) if n > 1 => { - logger.error("Retrying after error while executing SQL: {}", e.getMessage) + logger.warn("Retrying after error while executing SQL: {}", e.getMessage) Thread.sleep(timeout) retry(n - 1)(fn) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/StateStore.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/StateStore.scala index a91257454..74de90524 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/StateStore.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/StateStore.scala @@ -34,6 +34,7 @@ import com.dimajix.flowman.model.TargetResult abstract class JobToken abstract class TargetToken + abstract class StateStore { /** * Returns the state of a job, or None if no information is available diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala index 55847fe68..70518d529 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/metric/MetricSystem.scala @@ -16,8 +16,11 @@ package com.dimajix.flowman.metric +import scala.util.control.NonFatal import scala.util.matching.Regex +import org.slf4j.LoggerFactory + import com.dimajix.common.IdentityHashSet import com.dimajix.common.SynchronizedSet import com.dimajix.flowman.execution.Status @@ -55,6 +58,7 @@ trait MetricCatalog { class MetricSystem extends MetricCatalog { + private val logger = LoggerFactory.getLogger(getClass) private val metricBundles : SynchronizedSet[MetricBundle] = SynchronizedSet(IdentityHashSet()) private val metricBoards : SynchronizedSet[MetricBoard] = SynchronizedSet(IdentityHashSet()) private val metricSinks : SynchronizedSet[MetricSink] = SynchronizedSet(IdentityHashSet()) @@ -134,7 +138,15 @@ class MetricSystem extends MetricCatalog { def commitBoard(board:MetricBoard, status:Status) : Unit = { if (!metricBoards.contains(board)) throw new IllegalArgumentException("MetricBoard not registered") - metricSinks.foreach(_.commit(board, status)) + + metricSinks.foreach { sink => + try { + sink.commit(board, status) + } + catch { + case NonFatal(ex) => logger.warn(s"Error while committing metrics to sink: ${ex.getMessage}") + } + } } /** diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/SchemaEnforcer.scala b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/SchemaEnforcer.scala index 9a906b83d..7f105db1c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/transforms/SchemaEnforcer.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/transforms/SchemaEnforcer.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ import com.dimajix.flowman.util.SchemaUtils.coerce import com.dimajix.spark.sql.functions.nullable_struct -sealed abstract class ColumnMismatchStrategy +sealed abstract class ColumnMismatchStrategy extends Product with Serializable object ColumnMismatchStrategy { case object IGNORE extends ColumnMismatchStrategy case object ERROR extends ColumnMismatchStrategy @@ -65,7 +65,7 @@ object ColumnMismatchStrategy { } -sealed abstract class TypeMismatchStrategy +sealed abstract class TypeMismatchStrategy extends Product with Serializable object TypeMismatchStrategy { case object IGNORE extends TypeMismatchStrategy case object ERROR extends TypeMismatchStrategy diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala index 3f4b83cad..0ed4bbf6c 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricSink.scala @@ -19,8 +19,6 @@ package com.dimajix.flowman.spec.metric import java.sql.SQLRecoverableException import java.sql.SQLTransientException -import scala.util.control.NonFatal - import com.fasterxml.jackson.annotation.JsonProperty import org.slf4j.LoggerFactory @@ -57,14 +55,9 @@ class JdbcMetricSink( val metrics = board.metrics(catalog(board), status).collect { case metric:GaugeMetric => metric } - try { - withRepository { session => - session.commit(metrics, labels) - } - } - catch { - case NonFatal(ex) => - logger.warn(s"Cannot publishing metrics to JDBC sink at ${jdbcConnection.url}: ${ex.getMessage}") + + withRepository { session => + session.commit(metrics, labels) } } @@ -81,7 +74,7 @@ class JdbcMetricSink( fn } catch { case e @(_:SQLRecoverableException|_:SQLTransientException) if n > 1 => { - logger.error("Retrying after error while executing SQL: {}", e.getMessage) + logger.warn("Retrying after error while executing SQL: {}", e.getMessage) Thread.sleep(timeout) retry(n - 1)(fn) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala index da7587455..49d093ac5 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/PrometheusMetricSink.scala @@ -42,7 +42,7 @@ class PrometheusMetricSink( url:String, labels:Map[String,String] ) - extends AbstractMetricSink { +extends AbstractMetricSink { private val logger = LoggerFactory.getLogger(classOf[PrometheusMetricSink]) override def commit(board:MetricBoard, status:Status) : Unit = { @@ -53,7 +53,8 @@ class PrometheusMetricSink( val labels = rawLabels.map(l => l._1 -> board.context.evaluate(l._2, Map("status" -> status.toString))) val path = labels.map(kv => kv._1 + "/" + kv._2).mkString("/") val url = new URI(this.url).resolve("/metrics/" + path) - logger.info(s"Publishing all metrics to Prometheus at '$url'") + + logger.info(s"Committing metrics to Prometheus at '$url'") /* # TYPE some_metric counter @@ -97,7 +98,7 @@ class PrometheusMetricSink( case ex:HttpResponseException => logger.warn(s"Got error response ${ex.getStatusCode} from Prometheus at '$url': ${ex.getMessage}. Payload was:\n$payload") case NonFatal(ex) => - logger.warn(s"Cannot publishing metrics to Prometheus at '$url': ${ex.getMessage}") + logger.warn(s"Error while publishing metrics to Prometheus at '$url': ${ex.getMessage}") } finally { httpClient.close() diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala index 28067bf5b..416cf4c90 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/metric/JdbcMetricSinkTest.scala @@ -105,7 +105,7 @@ class JdbcMetricSinkTest extends AnyFlatSpec with Matchers with MockFactory with sink.commit(metricBoard, Status.SUCCESS) } - it should "not throw on non-existing database" in { + it should "throw on non-existing database" in { val db = tempDir.resolve("mydb2") val context = RootContext.builder().build() @@ -128,6 +128,7 @@ class JdbcMetricSinkTest extends AnyFlatSpec with Matchers with MockFactory with ) sink.addBoard(metricBoard, metricSystem) - noException should be thrownBy(sink.commit(metricBoard, Status.SUCCESS)) + an[Exception] should be thrownBy(sink.commit(metricBoard, Status.SUCCESS)) + an[Exception] should be thrownBy(sink.commit(metricBoard, Status.SUCCESS)) } } From 4c00d5d9ed1683dc58e116e8c02b0a72e64c3f8f Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 10:41:52 +0100 Subject: [PATCH 78/95] Support new verify policiy in 'reltion' target --- CHANGELOG.md | 2 + docs/config.md | 8 ++ .../dimajix/flowman/config/FlowmanConf.scala | 5 + .../com/dimajix/flowman/model/Target.scala | 21 +++ .../com/dimajix/flowman/model/result.scala | 8 ++ .../flowman/spec/target/RelationTarget.scala | 52 ++++++-- .../spec/target/RelationTargetTest.scala | 123 +++++++++++++++++- 7 files changed, 210 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd64da640..db6a1cb7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * Speed up project loading by reusing Jackson mapper * Implement new `jdbc` metric sink * Implement schema cache in Executor to speed up documentation and similar tasks +* Add new config variables `flowman.execution.mapping.schemaCache` and `flowman.execution.relation.schemaCache` +* Add new config variable `flowman.default.target.verifyPolicy` to ignore empty tables during VERIFY phase # Version 0.21.1 - 2022-01-28 diff --git a/docs/config.md b/docs/config.md index 40eb6a09c..bceb77af1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -109,6 +109,14 @@ Sets the strategy to use how tables should be migrated. Possible values are: actual defined columns. Per default Flowman will add/remove columns to/from records such that they match the current physical layout. See [relations](spec/relation/index.md) for possible options and more details. +- `flowman.default.target.verifyPolicy` *(type: string)* *(default:`EMPTY_AS_FAILURE`)* (since Flowman 0.22.0) +Defines the default target policy that is used during the `VERIFY` execution phase. The setting controls how Flowman +interprets an empty table. Normally you'd expect that all target tables contain records, but this might not always +be the case, for example when the source tables are already empty. Possible values are + - *`EMPTY_AS_FAILURE`*: Flowman will report an empty target table as an error in the `VERIFY` phase. + - *`EMPTY_AS_SUCCESS`*: Flowman will ignore empty tables, but still check for existence in the `VERIFY` phase. + - *`EMPTY_AS_SUCCESS_WITH_ERRORS`*: An empty output table is handled as partially successful. + - `flowman.default.target.outputMode` *(type: string)* *(default:`OVERWRITE`)* Sets the default target output mode. Possible values are - *`OVERWRITE`*: Will overwrite existing data. Only supported in batch output. diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala b/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala index ba2ad3dd3..0ced47940 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/config/FlowmanConf.scala @@ -31,6 +31,7 @@ import com.dimajix.flowman.execution.OutputMode import com.dimajix.flowman.execution.SimpleExecutor import com.dimajix.flowman.execution.DependencyScheduler import com.dimajix.flowman.execution.Scheduler +import com.dimajix.flowman.model.VerifyPolicy import com.dimajix.flowman.transforms.ColumnMismatchStrategy import com.dimajix.flowman.transforms.TypeMismatchStrategy import com.dimajix.spark.features @@ -136,6 +137,10 @@ object FlowmanConf { .stringConf .createWithDefault(TypeMismatchStrategy.CAST_ALWAYS.toString) + val DEFAULT_TARGET_VERIFY_POLICY = buildConf("flowman.default.target.verifyPolicy") + .doc("Policy for verifying a target. Accepted verify policies are 'empty_as_success', 'empty_as_failure' and 'empty_as_success_with_errors'.") + .stringConf + .createWithDefault(VerifyPolicy.EMPTY_AS_FAILURE.toString) val DEFAULT_TARGET_OUTPUT_MODE = buildConf("flowman.default.target.outputMode") .doc("Default output mode of targets") .stringConf diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala index 0afe1c299..afa0d3b62 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/Target.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.model +import java.util.Locale + import org.apache.spark.sql.DataFrame import com.dimajix.common.Trilean @@ -29,6 +31,25 @@ import com.dimajix.flowman.metric.LongAccumulatorMetric import com.dimajix.flowman.metric.Selector import com.dimajix.spark.sql.functions.count_records + +sealed abstract class VerifyPolicy extends Product with Serializable +object VerifyPolicy { + case object EMPTY_AS_SUCCESS extends VerifyPolicy + case object EMPTY_AS_FAILURE extends VerifyPolicy + case object EMPTY_AS_SUCCESS_WITH_ERRORS extends VerifyPolicy + + def ofString(mode:String) : VerifyPolicy = { + mode.toLowerCase(Locale.ROOT) match { + case "empty_as_success" => VerifyPolicy.EMPTY_AS_SUCCESS + case "empty_as_failure" => VerifyPolicy.EMPTY_AS_FAILURE + case "empty_as_success_with_errors" => VerifyPolicy.EMPTY_AS_SUCCESS_WITH_ERRORS + case _ => throw new IllegalArgumentException(s"Unknown verify policy: '$mode'. " + + "Accepted verify policies are 'empty_as_success', 'empty_as_failure' and 'empty_as_success_with_errors'.") + } + } +} + + /** * * @param namespace diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/result.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/result.scala index b76fd0f0d..cfb0fde7f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/result.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/result.scala @@ -302,6 +302,14 @@ final case class TargetResult( override def category : Category = target.category override def kind : String = target.kind override def description: Option[String] = None + + def withoutTime : TargetResult = { + val ts = Instant.ofEpochSecond(0) + copy( + startTime=ts, + endTime=ts + ) + } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala index 38be75e19..8fd744dac 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/target/RelationTarget.scala @@ -16,6 +16,12 @@ package com.dimajix.flowman.spec.target +import java.time.Instant + +import scala.util.Failure +import scala.util.Success +import scala.util.Try + import com.fasterxml.jackson.annotation.JsonProperty import org.slf4j.LoggerFactory @@ -23,6 +29,7 @@ import com.dimajix.common.No import com.dimajix.common.Trilean import com.dimajix.common.Unknown import com.dimajix.common.Yes +import com.dimajix.flowman.config.FlowmanConf import com.dimajix.flowman.config.FlowmanConf.DEFAULT_RELATION_MIGRATION_POLICY import com.dimajix.flowman.config.FlowmanConf.DEFAULT_RELATION_MIGRATION_STRATEGY import com.dimajix.flowman.config.FlowmanConf.DEFAULT_TARGET_OUTPUT_MODE @@ -35,6 +42,7 @@ import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.MigrationStrategy import com.dimajix.flowman.execution.OutputMode import com.dimajix.flowman.execution.Phase +import com.dimajix.flowman.execution.Status import com.dimajix.flowman.execution.VerificationFailedException import com.dimajix.flowman.graph.Linker import com.dimajix.flowman.model.BaseTarget @@ -46,6 +54,8 @@ import com.dimajix.flowman.model.RelationReference import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetDigest +import com.dimajix.flowman.model.TargetResult +import com.dimajix.flowman.model.VerifyPolicy import com.dimajix.flowman.spec.relation.IdentifierRelationReferenceSpec import com.dimajix.flowman.spec.relation.RelationReferenceSpec import com.dimajix.flowman.types.SingleValue @@ -266,16 +276,42 @@ case class RelationTarget( /** * Performs a verification of the build step or possibly other checks. * - * @param executor + * @param execution */ - override def verify(executor: Execution) : Unit = { - require(executor != null) + override def verify2(execution: Execution) : TargetResult = { + require(execution != null) - val partition = this.partition.mapValues(v => SingleValue(v)) - val rel = relation.value - if (rel.loaded(executor, partition) == No) { - logger.error(s"Verification of target '$identifier' failed - partition $partition of relation '${relation.identifier}' does not exist") - throw new VerificationFailedException(identifier) + val startTime = Instant.now() + Try { + val partition = this.partition.mapValues(v => SingleValue(v)) + val rel = relation.value + if (rel.loaded(execution, partition) == No) { + val policy = VerifyPolicy.ofString(execution.flowmanConf.getConf(FlowmanConf.DEFAULT_TARGET_VERIFY_POLICY)) + policy match { + case VerifyPolicy.EMPTY_AS_FAILURE => + logger.error(s"Verification of target '$identifier' failed - partition $partition of relation '${relation.identifier}' does not exist") + throw new VerificationFailedException(identifier) + case VerifyPolicy.EMPTY_AS_SUCCESS|VerifyPolicy.EMPTY_AS_SUCCESS_WITH_ERRORS => + if (rel.exists(execution) != No) { + logger.warn(s"Verification of target '$identifier' failed - partition $partition of relation '${relation.identifier}' does not exist. Ignoring.") + if (policy == VerifyPolicy.EMPTY_AS_SUCCESS_WITH_ERRORS) + Status.SUCCESS_WITH_ERRORS + else + Status.SUCCESS + } + else { + logger.error(s"Verification of target '$identifier' failed - relation '${relation.identifier}' does not exist") + throw new VerificationFailedException(identifier) + } + } + } + else { + Status.SUCCESS + } + } + match { + case Success(status) => TargetResult(this, Phase.VERIFY, status, startTime) + case Failure(ex) => TargetResult(this, Phase.VERIFY, ex, startTime) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala index 7ab4415ce..88939e9ff 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/RelationTargetTest.scala @@ -23,6 +23,7 @@ import java.util.UUID import org.apache.hadoop.fs.Path import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType +import org.scalamock.scalatest.MockFactory import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -32,17 +33,20 @@ import com.dimajix.common.Yes import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.execution.Status import com.dimajix.flowman.metric.GaugeMetric import com.dimajix.flowman.metric.Selector import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.Module import com.dimajix.flowman.model.Project +import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Target import com.dimajix.flowman.model.TargetIdentifier +import com.dimajix.flowman.model.TargetResult import com.dimajix.flowman.spec.ObjectMapper import com.dimajix.flowman.spec.dataset.DatasetSpec import com.dimajix.flowman.spec.dataset.RelationDatasetSpec @@ -51,7 +55,7 @@ import com.dimajix.flowman.spec.relation.NullRelation import com.dimajix.spark.testing.LocalSparkSession -class RelationTargetTest extends AnyFlatSpec with Matchers with LocalSparkSession { +class RelationTargetTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { "The RelationTarget" should "support embedded relations" in { val spec = """ @@ -315,4 +319,121 @@ class RelationTargetTest extends AnyFlatSpec with Matchers with LocalSparkSessio target.execute(executor, Phase.BUILD) metric.value should be (4) } + + it should "behave correctly with VerifyPolicy=EMPTY_AS_FAILURE" in { + val relationGen = mock[Prototype[Relation]] + val relation = mock[Relation] + val project = Project( + name = "test", + relations = Map("relation" -> relationGen) + ) + + val session = Session.builder() + .withSparkSession(spark) + .withProject(project) + .withConfig("flowman.default.target.verifyPolicy","empty_as_failure") + .build() + val executor = session.execution + val context = session.getContext(project) + + val target = RelationTarget( + context, + RelationIdentifier("relation"), + MappingOutputIdentifier("mapping") + ) + (relationGen.instantiate _).expects(context).returns(relation) + + (relation.loaded _).expects(*,*).returns(Yes) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(Unknown) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(No) + target.execute(executor, Phase.VERIFY).status should be(Status.FAILED) + } + + it should "behave correctly with VerifyPolicy=EMPTY_AS_SUCCESS" in { + val relationGen = mock[Prototype[Relation]] + val relation = mock[Relation] + val project = Project( + name = "test", + relations = Map("relation" -> relationGen) + ) + + val session = Session.builder() + .withSparkSession(spark) + .withProject(project) + .withConfig("flowman.default.target.verifyPolicy","EMPTY_AS_SUCCESS") + .build() + val executor = session.execution + val context = session.getContext(project) + + val target = RelationTarget( + context, + RelationIdentifier("relation"), + MappingOutputIdentifier("mapping") + ) + (relationGen.instantiate _).expects(context).returns(relation) + + (relation.loaded _).expects(*,*).returns(Yes) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(Unknown) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(Yes) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(Unknown) + target.execute(executor, Phase.VERIFY).status should be(Status.SUCCESS) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(No) + target.execute(executor, Phase.VERIFY).status should be(Status.FAILED) + } + + it should "behave correctly with VerifyPolicy=EMPTY_AS_SUCCESS_WITH_ERRORS" in { + val relationGen = mock[Prototype[Relation]] + val relation = mock[Relation] + val project = Project( + name = "test", + relations = Map("relation" -> relationGen) + ) + + val session = Session.builder() + .withSparkSession(spark) + .withProject(project) + .withConfig("flowman.default.target.verifyPolicy","EMPTY_AS_SUCCESS_WITH_ERRORS") + .build() + val executor = session.execution + val context = session.getContext(project) + + val target = RelationTarget( + context, + RelationIdentifier("relation"), + MappingOutputIdentifier("mapping") + ) + (relationGen.instantiate _).expects(context).returns(relation) + + (relation.loaded _).expects(*,*).returns(Yes) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(Unknown) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS).withoutTime) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(Yes) + target.execute(executor, Phase.VERIFY).withoutTime should be(TargetResult(target, Phase.VERIFY, Status.SUCCESS_WITH_ERRORS).withoutTime) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(Unknown) + target.execute(executor, Phase.VERIFY).status should be(Status.SUCCESS_WITH_ERRORS) + + (relation.loaded _).expects(*,*).returns(No) + (relation.exists _).expects(*).returns(No) + target.execute(executor, Phase.VERIFY).status should be(Status.FAILED) + } } From 6d3176a3094d49ac98caeb76582db71607c5df8f Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 13:20:15 +0100 Subject: [PATCH 79/95] Minor speed up for committing metrics to JDBC --- .../flowman/history/JdbcStateRepository.scala | 18 ++++++---- .../spec/metric/JdbcMetricRepository.scala | 33 +++++++++++-------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala index c30d0c42f..e69179a9b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/history/JdbcStateRepository.scala @@ -22,8 +22,8 @@ import java.time.ZonedDateTime import java.util.Locale import java.util.Properties -import scala.collection.mutable import scala.concurrent.Await +import scala.concurrent.Future import scala.concurrent.duration.Duration import scala.language.higherKinds import scala.util.control.NonFatal @@ -443,15 +443,19 @@ private[history] class JdbcStateRepository(connection: JdbcStateStore.Connection } def insertJobMetrics(jobId:Long, metrics:Seq[Measurement]) : Unit = { - metrics.foreach { m => + implicit val ec = db.executor.executionContext + + val result = metrics.map { m => val jobMetric = JobMetric(0, jobId, m.name, new Timestamp(m.ts.toInstant.toEpochMilli), m.value) val jmQuery = (jobMetrics returning jobMetrics.map(_.id) into((jm,id) => jm.copy(id=id))) += jobMetric - val jmResult = Await.result(db.run(jmQuery), Duration.Inf) - - val labels = m.labels.map(l => JobMetricLabel(jmResult.id, l._1, l._2)) - val mlQuery = jobMetricLabels ++= labels - Await.result(db.run(mlQuery), Duration.Inf) + db.run(jmQuery).flatMap { metric => + val labels = m.labels.map(l => JobMetricLabel(metric.id, l._1, l._2)) + val mlQuery = jobMetricLabels ++= labels + db.run(mlQuery) + } } + + Await.result(Future.sequence(result), Duration.Inf) } def getJobMetrics(jobId:Long) : Seq[Measurement] = { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala index 3e2508829..fff558cc0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/metric/JdbcMetricRepository.scala @@ -22,8 +22,10 @@ import java.util.Locale import java.util.Properties import scala.concurrent.Await +import scala.concurrent.Future import scala.concurrent.duration.Duration import scala.language.higherKinds +import scala.util.Success import scala.util.control.NonFatal import org.slf4j.LoggerFactory @@ -165,22 +167,27 @@ private[metric] class JdbcMetricRepository( } def commit(metrics:Seq[GaugeMetric], labels:Map[String,String]) : Unit = { + implicit val ec = db.executor.executionContext val ts = Timestamp.from(Instant.now()) val cmQuery = (commits returning commits.map(_.id) into((jm, id) => jm.copy(id=id))) += Commit(0, ts) - val commit = Await.result(db.run(cmQuery), Duration.Inf) - val lbls = labels.map(l => CommitLabel(commit.id, l._1, l._2)) - val clQuery = commitLabels ++= lbls - Await.result(db.run(clQuery), Duration.Inf) - - metrics.foreach { m => - val metrics = this.metrics - val mtQuery = (metrics returning metrics.map(_.id) into((jm, id) => jm.copy(id=id))) += Measurement(0, commit.id, m.name, ts, m.value) - val metric = Await.result(db.run(mtQuery), Duration.Inf) - - val lbls = m.labels.map(l => MetricLabel(metric.id, l._1, l._2)) - val mlQuery = metricLabels ++= lbls - Await.result(db.run(mlQuery), Duration.Inf) + val commit = db.run(cmQuery).flatMap { commit => + val lbls = labels.map(l => CommitLabel(commit.id, l._1, l._2)) + val clQuery = commitLabels ++= lbls + db.run(clQuery).flatMap(_ => Future.successful(commit)) } + + val result = commit.flatMap { commit => + Future.sequence(metrics.map { m => + val metrics = this.metrics + val mtQuery = (metrics returning metrics.map(_.id) into ((jm, id) => jm.copy(id = id))) += Measurement(0, commit.id, m.name, ts, m.value) + db.run(mtQuery).flatMap { metric => + val lbls = m.labels.map(l => MetricLabel(metric.id, l._1, l._2)) + val mlQuery = metricLabels ++= lbls + db.run(mlQuery) + } + }) + } + Await.result(result, Duration.Inf) } } From aa5bd11f6d38752677a181f022317214b2d9f7ea Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 15:53:48 +0100 Subject: [PATCH 80/95] Fix documentation FileGenerator to work with non-local default file systems --- .../dimajix/flowman/spec/documentation/FileGenerator.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala index 2b492fdc7..ccce6e754 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/FileGenerator.scala @@ -60,12 +60,7 @@ case class FileGenerator( props.load(new StringReader(loadResource("template.properties"))) val fs = execution.fs - - val uri = location.toUri - val outputDir = if (uri.getAuthority == null && uri.getScheme == null) - fs.local(location) - else - fs.file(location) + val outputDir = fs.file(location) // Cleanup any existing output directory if (outputDir.isDirectory()) { From 6b5e1f595b65d263d8d4842b6b195843a2e437c7 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 18:56:44 +0100 Subject: [PATCH 81/95] Update documentation --- BUILDING.md | 2 +- CHANGELOG.md | 5 ++ QUICKSTART.md | 23 ++++---- docs/cli/flowexec.md | 33 ++++++++++- docs/cli/flowshell.md | 4 +- docs/cookbook/docker.md | 21 +++++++ docs/cookbook/kerberos.md | 4 +- docs/cookbook/windows.md | 33 +++++++++++ docs/documenting/index.md | 2 + docs/images/flowman-documentation.png | Bin 0 -> 420057 bytes docs/installation.md | 77 +++++++++++++++++++++----- docs/quickstart.md | 22 +++----- docs/spec/index.md | 2 +- 13 files changed, 182 insertions(+), 46 deletions(-) create mode 100644 docs/cookbook/docker.md create mode 100644 docs/cookbook/windows.md create mode 100644 docs/images/flowman-documentation.png diff --git a/BUILDING.md b/BUILDING.md index 1257721df..30a3f4914 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -55,7 +55,7 @@ appropriate build profiles, you can easily create a custom build. Although you can normally build Flowman on Windows, it is recommended to use Linux instead. But nevertheless Windows is still supported to some extend, but requires some extra care. You will need the Hadoop WinUtils installed. You can download the binaries from https://github.com/cdarlint/winutils and install an appropriate version somewhere onto -your machine. Do not forget to set the HADOOP_HOME or PATH environment variable to the installation directory of these +your machine. Do not forget to set the `HADOOP_HOME` or `PATH` environment variable to the installation directory of these utils! You should also configure git such that all files are checked out using "LF" endings instead of "CRLF", otherwise diff --git a/CHANGELOG.md b/CHANGELOG.md index 0188ad7b8..30e5afc97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ * Add new config variable `flowman.default.target.verifyPolicy` to ignore empty tables during VERIFY phase +# Version 0.21.2 - 2022-02-14 + +* Fix importing projects + + # Version 0.21.1 - 2022-01-28 * flowexec now returns different exit codes depending on the processing result diff --git a/QUICKSTART.md b/QUICKSTART.md index 071de70f3..5f127d115 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -16,7 +16,7 @@ Fortunately, Apache Spark is rather simple to install locally on your machine: ### Download & Install Spark -As of this writing, the latest release of Flowman is 0.20.0 and is available prebuilt for Spark 3.1.2 on the Spark +As of this writing, the latest release of Flowman is 0.22.0 and is available prebuilt for Spark 3.2.1 on the Spark homepage. So we download the appropriate Spark distribution from the Apache archive and unpack it. ```shell @@ -25,8 +25,8 @@ mkdir playground cd playground # Download and unpack Spark & Hadoop -curl -L https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | tar xvzf -# Create a nice link -ln -snf spark-3.1.2-bin-hadoop3.2 spark +curl -L https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.2.1-bin-hadoop3.2.tgz | tar xvzf -# Create a nice link +ln -snf spark-3.2.1-bin-hadoop3.2 spark ``` The Spark package already contains Hadoop, so with this single download you already have both installed and integrated with each other. @@ -35,7 +35,8 @@ The Spark package already contains Hadoop, so with this single download you alre If you are trying to run the application on Windows, you also need the *Hadoop Winutils*, which is a set of DLLs required for the Hadoop libraries to be working. You can get a copy at https://github.com/kontext-tech/winutils . Once you downloaded the appropriate version, you need to place the DLLs into a directory `$HADOOP_HOME/bin`, where -`HADOOP_HOME` refers to some location on your Windows PC. You also need to set the following environment variables: +`HADOOP_HOME` refers to some arbitrary location of your choice on your Windows PC. You also need to set the following +environment variables: * `HADOOP_HOME` should point to the parent directory of the `bin` directory * `PATH` should also contain `$HADOOP_HOME/bin` @@ -43,11 +44,11 @@ Once you downloaded the appropriate version, you need to place the DLLs into a d ## 1.2 Install Flowman You find prebuilt Flowman packages on the corresponding release page on GitHub. For this quickstart, we chose -`flowman-dist-0.20.0-oss-spark3.1-hadoop3.2-bin.tar.gz` which nicely fits to the Spark package we just downloaded before. +`flowman-dist-0.22.0-oss-spark3.2-hadoop3.3-bin.tar.gz` which nicely fits to the Spark package we just downloaded before. ```shell # Download and unpack Flowman -curl -L https://github.com/dimajix/flowman/releases/download/0.20.0/flowman-dist-0.20.0-oss-spark3.1-hadoop3.2-bin.tar.gz | tar xvzf - +curl -L https://github.com/dimajix/flowman/releases/download/0.22.0/flowman-dist-0.22.0-oss-spark3.2-hadoop3.3-bin.tar.gz | tar xvzf - # Create a nice link ln -snf flowman-0.20.0 flowman @@ -81,13 +82,9 @@ That’s all we need to run the Flowman example. # 2. Flowman Shell -The example data is stored in a S3 bucket provided by myself. In order to access the data, you need to provide valid -AWS credentials in your environment: - -```shell -$ export AWS_ACCESS_KEY_ID= -$ export AWS_SECRET_ACCESS_KEY= -``` +The example data is stored in a S3 bucket provided by myself. Since the data is publicly available and the project is +configured to use anonymous AWS authentication, you do not need to provide your AWS credentials (you even do not +even need to have an account on AWS) ## 2.1 Start interactive Flowman shell diff --git a/docs/cli/flowexec.md b/docs/cli/flowexec.md index be9be66d5..193f5bb57 100644 --- a/docs/cli/flowexec.md +++ b/docs/cli/flowexec.md @@ -29,7 +29,7 @@ or for inspecting individual entities. ## Project Commands -The most important command group is for executing a specific lifecycle or a individual phase for the whole project. +The most important command group is for executing a specific lifecycle or an individual phase for the whole project. ```shell script flowexec project ``` @@ -49,6 +49,21 @@ individual targets with `-d`. the whole lifecycle for `verify` includes the phases `create` and `build` and these phases would be executed before `verify`. If this is not what you want, then use the option `-nl` +### Examples +In order to build a project (i.e. run `VALIDATE`, `CREATE` and `BUILD` execution phases) stored in the subdirectory +`examples/weather` which defines an (optional) parameter `year`, simply run + +```shell +flowexec -f examples/weather project build year=2018 +``` + +If you only want to execute the `BUILD` phase and skip the first two other phases, then you need to add the +command line option `-nl` to skip the lifecycle: + +```shell +flowexec -f examples/weather project build year=2018 -nl +``` + ## Job Commands Similar to the project commands, individual jobs with different names than `main` can be executed. @@ -79,6 +94,22 @@ This will execute the whole job by executing the desired lifecycle for the `main the whole lifecycle for `verify` includes the phases `create` and `build` and these phases would be executed before `verify`. If this is not what you want, then use the option `-nl` + +### Examples +In order to build (i.e. run `VALIDATE`, `CREATE` and `BUILD` execution phases) the `main` job of a project stored +in the subdirectory `examples/weather` which defines an (optional) parameter `year`, simply run + +```shell +flowexec -f examples/weather job build main year=2018 +``` + +If you only want to execute the `BUILD` phase and skip the first two other phases, then you need to add the +command line option `-nl` to skip the lifecycle: + +```shell +flowexec -f examples/weather job build main year=2018 -nl +``` + The following example will only execute the `BUILD` phase of the job `daily`, which defines a parameter `processing_datetime` with type datetiem. The job will be executed for the whole date range from 2021-06-01 until 2021-08-10 with a step size of one day. Flowman will execute up to four jobs in parallel (`-j 4`). diff --git a/docs/cli/flowshell.md b/docs/cli/flowshell.md index 1a54f288a..95501036c 100644 --- a/docs/cli/flowshell.md +++ b/docs/cli/flowshell.md @@ -36,7 +36,9 @@ Some additional commands in `flowshell` which are not available via `flowexec` a Start the Flowman shell for your project via - flowshell -f /path/to/your/project +```shell +flowshell -f /path/to/your/project +``` Now you can list all jobs via diff --git a/docs/cookbook/docker.md b/docs/cookbook/docker.md new file mode 100644 index 000000000..6b759d222 --- /dev/null +++ b/docs/cookbook/docker.md @@ -0,0 +1,21 @@ +# Running Flowman in Docker + +Flowman can also be run inside Docker, especially when working in local mode (i.e. without a cluster). It is also +possible to run Flowman in Docker in Spark distributed processing mode, but this requires more configuration options +to forward all required ports etc. + +## Running Locally + +We publish Flowman Docker images on [Docker Hub](https://hub.docker.com/repository/docker/dimajix/flowman), +which are good enough for local work. You can easily start a Flowman session in Docker as follows: + +```shell +docker run --rm -ti dimajix/flowman:0.21.0-oss-spark3.1-hadoop3.2 bash +``` + +Then once the Docker image has started you will be presented with a bash prompt. Then you can easily build the +weather example of Flowman via +```shell +cd /opt/flowman +flowexec -f examples/weather job build main +``` diff --git a/docs/cookbook/kerberos.md b/docs/cookbook/kerberos.md index ee031ffb6..8df93045e 100644 --- a/docs/cookbook/kerberos.md +++ b/docs/cookbook/kerberos.md @@ -1,6 +1,6 @@ -# Kerberos +# Using Kerberos Authentication -Of course you can also run Flowman in a Kerberos environment, as long as the components you use actually support +Of course, you can also run Flowman in a Kerberos environment, as long as the components you use actually support Kerberos. This includes Spark, Hadoop and Kafka. ## Configuring Kerberos diff --git a/docs/cookbook/windows.md b/docs/cookbook/windows.md new file mode 100644 index 000000000..6b8718db8 --- /dev/null +++ b/docs/cookbook/windows.md @@ -0,0 +1,33 @@ +# Running Flowman on Windows + +Flowman is best run on Linux, especially for production usage. Windows support is at best experimental and will +probably be never within the focus of the project. Nevertheless, there are also some options for running Flowman on +Windows, with the main purpose to provide developers some way to create and test projects on their local machines. + +The main difficulty in supporting Windows comes from two aspects +* Windows doesn't support `bash`, therefore all scripts have been rewritten to run on Windows +* Hadoop and Spark require some special *Hadoop WinUtils* libraries to be installed + + +## Installing using WinUtils +The first naturla option is to install Flowman directly on your Windows machine. Of course this also requires a +working Apache Spark installation. You can download an appropriate version from the +[Apache Spark homepage](https://spark.apache.org). + +Next, you are required to install the *Hadoop Winutils*, which is a set of DLLs required for the Hadoop libraries to +be working. You can get a copy at https://github.com/kontext-tech/winutils . +Once you downloaded the appropriate version, you need to place the DLLs into a directory `$HADOOP_HOME/bin`, where +`HADOOP_HOME` refers to some arbitrary location of your choice on your Windows PC. You also need to set the following +environment variables: +* `HADOOP_HOME` should point to the parent directory of the `bin` directory +* `PATH` should also contain `$HADOOP_HOME/bin` + + +## Using Docker +A simpler way to run Flowman on Windows is to use a Docker image available on +[Docker Hub](https://hub.docker.com/repository/docker/dimajix/flowman) + + +## Using WSL +And of course you can also simply install a Linux distro of your choice via WSL and then normally +[install Flowman](../installation.md) within WSL. diff --git a/docs/documenting/index.md b/docs/documenting/index.md index e8eabbbeb..7af0837df 100644 --- a/docs/documenting/index.md +++ b/docs/documenting/index.md @@ -12,6 +12,8 @@ which is useful for providing a documentation of the data model. * ``` +![Flowman Documentation](../images/flowman-documentation.png) + ### Providing Descriptions Although Flowman will generate many valuable documentation bits by inspecting the project, the most important entities diff --git a/docs/images/flowman-documentation.png b/docs/images/flowman-documentation.png new file mode 100644 index 0000000000000000000000000000000000000000..d1866bcb117a5d9471b77998b824645e68097ae2 GIT binary patch literal 420057 zcmeFZcUaTe)<5c~h>Cy>5D?HoK&dKCI*N+Y73o4yq(h?gk_1r@r7FD$h)6FHkQN{) zRYZu?&_i#5B=nGwKdxy-u&%J+~=eg&P;hAS9`@7d(Ywfi@Yp?RgKu?qP zD9_P-`}VQky>s*azJ1If;2)2}hk*aWUenCFZ{N9ncW+*Q;A6Ek>YbQol&VHUuSbvZ zW(Ym+&f;Qu@%wkti&cvspNDxGeJ&a;|KMz6(&gD_bniNohs0~suX@am*VPZJuy)7X z%}&6=(C|j~yJ=T_@ku$;p2g0^O<{(hxu3KW!h3fNtB_Fshmq>SGVwBK%lc+%xU<3j zgNKiwQ-8kix4-z8IALOC-%Q&QTcQbHl4UAUmi~?dWjuuVvc2b$gNI|a)k(=9E0X#6 zlcS*ajRNO~fAQsiZW^d{>Nx5sNLw+c0TWG<5#ECi;Mck5f14;EmVcTOhf2KBw_)%} zo|FIhWas~HKk2XI8up5ej3)m|{7bR^gZb*^EXR(qHBXjX+M2cewl>s$6@Q1hh*#^! z9p1Q1T57vu`xTcZ)hyX$-o^H-71xh25cz{pb;4I`Tr|8rHd*6syBjd!gY6us?Dheemc{BQ z1{rShNojA*`fY#T`CsyvPqP=}M2DXQBe;7Kq@k%p{imf{ef_RYN%S$;SUv7=;m8P= zbIp&n>8Dy3pdF+SvC_~YeaSk!T(v~<2)x6`#FGJ|3>NlN+lnwD{oc*v2HSR`(2fDr z`w3@cY+F&p6@RFEOF=h--S-9l;PinSLJA3ZDcl8s^0fpdsBVIEX5FQ1A+@87_{DeK zTGt*_s0IIF8HC^_s0#%l7NWa{5epKg!Hkqkf~8O!wL-qRT7=QM2r$|^<9-9i!%VtG zNZxvQQVcyc&-(dMzc#uqucn;OwSvgIFJhIWvS&olVmhbMtD z4|O3`Y!}oY2$M9(C<9YiRG}X7#F;*NG zJvHv;y-FIZ`zU>Pyx>vx>>UPI?YX+~{)je=+E$VvwBv_+Rk}YvAruvu|NV`$Wo75y zPT=6-rSJQ7Sv^a(mMZekf!T?pZM-(88p5^OAGhmIE7?YPw^t9Ck6%GuTaO4W#Wq3%r z{qDkY%|DOm4tmJ_sC%GGLgv|D=6^V<^5ZO0gn`Pae~3QFsRM~vk=Tfn$$Ia;yXQr- z2FrFuL8{?-h6^{D7Ndpm@(7h~wSo*_$K=PVpYxRfyJq~dK;b0AbAJ3?X%#rIsP2HqGs2YREh)WQjLg@; zd=u+*h2<_&zI1=S+r{}r*;JxNf&a%$`wxEluag){#K8K7w#JfbU6ys7R=$Df1TH)6 zo6l2EZz1kvmcJzPPmSVc5lJhjy4G)z-pxry~^(%{lv%v9`{LB*RX7owy)IG*s%2D zv%1^&3b8OP@5SEK=<*Qz`k$9~T=j59r_eMoaQUd_znbE9~OImdf7N7Yjv3@`N1yD&Ypqx9OVQYkkW* zo&BpZ_05-kdv6=ar}d${L~=86kaMGL%c`KR_a-SBd;=8ybyw^CC`_OG;1@g6_qGex zt;Zd}cV1%p9SVb|Wd}=>kAQlnFzL0*JL69B*52WuTLl;PzHhm9fxE6=!q=&)1eI*E zvgMA*Bg$HulMk+RZ`73(m(_cDwQ2pSe19eF-nU<4H(w1nx@qew%kK2L=2a6t+b0$M z4n1x|j?MEDk3D0h@|!d8is~FJGD7HM==AQhr_6cseK@LftjYl2GA-k6i7XcB_h#*# zo;5qo;8p}Wok$#sAMr}Jh>@s?e}xc}TfdF4_$=%HRw8_F27b88lpOQL zv-8vJokongl+sNS()JX_ldUOV7uMA$Z;GQh!T##te%Q8y$8r8r>qw;FAXrU)b;Xekk!^VZ;WVAmV%X z8r9#~2BzcS`a0X3_QCmth;;}%DD3imM#uwT8xvCyg6YizMuu1Yx%#8uCY&D$$}-Xv zz!4d!lg!`VVcPpT>`M7G=W!IL-2In(i)`0r{fDf7IPU*&>mQckKf3h~Tkvmt69Yrp zK^Sk=rcp7asW*|R`FXzAx+({{-tD;X&?wgHtZW=sSv(Bw*@IejLzrbaBrj^EgY`;A z*7Uu8BAQn`A4#Xut zd9bG`o!}_JS^d}5r&3z%ew_%ve`NyKGA)PG)s}1SXhW|?)Crot$Y)64lEZ#?5xLfh zwbs^0ZRn=yS1v-ar6GugvTyRL6_<*wYE}C^kTd2-LK@C8GL#R(zJA$kxdxAFdp0e3 z>SIQ6ZJDGqX{b;BOu|UDBG|H`y7uU8#^^hri7icD(hBx<-N89uYFP0HCu*!RUkv3x zoImhO#h{{@m(gR!zQCaqiM>O|5o#XBORD8r;$b{%#PWiG%kV0P4q~MxW4w6yiNsqt z%O(d!Kdhpqq>f1`d%{^0tqxrvbCf`BK>ctsqhJ$|N6a5wRM6~*JR-qrlYP~00*5RP ztQtqolsLK56D2y;8B3vRzjTXii-IQlO`dCn&5O&z6VBdrgPm5{_7NYx7zx?K46wv+ z`z;Q3^L{44@FNaRxxAjS_t-%p(L&H|a_bbz$TAak=D?qh|F1xav6HoEQXttkQD`LsvDH7cW{CcL-hJwy_#t#d2#AM>(4O9<%rCyaran2 zQb<>$kuOhra&^pe81Fp5AJ{k-6(!AYpx$`dI6F&&KqfXS`C2!}MB_?AZ2_LY7yV5#?v2Nu`xER*;Wj zbojR9Yc-$}Dmj&QL(a1k1r~@Td=$e-{rDNCz^*E4vLgwZU8Y7`UKGmw>|x1p6ZLZz zf#~<2-E|xszZdrCG-AZ)6_RjYEcvH!jv!;$6CVFHkD?c0!=>}1kfPIQqR(buA*J~H z-q`(|x^*m5DctW`8L=Gl^x~NNOlDX5P99_R{LszBgTqbk#CW~Qo<@sn0e6d9P&4%c zXZ+-f2#oPsYn_9gDt>xP3U#tEC0AKM2=iK1$$EQB;h63tj0M(mV|=15p#DTh%vIXIe)pl=t`#&9R9~b6deBQbnwX>pCH`P5)^EEpN#~9T4}+XNY$U=M2_vwC<2Wj^sIu#?ZS6ld$h`*y9}(a}TZW9dTmdkmWTSYo3rtxt znQ94BOLu22I+fj|27)>pAxdJ~Kr1U{kK2|5ItQEanrX*HY6t-z1qkiu6PGrqxF59JFJCnuCB}!_zw0Z`bl}utszZ3~}lfO?|W-`IM*( z!6cUUG$fYK1(x*uhK_8RjX$wvc8N6z(SXrX2%EjWCrfj00z7D$a7O9s9T*&qyr#)sHPuO^*6BG3+m6jWkvK}K%Ri~nH<;P=%agvBf6b)&i4WJ8HtBvEKl ze3?_)-ut;7J`8F8vRaI)X;UpIcc0#dryv@GvELY zaIyiTfL6Ycy}bET#QzM>+3>UuUYk*yt=42sg4p7|-absL8MazsNLWuIAFd#F1k|1f zabch_T3p9b@yAj8u+5OE=LtJ&a{i_@PoC}I!w7GT*vE{B$7<*6KyR2dQ%yfxRf_@4 zs5{WY0P5vF`E3GsF&N=knTj)M-5P`Htb2RUsvJP8QUpur+Eym33U?u+#eU<(^FyD7 zk@Bvu$2ub&oLr;&-xz*`t%fI-xBk}ED+jJ=HaucBMd6t(ag zEA!yuRgaJI;W9-Ar)XpNCcDR9bP|{EV}t2gj#0iLyu<;ggth3o2qf1LOM9O z@Tht(q;(9yuy-In_V>E6c^mEgp%MDxZ^nvs8|y6=BGz8b_D__lf)@+V$h#WrB-gpS zfrX!3`J}q{eY?d1sDJ^-S_cLy532O=@0-K&vp@8<9b;7>aX=y(^#i=SQZ{=-9S?6{ zKvrk9@m0e87tnk;fwsl>vA@npbKwPQL@Hgje7g!l8uQ=9iZ@_b#rWT3g@Iqkg%QI0 z*s-T@(dUW`u2Fjx4Pz`17rm4p6_JI$C_5)lH%AZC-!pu$xmVaP0C*%vNf0Wrvs)$w+6|7BQjd_&uOizn*{Vp{Lo&!-!$Kn*Qm#%Tzr@ zK{n;>4W=z;3%)`NvUN9IXoc8O9~Kd%X1?y%f=*OYMklqYDJD|GU)pS@F~60M1>F!_ zcHqFb9BMl;V)!v6+U^93w#h;Wr_epNU(eMP9g>(5Y3`!$O!%~vR5vy_v4I4(+X++I zVQQfDt(WQG-e0}%2i|jPzF!%e7%pj!eHVm?8d;@~s0mo!QCh-z^$QPfYdIhRoGO$Y2HDM32S|}^ainnC)%)rN{tuf=z!K^XVnT0MT>sBW`MlMJ;J7d!ynI zue_^Q?P^$kc>i?LG*n3T4&dt62}UGx2m*QzUq>#j zrL3)@cOa|ci`Q>0PmI=AU!bk`v2z$z4rujwV@0es2?-p~lNI+HF&?RCzwMN%@6)g5 zdbG=f5et=u0UHbQEtvt^8YROf59-J*vpx$Vl9AQ^0?CzdAPQC$dC%`nCsS@k)D&s? zD{p13GGio$Y5X7AiVL(veo zwCy6CAl2ADqjY-HdGd#T8sEzU)ZqL`Qa!<|DeP6fW-{T(q!t?}=1pq>Y8{uT7!Xx& z;SZ3E*+I8X7K5y0CaGfC*Kt9Yo-NM7m3MX^#rJhc=08n+aTokO$tLvUF?NL74yOh8 zMy|AH4(pz-qQ1E~UI6Q24ej?NSneP|aOADL31`2TD@aJH$>lprOX*`AAv;sUNBKJe z`$@7oIql_L*lyw7Z-oxE=idrVEO}agr~XZ{s_qV{uCnDL2r431=yL%wY=hn&AiT*J}*Pi^Dl_sip)1-6p>&Q6m zwax$wdC`JPP~FB2F;%$Tx3ag!i6z#6ffSc7uV|f>jcV1Nxmsh?^)VB||HkqwW^?|lsWn|=B%k(%Wa2r^vPA^9OSV(Qm z+_Lf0Q47&e*nQNgfPl+*N~6A8g`&Gtu{#YInWwtxepXL=!_uLpe%qt+tpYLjB^W8#WrFhT1++kp zz9pnU9JyNaY*}6vwY=G;ptiiG76#Z`$%_xV_tD=pd$%;o@7P_d#ZDIFt|1-LuVz{YU@piXC1`;;XGQ(_*Ghe^yOp_F*aCNPePQ-> zMt#Z;A&8PHVXY1A+t~WJKBn_K&fv|io^5-4vnXgTEJEanq3E+VAKgduN9Kg82cetI zmA4FstBMv+^P(QNz9Q8}V%pZi^XB|2eoB#1574 zQrjWDu(#Yh?eW!gSHQ3B`B41;p0sLBXeC?-~wWLODxFx%SSHVgs zkVsrh8T;B{f=8}?teWK&Xt$V?rLWB;UUSbWTKuGMLtizlhVbXqYp=0G@kDu0&eE52 z!kuFZ!a{1Sy20rL{srIrduJ{$p9CTnwjw|n9*GrqdtTKG`Hf#fptmjiTy;fYk#ejT zTk~sa(YLxr2$e4sjT93LHUNxk3 zOv=wwTzRqi9SyQ~>hE&$?kSL&K)zq26W;Re*{Vo@K`-k7p%(voV$P>E|2?@ny`@lv zeZZ*-02LftG$A;x3)7+wLFY;c!G^Z0+*Q*n=~^rxjFkU1II(|fwbM8KlwQ70nQV8pI5n-fqHOsgVn ze(Zmk+x=5zGYY`XeE=}rJF9H{W!H;82|EIcQl&ILQ(p~#YTljWp=2x>w(cM2ZMi)P z#6#KRX?&zo`IdY3sCw(V5;hPBKuOK->)1z~tN`P80H*S6P)RQo2t-7y#_o$A1_{a) zS)Jgjqs`Zb1OI>fMvufx$?B=0s^5?3uf^SYdkKR-(>9o?AprIG3v^ zJ-jHREAP^lifQI(lbC~MCBnNmc}*lE{1GT|e$+x`iB}u-M4P-@W#!q`AMTT%P3mp@ zQ=Iosntb1dpWh6%NXcGFs#L$hYLVOMY*t@+>oJ;Ub6YyeYD`g$UWsYyzJ-}9ENlOU zj*3xTExM<+m^A{eBdd*_2t#c9b}IDC%d}{tDe#}8EfK>xa%Ax8d;DyZoJk%4r7s>i zRSzGsx^nGTy1S(`C!Py&jlTMEla#?5aH^j6(_*z>Wk;6I!<=3L+~_nuNnx3et3Vyu z4MhwhGLBt5bVFMp)U%yGJW?e7!o*VPbpLkuwAYgQ_4JjV*(Po*ve;UMWADImSP(d1 zhOrZHK>sa>-1zPG!8oNeV#j5}%ssimOC!x}ceI_B5&8})xKXKhOMR; zooc|a;v=_CVL-EkQ5OFHurKWb53pOU=`92l316!%Dh%Dl4zpBY!Xx^6@wI6ItlUZD z1kwb$Q)uc&Nq4UpR`J!_4%(t_$I@=Q?8SnJ)_+EqxnELsdsDin7jb(<#l)y+U)^`=A#u z0OC3y3ly(R%t*N zVv`1#`KIvEtwE<986Rq24)%0A-OunFXb9j!I-D3+GMQ%Kwq<0(1%QjxlJVSAHX~om z+T1&J15D(ps;)4NM&n(6_H@#=K>Bau!nkHEZFoxS$5TPCll*sJmj|)Krz#)%T_2&v zAUI^mAyIG-6gTs*{8skP>%D4aifTuy)FyU5?|pndMf}(?C9_9*x#1# z8U=^=2^?`5Qz-x<4kLoW^{Re2fk_iOE&I&bc{KT~@*vhB7||T$XC|JXNR;)gU9Byv z^snjyK-dw>H8p{-8n|zl7g_?;*nh+@dbn9F@~*pAhgne~Omud3K3WM8CZOZp zL&v8eG6O#ofjDO>W_W^u$Clak*bmB%fqKs4DO2c4ZxGbvn~F|-a}qp8+N64GQJ_Tt zwMu(Ib%6x$60-eCpIHss(>JVCJ~3zCV{eO*3bVztJVT=fC`ihw(&oK7Y-JiT z-vR@h79v*WLA1}7?(KcKs1gM$@=Dn6tfaL#b2dRLt%N>XZG zA#A(-99nz+g4*Xq8UkL$pkiBLZQfA*iw+<5POn<~GXcDuE(MPEC=^8{w+S=>4-emK za#AqG;k+nHlFv9&%;xfTw)3SdRLLc;A$qRq)9E)WKe=c&oyGr^9$Bl;K@ZW^nk#2h zwOSWvp7qU*aq5o*Xm%NnEfcv~p3S8RbS$5mWQM@511WYz(rOJP5^G7sPy1)Iho)(p zKl=g{;>Gv^7KYs)0b4?y=zwU?$UGgvZA(Q7qM>wRWkA@axZ^0VVhawg`i1H%Z=vn7 zVOL?5byL+4RVh2dMxi*ehk4I8{&P8iDl>OD1R>1E;GLVm0WZAl8ZD9ZQ|KA#SiU}1FM}Eg77%J&W%<4o#;Ng(p1sZ>*PDcCUoIqFWv?M8hPNRCF9hzI%)A3<6oRR0K~?JULzSLcGaIhC-!kO{)PG>03#eb z?zwl~aCd!RC<_Q(H#spc@Z}0)KIQTUwvb=5Se04`pDO0n(&AL9`)78X({kZ&3N~$i zzYLUYsk%3GCWY^2^4}H2Hw_ijbcD11E8iBO^|GWfXw4xka*W#EZs`yGet-nZ2Q1$QbW3){?j+r66Nl=GtOYxpG#zk8 zmL(=I$dBy``*Qw%mYnhyrF0dMvMX0vL9i{^GFaCMhIcbl{OKafaB6@l<|?mH57E=p zJ^;GDD3qkDr^#4<&PW8J)2g(?A;sw38W)U*G2N3o^^A~T{8KphZ|Y;7NDzN=yRKAZ z+Sw1e&i;4VisIeWJ@X`gP;*W@&^;>cl)oONqqP0IGE~=o>z~|_PR(Up0Cetc@Ye>% zqLog2blb;*dYsLJ222QMvuPV6j)s;hGAOJE?v;Apeu8i3$bhSpcHXE<9~@Zg5&LK5 z%XW|_!}g+mfy*3!R=x}mD?@64vMl)xww6|S%>^X+kf!aVqR`#}5wQE$I|p~`x@@Ep z1*TGPn%#x-$V^UDwzTQGk3E!{EfSg*9{RV!syur|Y%{e_)qJ!y7x0FK0bb~#D!-Li z_bi@A{!KO8o$+9QolwKgH1k~ZyL4D@BPQdC->$S*Dvd0KbRRHiI^EShm#D8+(nFKW zKoKBv*-`n0TqZv%+kF8q5K6DI^A*PqD80zb)7mR5@rQK{>^?ZF=Mw6zW^eUHF4AkA zCzHNJ@xXK zh$vfM5Th?2KU4iTRmzX1QzIXR0oBm0zZwh}pLPHYly`XbIR1^Ppo0KNA@lD9DS-BJ zX@<9AU*Ity-s3*E_<~XW*Khk9Ozaun(BJlR1Cg1udM^_rk99b%BjC?kfc?O}os2g< zc=h;kl-=`O7RKlkz^9Bh1s4AYpZEBO*1|}LYsKAW*f9VnnB-N{h&{a=s zbTY1XZ`|+Mrf;KRDbr7{Pk%KXAQAv^EsQ!+#0p@^O1|o)w|~^m7mki_@gfg>ls{t1 z7>sIhIXD)Ey}5f>$@8J%$?tfd>q~6FYlplU#%r?`Ih|9ZKWJA-cMclk%lv&HZ!Setv}-tSktR%(XJ zK`TvQ%+_2RXtb50Ke_$rkSrG}+YHyeq0&}qIqJn1y!KI>kx>z1`GgG|b#!y@9m=tG z7KJVgg!?=-GV$1|SvFsNgZYhty$QT`dY5lZJ;NVMUP*rI*C7ohK*u)%jYhjt;PZ^T z06t7ICG+AQPptPl;^-Wbq<6=)*QI7&)ml;kRSYN8N8*^!6F4>FNd{ z*o*Ozx+-lcf5R&yk;{2S=e<&g?t8Am$w39GP|K1MDL}wR)Z3_`A^Mthy^+Y26Rw)PdG%@FI6Ovx&EfP}NbfJzRSGoad2NQlGs;-IjVF$TZ4fI zwLbmNKK(vVO8H^~&`6;-9!@401CDwI(z+TT^ z#<*){cbNcofJ@(FJ)#A*5W{fU(Nl>^c?x#nkiySqGiiE2Yku0<;$(N*kBszDi!-+( zC!|0a%BNK^TWp*9j>fnn!hix;z>`}k82*LyJ_BH?!f3Db89;}N4;i{MFf!-V&j|)H z^;p>kOKJe_|td2`4V4U(Fe=|CTQmrVK@Aup^VamU| zXYu9kaorR*XGk0h)&)1h)locl5|d8 zsc@W)+%B=^{oMFEEw9K95XaCXMIBe{SE~lR=eVyx7K1{wuK`be%d`iN zNkeMa&UmXLMZvWYvqHjBr`I<>MvZBgZ|^s3DQ`OAzjT^HP!#0B#dyQT3IXgn_ErnH z3l!$bijefzDR>ZPN;GdVBS#Tve*8@O$ZM6y=+wz^rZQ2{AP$rr6e)TMO6^h6o*b%| zX!8kXIJU%qdp~;EAFfQ{UK6vq+9nPlX~-?1AcTg7#d?um0 zQgMnc5)f~x?gLWt6_`RJCPl#XgozX|Hi%<^!Ni#7>Yj&VDVqhr_!{^7oy3jyda}05 z(qf0tEykSU+ka}V>I6ep!Udr@%SoG4AinTa}*>+TeDmlSd0ue$z(iSN$B(1 z`jM5PIlRe#Z(Lpg4wFqmN?S$X85yYu317dg6s^oov5U5U0_Gc)C5S;)=yUS7^y%t< zKM~5o(dG44?K-hk#~hH3KQK&6pE^wrH~Gkz-b=)c1OcP?H@i&YLfIY5XZUl$)5lS> z%mu)Ft;4$Q_bo~0oL(;(*-xMQ>Ok{}!~*79S)eGLC`5z_P>jB{3=JsIac)&ZfKp zDiwGd?zgj2V|Df=6=o6P)r(W(T-#Lh3%7f8W`Cv#v9draqNbSCZ+t)}x$<0O02Xr_ zVy*Jm_*}gic%xc_+!{5wFKSo0sx#ebn`)b62RLjDY$~&4rx+Jz9-!gwEF-K|bSkGQ zC~3`RQPRJ8gc_+uLmA#%Zv7Zd^c0U3^AFGR-cbNo5o<+6lG6H4sX`yw>JKg5AeoPx zubLQ656`NkFSPw=eeVwcY3if?ETRvn9#RiJGP0Q|)%p^%v(sj_3}3(dj3|b^_oz&U z?|(rtYjL1#U>oSKVA8Vz{#M6Jd)Et2eNCtrYSB%Rj;{|GK3i}SaJG_U9p%jgtg9?G zm0dxSO>K&LE2(|W3#@IFHUAVi?wAHgQIT*w)Mi(uOP+^8Za+zHdbP# z=;MxESZV;>I@fpPlg~nqWEhhCsv)8eXlD-LBsFVV@mca3Lu{WC3&yx>1U4p12Z1g) zfVcmXSc4KEv$vp;oS6I8c#P)uW)+mYsg>`ZMg70W&c#`KYh-FX-0`D9F9! z4c-@+zuP0av6-a&KV$cS4B#CS7^=wz%?d%d8rt2o9HlemEx0EDId;KUht`U>x98yJ zv!wFnuYV`l8{s36&{Ln>zM!D!Yji4O`I=T zPjHA|3_Z-L&cyWFLY+5}N@wfzB;bGI$5Lyy8dg=%F`4IT$&s)WGA0o0hHw; zaQ)Rn?%CDBJ;GH?h5?)OfbE8A@W!eTcq23myiuCfX*BTDsMxsCDDPft-)L=wv-M1z zh1VE{0D8ddfn@@ZeG(}3x?@Fri%at==c&9%o-?5n3<%L9o65M`n8&SPZZ-Rv=wQzG z0ZVo%YcDRTRmW*_Zci0F$(;3gSf1iCp(*ZcG1xhr(xKWxn$VWtZvO_xqp`jw5ub{! z5bvD3;r5jqs+wf9#Bw|!kcIZ^_%M5d&i4Fmi{){@haLg*lB`uTGC6O7!DwR}`!E}^ zVutacO0Nk2RAhN_k8^C5qiL+bDrL3J$2vpNFW`Zhhq4m{l{3??bKE zu0Ep^3gRW6#k{H~P)a+4@mxku{S{PKoH>4Ev>ZRwW0ZRaRew(@9~ZDFpN>>Gb-VhO zv~W$FHdnQ*SVE0xpY!0u{*8x?a>Gd)Ts7>=NqrS%3X!%fb~SGr$2@y(!)D_<(rq!B z`A2sa+3F?~Ov)aW{j%&+G1(gX;Hk2$r8IURR6>bdlryqLpotV^kiQ0HKx&d~G?6nb zvje^e@K$n1DtR@&hClIYwAV(c?HF|R^*1}!4lHj$fBqcL7rZrYDZP&hjE~bAhAt-? z(MBZRD}LJ|1os5U^#tPxQ~j-;$2dfdHn&z5J$cEr(E+@-g=u=%iv94Hjyx=M(k_yB zK@Ac-+S6b7@KFvpwZODGzh1RqcHW~I(gQvxP$ zq|>Gj;p8<8`<=^L&lk79p(m^o+(qDoL)d^v^K1gdOADeQP_%QFNZf`n)CdhC{ggTpY$2c8j$$< zR+g6BNz1wVw2h*TtvOhoYj%9cc9(a3aXkn{_JLuozI??ZHh439TZ@Y=w*)oRoR=4k z@Bs;%KfbWM3qV*^zEfI?=ekd`O2!1rsjkL_jF8SlPQ&zu!F3PkfKITk3v&moZyh|Y zzVEkx{WIXe1k`(_E8a%g%?|<7)f6GC*W~BN)^0@UO=x0bq|iS1(6j|!mZ#}I6+b*U zJiL&&1|uaE$H_obrQu&F9^4CM>CZk z0dX8G2y%8aRCjM0MPwK*NrxA)r(Z3HCI#4IekRiN?a8Fr`T~!&3g)!=3Wae} z_&4&vH$)ZTkn-HiAJJcO_c?1E0d4%=Voyg3t)KPYZnQ1#zytl5s(8bs29wcx)E8Vp zd7aG3{?i``UQVFO?admB8ZHgc(zi1CdG)~To@KMTrjJ|?JgDLHX$jnc_N!ijKl+@J zuA6F|p4Y|?)>vWmvB&$b1|t;J8tE{Eg|fTbwA?!Hr{Q$4mhW8&feB|v5VPX$q}|as zO#c^Unm)LzOm0ABRYFq>56bCDSB-ZXpPvWc()CTYI`kd?X`Uq6*=76R4?NF}+xSq6 zYcm;e$f^^p1Od}lApRfjIZ>!s?8EY*jj-jOSI1G3$yeGOV;`(KbhzMq>8V)>|59c*wjch_qK9lFkvqwhf&Tgmy@CsPVeUnA%Z>L7_mf+_)I(uNKc{)KSe zL~Ws9A9TkQq_+R6p>s*knDPE?@|Uog9%T$scHohb$-vrAUO|nk!Qi7-?v6&wH50^P zxIoES7eQANPZ#?auwx|DNT7T3<9^$SPkpb4KFqAC#z{i^E`C2DMVDC9KDl{6Hu5e9 z$2#A8d{*FiqSL_l!I#DG-YR+PGQrZY_%sH9MvF>k2Td;fqO$^8Ul;p6~mYKfzMK8zSxODU@B+ zA-R5{=Oh=QdIFuJO2SOwZ8JNQAsE@!@3KxeGCSN{Q;Z6yH&y8Tpx@;S<2*_EFbAhB zvl@Dq&ynF(HjwP1$Z!by-rS9T#bRL#!fE842c-Oas`mtO4RQ}Uv~F!8SFqfgX|}`m z5AwG{wm_oh4{am86*gbQ_QeR04zaid3~+;o(?mEeRGrKuP8-uazfWO;BFW26wq6z% z)uTj*-f2v}?G9fj&ScoumTinp5X)R{4zFZ{X$B*BQVUO(3OH^;Y2KqDk|t&#x+d|eO=d4O{*d3g;+(opl#tAbKJd%ws&a^h$0GPT!Shy)Su!vVrTD;d z;qCkS0@8&J^(n431;b9rv-1Og)&d|J-Dd1m{?$|b;W#et`>g@9F7l2jt096t(`XX0 zZ<`27FuPl?a7ojo{dUUPk*&7x3cqh$_V`8R3VwA_v?KSO=TYwmWFz-H#~_!1TwRsR z{fA3FUrnffg@|*7Yeon=57=3l0V@?|g65`~e{B#`VgB>^?JxQ>Nuzk3WT8b|Q= z=UR3^6qbiG-m%AX%(j`k3kg@|!n7Xhu{Rhwnan*)O3#$4^`!K4z`@v37g2diQgB+E zE<4;P^lAy~mwO@iGSi~x%pautQWrCK9>%mSf3~pFNlLgp%5r4Gwe$NXg@2~q)p@L* zTP^&v_4cjGmLGRTM43kd8Cr_1sw1n_cD=Zez*99+ zWKurZ$8BJyv%8B#kEAIxFI9b`dtz%hw_<1dT6;l@GR`xIOj)sb3Oq|eVUD&_XXpsy z^Opps*{=3vaieZ;G>&6WHDaLHobF)6H`@1Vc>5+i@`;hxce@LX0xU=jOyul^U4*mBd z(|ol}(Foi*LxE|+|FUw2&wX}0dK{IlMgY zg6GQ|g#b|S3-p_>sp0Exs+9NXGUHtN5iO)1e;P7s=yt1BPW3XI;w4-61pt-t`(-8AQFE-i-HP&87 zSNbiCA7|-!k?lN9^}kOscV*dNN6n@;Z-U%cQLvs%8IRuH7zfLl=ig`!ZSKE!Sf2DP zOhXf#KLUQ)pU2)MYhHIRW%+#rQO7kucts<<_Pp(th=xUX`}$w!tT69xbbRKNO_uf@ z)XA_d=%iFzM%I7Mh2}JLMG4d?))rO-U(_7Qf=O5{8cKNtZCvLAvPluc90D^%=csG; zU~fZ|xxiHsxd^+Sol+qcw&2>~^=mEVOCycU%SG(W62;G3zWTT>74;iX95?g`QG)nUsqJ0bj#+Q{H|tkhP&_d9VfPdaa8e%4FOdT zv#yb{H@JnYGfbv!MdCj>MiyvA)!(GcZP)Coy$Om+>oc!E|6mm?8d)akkBL{9Z6q{8 z=#Zi6c#K)~EH{>YOG;+RZ6x?oO=IbU`SpLwvsHeUJ2# z{};Wr{$jHb$zt(8a9uD(bdW!2DrQM(gM|W)aOrPV67YWKJY19jx;dwjHI?HOIrr8J z7zX85o>C#cnUGsuQ~qnSPS1Ooe?25tBJazV2_1HA2D|ZlPlmi|!fLl^Hql7Rars$` z=dCR;6Pc`=LAhnx<9+)w!F3gjMdF)&am}67*u02_S~4dUo6sWv)y4mD7gDnKO*oU+ z^hn7Q$bnyt7#AsLmCs9lKa4UmzrJvt5A>$<8Xi!l)(sEJEk2_V??uIE9`kBU+lGCy z{9HGrYmR)0pql)UjQ5y!u4p|Y&b47({%AN7Tp!jr=O&ILZm@_QQE6+an9lzCVXlu( z6!ect`Ed&6KFpnwX(*eMd;76arN;zrOzuW$Qbeo{_@3aM%ol~xY3b3D^t9klEbBL} z-&gv2TerXRovRZN(%P;4PE$?Kx_wNl@8Vd{W#f#P1`*ka_?>L*)TQAEpW`3Wd?r78 zyXw{Z-;eLpW)B&hZ^^w1jK_dJr5-N$l>OoQnqZ6T(JT`Q%f!;8-0sqCHjpQ5ql!q2 zElq7(qX^@lnjSCupyVR?m?>I`kDH$+1-jf?=lL>N{QQLfz@1r-^KQYRah6+a1h#YT`u%dW8s*s6;Jl`DKlzns`2=z2zer~HIf%Czt zY)M?OZ+~LF1Zl7y{A^LFO`>q8Fw&^?VWFA&VEg`VAQ*~W&kT##=S98xDaco7Is{nO zlSAL^i@9!?(t0b|Nr<^U8R!N2}P*Or9vpj5TXl3l9)4#oEe1?W~+oG zltMY=l;bd`VZ&A=!kigqM$X0@W`>#lUasr?`F`K;>-v4ae}4aZ|64b&@p?V>cs?HY z)9d~{aLkx<=>tO6R^pCm0Krb)HCgx(qabs}t5d<}kKL@Avv#1Pp&;JlGfHRN1NN#@lF*>Hj1MYK&;VgC>eheA?R+1%^+&qec++wSC$mjp}>Eb1UIIUF0=%qP^y)QTRn^0;>HUP1G5In6h}-7oOq z6EH9jCIWTa!{O%~ZH4VF1!>_C(gH@t@pU%~g3;x{oVNWM^>A>fRqT#!iXWH|ZG0zep zqtwsq@0 z#tD3s^UFZ3MQiEv^>L8N-itn!_dC*ay%ZM45oXj|vMVWdz_AGt8Y+C7J2(8(%CzY6 ze4zKrg5-el^rj|0fZ6H@T6K+K2XhD#7&~KJI}}USmgIg~T2_*3w|GT=(5)~Im zzBHX7^Fi%v()qjI#kK2|s;P2v>zxl5K5e8?gd5@3u%XU^?NYrfsP@Oc%c=D1z=9h> zS@X#|(dtkPSO^akt8c6heYCb3**&^Vp8dRtBvcLBW=1tXegi&pFr+)5${(s@xD-%k zf`JBT)tuHfv&7z9Qm~8MXZ+HMmrn{hH+h0Nd;?gD(la{`dvbiPtFAd09Oe1sMoT@8 zyPLYK1R-hX;+v_*|At-Mc+h+59NY*wdI+LmD<~YYQP_yGt5=)x<__UtN9VG7QfW(-N{jpj!0a0ifCGe)y{6nOegw zUBK}3ipTz=nzf=Z`&wK6@3U{3*|}a>O?N|TH9BpzhMAm@&gxC&W&xRsTB2vcBiHRW zDzA2APCMtA82hYgD8u^lW8TyJ^gTZI1TJk?*j$6^w9cTs{FJ0k&-9xG%X^JF%OLwg zId;0z(3HVb*NudBJ`n}k_b2O`hOX%mZ9a4^gBhnqVV%N~*`t*Nl*MHj+V1fmvn5MG z14boHu^|E+qba?}i1roK{!`Yd5f3vPH6J~9zhU0|huScce(O z0yyV@(#}s3&nmp0@AzfBI8ap|Uk3r6JrC$+a=R(F(zHeUb6x0cHJ-4*hV#+2cSUBMJ6~2>T)b&mnH^ z(C!d7vpd8^=^TWF-5lARWq5sl@Z?~YvzeNu#0$fUf)}|4?iZ@!BhbNP>V0}rQthvL zPyHqqirg59az2CiFq=@X{v;ZF>7!_+T&iYweTe=>9mYXiG`oRR$~dvmm9N5e_aopx zlp~X)t6WuIpw*VpR+eeka8~LTm@17r!XiIa&c1fFC2QFbX&U9Q|9Z046}Q7y!S>7- z(eh>=TldK#6{l*l^(33;qU+J6{Q{BmmD`LLXGW^RbK4SH@Z5 z4s_0a_!Bzkh1BwM{Q73u@n?~%Xz_Nc1p6qJw%z}@GQW7 zBc7G5$J7fo$A!EudG_r~Ssl^PvDi6VFBKHDEtFErjE?v2P#X5oPg!C8@AOUz-Hyn6(9}fX|JX_odQM@REjf8gm zz}*$UAz+j~sZTT#V+75bxces?5T%?BdkQ_yHMy5F6fc(OK?Wvbi_6KbjMPu=Q{hS9 zMvM6*;C9y*U4?V}17=qif_tXS<*QsL%W`?U0#xZb)&2JM(Z$02_zRKtQPR5MH>ddV z9#qXI%O_M^8)i=>g`q=RrvuW)Bd!O-x~b2Pd14b_h_+UBK@3s3;%zY+2hl^!>fuwZ zeO0AedM-KJL4e;wUkt(i{?5C|t~RH;RUEh*Dg(uKH2eC`PYSxmuDxTA1EcLU9>ILu z^Wf5O#PrSny8I{^DR}?LU^1cXcbQFmLC=U(w@AM)7C(w*zV6zTR2K~{B-6*0tSA{$f1Y#HKo!4 zNU|i%R>lhSA7Jxt|2}L37(qFk5^!P{S0!;-Y2``;cSciXg%PL|cbQ_FZsM2Ggw_4V z2hnFCheNfNshPDI$Ni+Jd0+mJI1&~Fx_DH0HiF9@eGgN9zpO+#i;ZG@h!9z;`!O++ z5z@)ayp`IkXqf?Us#krqCh#E_Vfazmi#zGS>)xW7Z)Ko&zt};&Z(t^mhf#L?D`Mz7 zU-k}y?jLD1aEgSAcQB6?rJR21K1rvA#yzH)qYZ{Cx0qEtoUcktH7O|jyiAm3jz_n= z3FDA=`?zP{b?oEzlWnFv?|LV+fUNsQdOuEE57_?DA_qP-sPxnv^0Hwp1_p+t zg&LMM^#VbJt)_%PlB&0MOWgz(h|*}N|CSabB$Bqc5-F2(wlB$hWB5k{srF`|92vMJ zf$k;Eb~!F0-1xhWWbV zYD}_4v-g$++f@4T*soX?P$UiHSDB%_LNhjFA4ZZZmnH%9%EDwUd4u&hj7!4@EiOsY zyerumgbN~5VLpJz}ftxqPi=uM~0@m4RBou?p zPTJ5)cyPehj8mO}9{oYJcHEqr6g+-v?M~GgEIj6wk(UrZ;zdFVUyAs07{T7!i~Tj& zhIMUt5k9F}hK`at&lqt?l^G{^M|B5gm`~r?KxnVcNSdW|V!wu_XlD7Nla=w28ZP$0l zH}8Z6ZCL=^%f|&PCW--i`o&ZpT@60(;}F^!-1SQTMtQ*D%q-k1<6oeC z>_h5$_pMDmpu{1yY`y1T zG-;5DTT#8pC{?StCwI$gJ((7gN|fiz^l2$IGUuv@r=+Vedaz zJp~?5&L`($r#=bcoLHaNigo;yz4C`Qgosrr`uOGwzSg-@3Rm}7bzL4vg@Hc z%91RchQ{xo-c0skG$|ma-saRC6sciG)s_#b`DqWz3B_-TL(5Pk@V@O-sW$bE9bT_D{kZp?0HBYcEInpT z@H83ndF(sPI$BRZlLu44#Q&ORG9ITXE@f$c=Id8wmiypA`Ggz_K!xNkINn~Dv{X%c zbm^Uu`!;7nzoa~pE8gQ>XibSIVadn=gRDQ@pS5m;Y~NTRi}=2xJ-M~v%ecT9J3tZK zIi}TnwmwR%=-7Ouwp^{Uxd7IL9IK1ma&Hl)qbx)XD9uEv8gUsopKuI&PRsjIuO1(t zMoIoM*l!O$py_xf4FF4Yl_ARxXVP{b`w6@UPsDXZOv(yHP?DWh8Umo3Qp)mIJtFx> zjf6B?=0;$eJ6Z3WfLfT0nH9}Wy7`=Kn&Y8lE0z+b-dq1$5=|O7mhePD;)hA;t1mG1 z9ay3gwZyYE{rgb;hH&l0NKEN6n1r>rEHEM5{;pZHUvZ{tn%BtthuryDiKmDp)*+Fm z;mVt?d8Hbc0?|SN`RraSALFWj1LPWoXxeF=O11+(g> z(pbqyY0fzg$TORnrf0x0_g){aG&q6ROAr|tdCInb0x9bbu4s<)P3O=Agk_Yp{JFbF zjh?i-lX)*xt+rL5^;^h`-S>q?fM0%tv6T7fbGBzpo;kk3i2ug1_n!wBu8xMBLVB(*qY3 z36C7vFp72cqpjb$C!*<{!MATL4if5<%G1h^K(vrs>s_ipFs4%3MO0CKX^#E4PP2L5 z(AHDcbdt7H-Zbc+i_nih(QGC#Y0JjMsKWd6)2e+m`(+@%mX0NUiS%yt?|d+HEoZWu{*$`wUbsRQDW&0L$M%sLB|2xx(qZGoUcR&BKIn z63VOYU*)uNHBx|O%|GXAqq2~4^~2j?9fxjG1H|o*$ial+xefIY=qU(pz=H&+Uh-`D zakp->ZVUhTvxGhbz58xWy{@&W4=d+cTr3)Kaui-~7g@g-{Ae=xu&5H%6L&mXNJ3@f zS#*)v+-ldNn=sR+s=8-{34K2zgtWW-NlCTR%fol05Q=|Hd@L!E5C$p)1QRGyd0Mz` z$KFp@-|m5z!T|sxvsX^Q%$Xca-MU}q3yfNc5_}Mh59B&BE@iE$_HC^d>M^g_H8+2< z6mPBh<-UD&y=Yi9}-L2dy|W=B1O+3l{M$gG<`WD&?vZrbRH z@fdh6B{mFj7cRe(4)$jnkDjuAt~k+pi1+tM^p@#x(h(iyQ8$zbC0^B3#y>S>l?1Gu zxSoE8=6m)4NqhFp>Jx~H=Lkt6)5uC0=<+{DUgN6IdP_u})Nj^b+=vjuPxso+tcVjY zdW+jsxPv`OtnsY{F~V)D>%hdE(s|qV!h-Yf96NC-Dt58GYnA1YZj!C7pcNmuGg&5y z>(^Gf`Ck%86<zUYHPb6FouH%#JleUsG1I7uyIV6xQbWpML5?~>kD<%Z%;5wt)`m|D3*t*b8qWRYm(`0vq)U&P)n4{$(#61or=1ft9nk(yMbwR7o;C3A z!iwGithmMjo>u?3yQo^YjDwN*Y_RQnwTq6W3q_A1z~~iBByKyru&E7pww59C72GSB zQ?9L%2^b~MY5QABy952^GJpx`Eo(Z8XEx|J%=s~Ysn@1Ke=|Jhav1o297bK_LsGS_ zL9W1_i!o0Hi~vOFykC-)yIZ*9)&h_S_=g&*4$6qF#qP>;lu9>GC}4DW0k(($fCgQS zv4@8%mw|CF<}9{~n*(Aa-a$=oX%0;vJgYUym-5ix_WOf-*&B*t=;?z4`3r^SN0o{s zkAQD5!#EecydgxtBCNBz=&C;CH!L^i;>;=h7VSstCDZ~l^-wmY#~Jlq;(xRNScRhP z>GG~CRo5FyUv5c!MQ-P(6DPJzX6s_35)U7nhn6J}nYeeJWe-(^&*1G0PNSzLBG3nd=C0ltOXk&_`syu{x-`>HCBk@C~rbT9sb_`5_+VpkEyjtO! zzvV_Dd*)jw&sLYmj@*Qo|8~#onJizXHR34os4^iSEtfXaeuv~=3}?1BuGe4DbiNwO znsfdTTYndGqPhP!$O%c*GFbgX9J>#IjOJ^=G7lT_e6Z=oS}%+u-@sSb)OE!tQvydQ zxel}Uy)^qD3;}vDCRSR27DBibf*$rdA}Y|vV_mQe ztXmQ>eQIB?()x29iIhk)?Sj_00q>rM{(S1^Kh$dhdW72C4^q7702V%EneAXo`nI>p zpV5;zdsDe5+_}8V{?)xN<-^m#l=kUR^W_|DfakDAM!-86_(sCgdPz*M-RT#PszCKV zLETF?&sX&*lMD|CeJtP>Sy#0fD3Sd8Y07JRK1{pI#N_sDZk1T#A9VQ0brtz~E6lJX z>?6ABE4>*qez>Ihrv2fMrIRkNg`!;=T}s^9cXu@FGppbudCGkcjYj0a6(4-$TJqx~ z_1+dXJl%S_Wkrl2hHAYGKI8K68vYLZLnrNAuPY<)pL;R<&oc{%UTjnX)ae^4hhmQb zkEX@^hIlx*Rl|*sJUlWanAee(wzzKj&m-?I2xM>&(5W4QJY-rmkSyUvuK{}V)cY^j zh#x)uf)7BOHC=7aEI;I(KdpNDAD{mh1#vFzmvQNF&m!p;%Sr-b(!2eBzr=D|OI*?K zYoD_o_FVs@G_NF^|8VlbKQNbnHJb*~2>Tc3NQJ@U_~}us<;j1P*T4T@M(MA>9=iiS zfj@y47~t>h{!h`>-{-kWqo>OW%Sz@mj+363HO=+@-rxS?vo4+6(_jJI2Fl0(^{4#( zdjI>=PQCf-@220`5Q>gV_y-~LAD?vi)?dFy;01V0Q*-P8`Kr6u-~aoK4TB9Kl=?~h z_n!}){}1>5uj@bCd*U2i0v^+!20X)+GWmaOO#k~A22kWZ0LXs=|9!sW1-fj*^2~qy z@qZ_jGM@fb7%(h4KI1=KtR2e=m;zR`b7o+yC1%{;h}n zw`u&_ZuH-#@!zKL?|0z8P2=D0!2kQEfzB}BS!&|JLP*7a8tC8#z^!NWQIs6GkGuGN z4LOV%boZ&A`AABdbM0lPpT(RRz*3UB(EI7_%g};ODAk zk)v1KF6~_QO&4WgW-u_D zg-TLHrRM_g--p+r;{f*13A{iglYAWS^2>11bI<;qNSgmzkmh@tN#*5OE#Uc-z$hK~ zp?>dj-Ga08YUEK=(%FS-SPRU!);h-Kh4(K01Hcp8>433Euwuus03HzJ^|`K*r1Q9a z1}hFH+67JzzNG_v&hA3z8%bH!p$osV^xKis!Z~z~j7yQ-COOrUti$a_Ptgm6aXuYAt*4fLQ0u-=!S)rgs^P z9{`@>|HDWF4#gR{iYafognhO^TTUm+CDW#jwnsVJn>SRDtCrlnWWJPIjvOZiWsS1` z2$>XS@VE%$z1y2V)GT?L%LaN{pbb=0B_W{4;*cE_mo-|AUjRtZ^UfauClmHCzw~#Z zP3udso7ZFidPea88k;^7dUGg1vw^g#Rt2{++`RUQ?=eIrJVZ~kLxOx+45*>i$1|pX z+h?3fCY0e2mwBhU@=_x{TA}zFOQ zoE}{r)Jy@yg0Z|JZ&T~2yD(@dcQhLC4VQVR#oR5X6eW08TlkP%;%=@Y@;ZQ5rNARK zMnFsWF+|w}@N%2kK^GCVS^yvNQxaA*8d~y?r45vnX>R4RohKvlaV1wx6M9O_^PYM3 zRb_kD)Ap2%Cz4&DZ%5FI)k|hH;Wj}c!`Z#M5-~Xbbiyv&Ac3Ro3RfA45!rrab7ksc zbpX*L#WYB`y3xV4CIVU&z1M*q6GGU^ zXRl-bKKyjXL`Bue4`K5{}q>&O{e*UVhy_G3;h8PyBwv`Bz>5*5f_l2JW&lU zzI&78L=#y=JT6MVsaZ4?+5<~ZaDa*lRk zikCQm*&9j_6xej%KMW{OcJ;b%NfkR8Y%Dj;f&YfNOa=8!0NSey7nV+_F9-!mR9_>7 zSApZHHyIb={KDuOoh3sa^!yO&5slcW#sj!oCoAS8UTRiw^7Ig^RYu_ zAv2OspFU98Hcj*u(+a=j1{#0aDcs@d*#JRhQ_FM>V-NVkX4gpd0n^FG0|Mdz%G zVWdwbv>!>Xp>EoQ;i(#4nb`%`XJk2is&5wvlPMEGr>|d|P!n@PFovn2v+=WNzoCf+ zcIdH`5nvdY(%fWYbV*_McPy)eG2z-MPL>)qgq#l((RVaGoyFXpnz$9B9TtVw#A4gJ zFRU~><0z$^{p2c)Pjn(MGY>awhW^~j z!kc1)jgUux@A_KkeIu4O1ytc4m3;|QSCw-`rRL%sRc-BhH^KJJNUJoQIE6SdWBp34 z1GKY+Tu-~w3cJZ%cs_l}2 zTx;f9Q3=84Z;%ge0mgv++@X<3s{?Q?=Q7wf|*$j$bZ2 za-g49ECxGWId(on&l&0>P7YA}*@-Yq)cYiP=a8N{z*51EsM*KN_h@>aYM%HACLoRPL0re}zPd5_m=EVkR&mc)BcRH798w#+>^A_3Gr-D6yMeE&{ zh;=;DKss0HNd|n_IdP0^Il#6}$5%Z7i$m9LY;_5z`zd(|M-|sKT7#^Zw9c$O(GYFPT^&UHO^P zA!CL3oMTl#2Gp(6ExpCd?EQj2l;q;&UA`zN_spN>iZMz11ujXqo8BO`=zU(VncZMM#wZ6_GoMdw%%b8rOHZViL}>fvFd{eSUB@{alR+ zx92ha$+w{rSQ01nM)BJa$o0T#EjgiLBw+WFa!G*>IV22vKgiBk>ai!~Zo^Um5fzNO z#;z4RRGU`6+|_LlzvVRqEF=r8r#8lchF*##ddFxxc_B}e?E|`l5|*koxNlbFZ}v51 zoYN*{+FQCgiC3crN|wUpIP71A>x=McLD6;Wm_5?y_o~&y5I^9ZXV}E7SX2Jr6uGdU z(ON$`4hQ?E(S#O!8zgj*cU=WtArmjqz0F4$RPkcm!VKD&!pp4BVG~)v7RgT38NaL$lhjSXI<3J$#&*;#GkDSF;W zHlKQ6NMe3&i?)B&%Xt*V(d&=i8+=@DCD3s|OU-hZZu)oL;F-Jtu(5#vD_rKy6L=t@ z?*J&^o#Dv-J!yNhL{^nCxNrQ){R{oE%M62|keOjG=}K&A{#IL26dRq3{T;q5PSE0iJjJ zI9^-#2IX}*tvcIb=*=0Q^HWQPN6Q$wzF$3pZ=WlBoxKAch@;lTiCSyn@>#`lWVg{R zU1=Utir$83A1hrlGrS?znESXMf{$eKCK#EUTCxFy)?iCTe@R8SvXoEC;F@FO7jnmC zp@es5@ey{YCK6!uzthhdSxp6pysKW~)Jp^l7c-SNe9v-(}=K#oLBf zj-RXPMlP*Jw%MjoqLW1Vboe+LSVgZYKp(R2gMtGPQS#Lg{vc=rDko^8n6ixd%r5pV zKtnJ)InOl%>*n=cr)mL}xfWjMDZLwK0o)Zp${nl~6(XZ6l2<5^zfdi8VIlAb#FRdV z)1%f+U*qMs$bCUFA-}FgKL+GXp|{U3ywOKj2^Q!Lwn++Z3MQSfDPg_-=2-(|TMV(c*d`i##9a0d^+SQj<=l~T;`m;hkG)qlnA1!%qL4Y-!JU&<$(Y_Hl6+LQgf_klXl zVAw-55k%g?pzG9Hwk@H+UWHTZ=43Sj1YPhe`PTdZbP=H)wbE;E2y|ABSWwFLxocP9 z8h~W@rr2*h;!)(`{DzKZh?y5#2$o-7=kX)ku>u21Y|B|FHMkzAG1dA$t|y;T?PCqO zpsHH=`tZf)k-5RA3nQ~~x}PKX1-v}j$$sGM=>9AN+=0yI-HqI=+`Nsp;h!p`RJ*7> z?nf*j6i_AbeOJ2m!ci|iKd&FHuDj>uq>}4BNoIHPvK5CQhB&GaWf%}In_N70EO3^8Dteo}zA+t$dRFS#C5C5T7Nbt5{PdqBsg3DUVr7ka1M3^g*e{ zZNGZgZ3G{=a*wiW$g0xWU-P%N^G~3^wp}HQ`j=B&k?FEGSYDLcclsh1KV^SYH3bnk z30vxUTZSg>eulpqJB@9=0nDN9jzcwO&y+P+7v z&E`_{-!Yw1?Ss4S96kV&z)6mK28`2Kz{;&hz~FzK>1GiY+YU>o(gMSYNiGgC)9FP{ z659>QiKeQ?F<$DcpGchFq^PWQB63Zfskj9eUrVvlHV-@njAtr;|#0Lp`q#k7E`E$oun1dzG zJV~RXv+1EIH^i5$^2S5-AI%!A9dn=nZyS0~jW2+mqvwNkV@^^&JY{6jrNf0C%3k?7oIKDOPn5-XQCd+e*1<<-?rLanG@PRx%mBN z@55-MFLF~i9%wx_^IXLx-wYTT;|fUIoPBXZCSaJXGtCr#_>Mf1PVfs8vlOem7=iwJ z$us8I5xt&>VuIJ#vxH7!HI=dmTbI#|o;>l{?bqZI;k35U;x*rg(qB^xnpV22_GM3t zK)Kv)_732l8sk~mgG;a&ZL+_04?B?o$kzL3C!-*Bv}y7|48Jn&`Ujh9D}KSYYfsY% zi_trhDa%SpoTD8v21z~`3m?c)?5Kc^pfKNu9~zrWhhfO76OB?#yxaFbs6Jtv=rCuP zvD)-krr-R5;yNIIZ`Amwt)be0fAg95+Mtlzjx7&Ep|PXmJGBN>tD(Ur3aMrs)%q0G zxAj^KU+LS}yWO|v>-Nak9ZyZCdZ+k7U##n?+K6{_?XOiQXYBiD`%66pwcQ=S@Ox8{ z8GN{pD5YWFW)ch=Fhwakki@j7wDmIw1ab`60-jn-g_0$AE2MiW6)h# zxZhkr!R;`QBEi*0x_7w7ZUwuSirWDpWXtWKqQx?Sw2zC;xZRo9#AD6KtPMyxOvyGT z!)K}#w$Hjt$)V&cJRUb%B-zr6#)4lAWy4sA_GAwHPpeo^@uarh#9yXh_{ zs?j=k858g>kv`^0inG4!EgY~R6p9Dn2iU@4(Ot=Y3a7Uk;gr3VKJ23GgPQ)vx3}vK z#~j(&hkm}Sl!nY*W*Ltf2Bxw~0O6W2= zJxooqpfbbW%vU?eu^DqoVC!htTc_E7Q)Z#hCQoJpM$8~e>yS+O3M8;=xeA%kfy(gk1Xr8 zSIPLdWehNAhMQk|c?5zC8Nc1!EkrvDF?apj zAxKz9Mw0Q`-jRiLz0G+Z00v%1_P7O7JZXxa;4FMo!{6Dr_mRNh6bu9KvikTx@iO1| zUA#;zCp$i&_H1CLg9*v)%Fs(bx(bO!UHmM;l1r%5hQIKjQ-nE8C6& z=ikQ!bk+T)6!4=^`V-8zY(Jn6yh;$kFKcR@P%w?2$cE0S!$pA1uboWxfJSTpuzKEY z9WSz}{XYFbV?^tX^tIk-KJ|fqLM~aPs48J_Mmw%<%~ft{qqmu^gHa zk4_5h@P`6l?uM!~I_ut}yNZsSS)KzM$7g=XJkh>5q`4vrct5@aGt648G%4xbYCS~$ zatz=twJj>6cJ%6l{C3$A8)ct}gk);B-W^wn^KYhviGRfUOd9SIK9>Q)=Mw-<14`n3 z*=0+FNr`temyYPsce6G1j+Thkn+idf!<1)!zPQ>Xs>34=ZV>&M%Wl5|+7kv-2uE9c z6S(XY&zh+!Nih{8WR<_?)-p7l>enqK!J&}6_fQO`%u^P z^X`|49IfoC4}3Da^NFdV@st{zE>7vt=QfE={vi!h-`=d{%q9b~?lEfv<;OeI-`=7D zSeYeZtR&=1dM8_PJc!1IR>_dgNz_Do`F&aOZf@*souBY!c7F%9cpy3vn$46A?Dvvw z0j_ZxhhCzF;A8xSY`lJj|9lSo_muv2L4qrgmS=j;pkie}o&2&WM1D*9)@-1a4`Jjy zN$a6Rf{L+~t3TIugtaf7#l@FJpdXh=hy(%&H0SF|R1ylA*^xFr6I;|rJzLl!alJbI z-GW)KmY-}ZlDgrz$@un8ZM=8K`I=ygd4pciC^zCkZF!U+Fm?I zE_6`q0mu`;+rYoAy=z9VpcWqH%qM%+)VNn~=u$s&@DkWlLl#jwi!*)00M_=6b6Ccqq`__9NCEXDF^VEj*jBnz^(PxG zWb)7HK6_;V5__`wu*9Pnmw80q#?_AuF0`(nT>{eHv<9O^vKRKp?Rq`;p`R$Cuz`Ne z?hXLh1#7MX$rubqji@uIvvF8I&UVfvMr5vPly#|^VM2qn988{G-3D^zHjk4$0HOc9 zEjZuuYPmX(1H4gLWO08SbB_%!SI}T5aLxfSN&nf z+ShvI^hS$H6{N#8`{T5LKmf6O>-a@}i~i{yBd>qu^^$-_2o&KI9Mh>j*mh22A?Hng z;zr7a4%pkarSl2!0ahYDTU{jNnP)X?yeYsB@a`*1ZJy=Yy)oP#+&9`Qc5msi%P1QCNEBl1<=m&aar}s)KMp`)%+Q-@hNG3VHG%^ zfxx_F{J!_`qs%sSIA@{2rYQB145_>quk4GxY2ucT*PJIJw-6I;N2%M-QC?iFl*9r^ z6HtvO0>(sKH%%TNe&|0+8)|W3`avx+V?=Q(OEtB-38AORlye`79x%KDs!og-kKgJBqO<>81S0x<2h3OfRIDcIN$yk%PY=xM7gahvJ2s%EdUa3bVg5{ya)$B z;ouTRz%PZEdv6`PKojks`fO+?9di2*xjf)D6L|~*=Qt9J*0B z5$h)d1gRdF`UnGFcWy{&3oQSFuiwvdkfA6~EYSW3nUE8!3#HqeaZkg3qd@=g^xn& zSKrup&S=lb05Pc@ENxAK7y}DYOEMws4@(+1115cLNXxG5VM<=)QJed^;{y=MG0(fi zo1v^rV3~>NvK$gBJE~f5C=+g4{*#^AwM3G3V=CCnOyIpZP41wFHZ~t{jP4oTjy(l$ z?+S(I@nG;njavKdf`36hJ?}jfk=;

l(mtVfcs?BFc2{tKwdz4b}t|_W+W>F+5+Y zZElh9vmcm{gAh{O5iqY_YtfQ|S|EngQvbkd$wm2a2+Nc`e6X@U`b;ic?BfTO1i^Y&sPXG}C%PFoAR(`LB?<_DE2DHrZ)wuc>#|CScnniyG?05qP#p4_mF zc|u{02^>%HnuhbmO8&(%PJ0skM}Q*kmpd>Lj3WfnRBXYu%Fjb`^-wg42CmiZ^|3%S z!osY;q>X2twh&@v1pZ?kBXz;yD^!_fpZIq>TgK&G*54C|+WeSzIp+v+M*K$qHIfQZSk;$Wb$C(SC-EFDV0lRiidBAlX7Er+T-^2 z?rY0Mwur~_BJm95nHl|bsi!?>p(#tHk;pO2 z;=7zIm;%uF1puC3V`CUOMDCPewX1)I=ZOf*zB4fi%c&^mqb1$$AmUBcN?3tg; z;_-t*e}&FW$gR_yg72Xthots!q7#(OZ(uq^Uy+KaeM(e>3U6|pU%!``m#;XSqX7p$ z(je{)iK()e4@pK^7C+GDR6MQ~;cNmiII!=CPc=7k5&S^Zu4_Tj+8_%Opnw;n16lV| zVRdRmYiweGT@@J>=rp90pn2An@r(5Ju933C#=<_B?E39T@yvu^+DrgOKmb2ag&Djh}<+_z16=Kb%1`#v$`Zd|SgFo$JadKGoaQXQmj zIaY;;C6*tAOy$RyD5s`(hZNI0g~p57b+%ybb9U}OOL=dyD|}+;9KENHC~*h%B8ok@8v?Lap>_gur=mX1;I6{i%EJwf2v5 z&aCBf$&$C+Pr0AFT-Wt}P&z+fbq~9xoos4$9Q2&-8nS*S$)TsFg~u-cfyi+9T$#NU zwzP^k$`rYXg@2Ahf=}ZJ9U4_3ukm{k#-o>q=CI0Sk652F8-#NL@)`+{Dyi^ci)bX6 z@sDc8Y$`Var0x5#?x0&{+j5JbV2z|>CWsT$!)Rr7p{C%1+K|=JeuKN3_C2Lb?bCxx zc!iQ2+vC*V_u}5%1cG$3{w2U|2Z;e7P_y!Ku?X*0j?Y8)HzUb^ZQFf!<-x-fKs_7a z3C+p{14O7%j*o)l^Bj+3B^3YO?fcth*M6r?ojU{&fDo#WEb1|;6T+@$7`({USR|45*o)L{%CWA-|)>U~MsQTOHrSU{It&XY~>#(xTv%);m z;;PK=R6s@9UVYb}`0mHLfJ+?(IwBrwDB~6jk7FR^m>Qsshm&Q+a-j5$S`-?l|KbP! zvJ3vT2Kvem8;E2ts?8$gD;9*u?YN4fb+;s+tY0DYmuICzh3x@8byu6G{GV8WpWV*) zjq<~gO}-(Y!7~2n1P&;l#U{|`&TKbaNZfVS;$owYlmFLq|I@qNcsO}R++mc(D2TEI zX^{!^(f6&I}N=``=4&k=B7EPiz#wU8U7fX*=v%Z@F4bVMC>)w#G@8=M5m z!%1)!h(&)uUZD-frynx&BI|_=_ev8W`ymqieKPmmo*9dv`fQ;n&@X5e5w9=UT;W-$ zTrqJ3TMd;^?IFYUj??bC?ulP%7FiD~~5ME`4uKfgEtJfH5) zKQH|M))BUH+436eFzx?n1!op_yB++QEqu$YY3lP!I@uGySSocfm)`OlOTI+Dy(!pe z@z!2n@ZN}#1Un?E)v9HYy^^+GHz{*d1(3Wn8b(+p+4m!mhx1ot?XLfP8`Sl4yyY^B zKid+gu!?K{jj=uP-kW7Ku3uWS9y8e#f-D_|V3Sy?uphFV(|@t`$1-o+)jSPRLr7~} zQ*RgI@MDK=tWjMzOW+!x&v1~kSPM3moxk?;*Ew)#{1A@eE04eGF$jk+V3O7K=?2{* z|M>tu!-Vtzn_p~@_~X7i6)uV^w~pL-1f_ojT%Q!BM+@2;sgy5W7+Nek{;O^Nw)jSJ zbw#@3C+PLI+rYkDZv<+7q^jK#hd|V&U+w#+Qoboi)|5Y0R}8VlQ-)lEZg^@u5`oG* zp77%ZRXqr1m6|R(_NzA-Q@hA7Psyd5{Vq-UT--b@9ilSM3Zu|yyHyFsu@YV9-7$n_ z-(QSF->DbaIo*BpU+7VTqVt9?Dokd1Z4e(n{P}s{d-k~b>mb)`kl9DTAxYbPJda=% zchl2x*c!B=3iw3?Qx|t5X5!8j!>HB~$)qnPfcKkhdbG7o=w&Iiz&R$dpZ}3O&7bj* z(;@%W>8MLb);#B-j1b@jUb6*_&(FPqA$Ok#dPn`@-47nFrZPvx>MNb&naEqSS=jfI zKD+^No?fSj28D_L-u3NgIW3ewk01b@laXmDTeuzox6m?naRMTdVmb*dM=E8R49 zQ_lQ#04l9FB$McWWxQ_>m63j>`5dJ@+~tIBT!>@g3gVSN ze}LZp1Mc$dn~()9u|{_Md^3yy5;rv!6|=y4xZ6B=T?^smm&){WqeTFt?TWP!1F7`f zAcqM-(VzeNpJViZIQOb~@FwsC*VJ=AzeHw#=1Tw1O;{k>hD6DL90q=6f*9~VPL7UV zfC>NeK}ugmXYiT*7ZJd}P4p9B(l(~2GtNc+59e7Kn+X;kj zFY{>SsA7nuO{yX9aQ74;0GZbLykTshW;Z_qv5rg>kI74y$fNr z_fJdgr37JP4VR*j`8jVV);bRU#7A`{0x@ol#BYEP;rVz9!BI_a)h}$h{KinZkz~gn z_A>o_*JAOPE}4XuWsxvY%qphE)z~DQxx%Fii~ZT*ruZBP;5Rw^m(ht1Dgp3=!JLUp z7bCOLG~NHcC4O036=@79DxQ-W;uC24i5t4X2z-g9L=2#|Z`ya8Zy8MOXjoq8FKjnv zlkvJO>Y&F7%QFmDyvg$r5|Y4BdiXK#kx0kP66MqZmjylO^74t6sAw+%H@87@1gn)4 zv}Ws-9lMk7t>C6$m?(0IC;IEl$tJpCGfZfbVy8yB`fQD>;7{_wWq`MjmP7hoK8+YC zQVk!si!|6scTYXnfZrwwa3T$BOw_JCz-ZNuuxgOKI0E1dnWKbchd?-AC2c5=V>REC z;dvdtaMreYrE02KdBb{KJ%P#M#!nttz)#RNn2!rD}`Ng?akFFRFTNqahBooXj3&cQzMT##BBA}6=42c zKQDwj7b_%JNv}Z9may%*Hz;2xam@;uA48Oy?=r{iZD{G&dihK2F-Sm{-&X|Kw~wTG zK|h{o5l15|;FTuF6AgtIq`9agbjhpXuKan&hbzEA%zd@Gg_g+QDpseXC(fDhMw@u2 z8bJn!+lds5OLDy}U2M3PFQdB6pm|uhca=x8uQvqS;Y$wVajNFRjEJoY*DdDroX*pJ^Ng)Cp!J$~Wy^B?) zduB$0Uti41aaM8&piCaVrQ=k4hr*Suc9(%#49q^-kwncN-!H_q^4(E3xe`6+-=~k*jf~WENFs9wL+6w1R*UAP?#4J>-D(<=-S6&I?zrN=3(#SF@Q^$(& zM+f1Jfp1$#V1EmlaYMa5#&TBMBB)Fm8Q+S`_Q-7+q`hCo$Etr_#tSrmB9ksD_1g5R z;d38wZY5HN@Y#+2rVzqRgV0HK0oyEVl-(wa4B2mdO-T9l;Jv>Ner~nAp|Vuew@WwQ z^122ZKDew+aU+k$q{z**5H}~jF3$<2DPC#VZeFJ{3tP-QYOOc)!5h@c{*TcO`Z5jjH_STB;w3Mw5ja9KijaYp)k8loAJA~F_tU!2nu@O&D-{UMQ z(9(z94{OBE`{ErFf|g-rh^-|F<+h?sGe_bY?SDs^calnt*6;jwPh{1JlOQ zgQ5BNavsOIEeOnZrJq2sa6!?|8ad(eahTK$pTfSl{=7wdH~TFl>L-~T*Z;~Dk`Y~kC0 zaX`uK3q(o5`b7AK$;?&yB{=6O=KapK z#w$-`#4CK-{7L|2((J`bWU#m<@9z$>Tc!zs-y>5bB zo@C;6n|Mus#(|bNtWDC=ie-xW8`jZMe2po6e2kz~Xm*I`sm1V-CpJ$A#PYH!y&V`T zkY9N?8}(xBN%)tOq0LJ))Vnf)`t6&c91g86jo8~fw?r03i%k=NBme74Z@Pm#4=999 zii0K5=h_4R>54{WN#&FCXHfHPCZ$EYV%HZW*ZruHK7AdxW1~apP*=g=<90I_w^gS2 z?kl=4^>Ij*I!1+p_}y`4mMrkO2WW3_&kp`ecnJ@cmBmK4p@Oz=RW6V~1?d#$_;tD#R1LmJ1O{-& zsanM2C1MI@M?z|v)Lb_Osf!iI=h<~l1O(QZxi1J=xX_SHr?rY++r|})yRr40BO&NF zV*}uf^^^u#%KXtZ@?VbZA?!#xR-O}d95;>^a~c#!E79(p8$hg@mlW8+vQ>3CAHM0C zm<(xgEJ9h-+jdO>pFCgEBFIoXFvic3jh@WXzf?;nwNHy$CZpr$a?!8viR7v6I1nes ze#5=C0jo$#&Pa z7lzgAzr50NV5&@8ZdI3i`|)|@QIa)!(vHoWj$w!0h+98rnR|p#a8?MD8X3$;!#>m` zG*0F`hX}@F?jaJkAM2F?yPTgOx0`FIgcHyE+LNtEUFk;K?w#>WGdd1EciUm_U7f>d zqOqx2@Q&?AYk!CL3Hi$+W*M!keXb7JD2T0}vHDCSV*NHCSAR$~hZ974QmX$&)Voi{ zhP_ZHBcOC{BlauBbVzCQ5_R~!)X`%xgx8j8cFjjDkW#J&eTTDR+Gr_KN*;JieT}yn zhxGK?^*4L+--^?TL>Mjk^zY!d3oww^G4}16EuMTDTiMEVAj6L?dP%_KRMswSMAkP9 z)s1C&Z(iC6DK%erMLW?L#_AE6eQfHiaT@DF!&@Ws3CA%vr1nyjr!uOtW7z8ioP9b% zm*VS9l|>a2E{^yliHeiGrbz0eFV0X#5|1-P?M~7yQS|3} z`4?Qgu0fCMysVHGnFUZbAy&?m_(Jr>T18Wv@! zy!D_A&|UFoeg6sT$u4jUk zXluacsgYe&9F2oXgKnM|i~Y2?cfiThZ2mp6kkD;TLhF$c4AE!U59ceIYiMUC!e{G2 z9=U27)q@z)TGsiXA(Q)PWKlxmhXt~+9zfK0pojxVikRreb96*3QIUr2QhSHEAZThA zSM1R%9*9GywSi%-T)^`@Z{m>L^f!n!$CSdJ!IL!8-WB;ht`}X&^A+59UiMtTu0#A` zJthSig;;iG*QEF2-Ad$l$R)J|h~_b(=lVJxlsHWpgHLQOnY$4Y~=W`bH&jQZRqKPxOi5hhov>K z>Q1|y{sl&BlKWDarO2+Ks8~873uQd-bNTu3v`W$yGtd%iDa^k3Iy~Dskgnw@cl_lx z%+nR2hA3Z}*Yq}7=Ef6x(UQFkz|rq&qsKhJ0FzTTaoRKLvr9&9QjREY<=Tiik;Fzq zS|SIm1TRQ+ZG)?@Wy1_4byxi()XreZ>7AuS={ocOTFrSQuAWBX(@+mss}fgAMeV zDsQ`|wP5^|&5pLGLJ)Hjw+yYZ^JSWHp`B#OfSSG6R8~&gV)4|~q0TSuy@yq|Vz0TW zSx4-T>0b4b;}yFLwC}D7E$A6%>*qkvwI|pl2Bo+Abawh^y~oWGK1Dx}E6NvmM)5t1 zF3oMEG!G&gEM87W~7zYOAw2gNS_6+;%GKHY%6aCeXYl@xj566UbFfiVyG;z|*d}TkpZnb=k zdh|?yM?UAZdS4)=V92}FoBo0GJ*4psefk|jIO+4gJ7#gw64*`kUPZ(*}1&(VI#U{PUQs^GAPFAgNOT7Af0xx|4VI z>#NHfpux0yU26O7GNQoOp1B^k>AhRg%=qh;h4F^c&@4k11D;DK5^FB_>KxCrqW}|#zU+!t$~2v9FbUEc z9~?`^YkcChV9uL$X)KaWy6S=cK)2puJJYIP#Gx-TI8WRzv~5=ZWGWwB+dF5PcZtTq zKPWPyy@B&^&g+dD;;2fvsj*=oOD3dwlZ9{xKsg?~0n#Kh4JaAAR1~v_4@kSlBt*lA zrlr-JE)x}Nzy~K6iwJA+7gq0P;u`X6Z(--U#?l=h;I8b>JkWm#0MeC}qlF>mlmpvn zr0QEQn;svYb9`@Q^Ae9k_8Mg`PjwxJtlnjLD)wJ$=708})v|!fl#Ju=h11<>3>hDnvWg8-^M#*k&9rxDp8RV#u{GSBIm+8v>|} zQ*?l@CPUHl^IadJc0O8K^oaurSbU9~GRQsByd}O-B8fk=YUr>#OTgoXp(dM-4@>7a_GN#?Pwz5sF2P?Jdv}-wro_mm@RHw@!?7K6e@532~ zT~N=zWED>F=;;CN3;Y6S)k>MRTIAF#|FzZ?g@kJc6m|}-nK169lM9#jiz59(>vtvK zQ~n%Z`KQj$o75>!!+4T{JvH63CT~>XH@UmY<>hejrVtj_Y_OQaF}O9~qfKE%$=vg> zh6SE#JjTLO4)e0TLf)_@Dj{vcoAxMZpw^?T_{nlXD%O!Q2lG!L5bg54qEwu}?m#g8R=8!ks z2Znf!;JZKhrG(e#^A!&WFMk0<)&7 z`c7#wk#DXgnSx{-l6upuyPbl)Caappp`o%QiU?9yc8On*aFy`2u>O7FXOcypEOu7s z(-~!^0U>#r6LE0kR60LGMY5XcJ3lG$y@=1h{L5Gm!21|j+m9@ivGcX1_p}jnzf~&O zAq^s@oS)69=e2Yn5Ht*qKW)XuK5o~$AQ3;On74hx0oVpICIlw*!^drfcba9dCy(c_ zO8e-^#mJhfeDjRT9 z3{PF?uaME_lmT+soi3||;<3U81LB-06wRKUgR$dACtAOoF=DKvncseta=g#)XohmM z8veuXJ8L~g8=`HVHQw|Yp@VEM-&DT!YF=eefsPNnpt}`2V5LrS8Ys}YyKc`pugm8l z!WmO&9vLMQ7;C#~_pHedn*-E@cshVAIAaEv_(QF#EyyL!Yyeg`YFWs*3 zxpHZWPuun#yVcqiMx?_Qdhx4N3Nr)3uczJj#6kBNEfIzzGiHsLrBWoPQ^5Z~%ciWf}@(jzL>myS#)ij-G>7zzE_IOd+=0+qw zw7mQ^LoE<^{%fWG>Bj%??-xseWF?FK^+*zst}NQng7`*am*nmGdP`x!m5>E7G`2SUOv(Zrzrd6b)g63`{Is`)SkG z*?oUY$9epIP~Gd}PZTpWfu-AgzP!qC^@_}BOqlhkR*+`nPy42I`j_|9>uf3@y*`8R zItCK5Xup)_MH}4}4RUyFdH#pvx^I~{JR=u)FN0K| zImeoJpJ+=L{qg5!Nbj5R$QAvvMORm4geGMb9miDjb1N3U<%a*~CRm_!A;myLxjX6? zCj4tATK*S<#1{z{b-2Lvqh0%L-_*qcD??07tmXq0oI15~o*N`n-}yZ?=I?i?iv=zb z0s6>`of`j`@OSU??-cD{OZA^w`1|tt_kjJgkp4$L@1KSA&qDfh^Zft)TyPl3gBov7 zdI&ByJSgvj)w~f2VLBUr9FcFY=C}` zJ9oYd6z&G%?gB?=zmh>N6sRitt~-#6sz6BFD6LuSK^-FyluFli$MLJ z2wn+kKZ$J(R~NKsf3zE;D@o0gG~Iyl=eWi?lw4ut*}d3RJ@}B$fGrw$MOQqwoe`{J z9X~h`ZUvk1BUtUd&YV8!n&EwEaIc?NcS!F6P#0rbN&CovaQ7C!4}zk7|! zfjBOeu|-*ysWNO(`T!ku!Zp$2gq9WzbuBzh4JCc$g5c+SRX3Mfjzk-=y7L*7NL7=R z!(^=ZZT7y9OqbIATQEQ+wHc_WMs4RwA{_|55oS#j*x9Ol=lmiJL^Hj^vyo5Mb2Cuw z32YPf#J0U7l}J};*?&pM$+)#z<}6^|Spy7&m|d3f78*=oBiYhNDR~~{g!iee$-r^_2ENv3$(|KMpa_5$F~#kTW6A~GmUj}XoF%43#w zyEZYLs5^0MR&-7iSA!6cON)kXtmt)O_PvCqk7WC76&6k%#s{I{@^Cm}1sjnjo{((XBR3fk zM@g5>u@grMoe2C~M!9}kKtqjND=TrvjVc;R_G~{%T%9DCPcG%OvDP)E_4%JurWaDz z)h@r>Y^eSO=H(nGsck-=A&2!(bZWt3Mj3ly3-9sMva-%5^Vg;*=stiLHg`Mn1zMgt zWgsawVimz_oW(3MF1Hae(k4FfWPew<0u5Wl5b?NDe@=chgw<22oli%wvxHT~MK5Z1 zD}gtbRFVrSy5%0r#so~l*Xvc~J0%+F?tm(cY_azq`mAS;Rn-TAabJ8UPD(SLKqtAo zjSN{OmWLMNG3hB7h~tWe#;mWxRBtxzT^1D5+h3MRz1Uj704-^?eI+xDJhtnd(c%>V z6g>QQMQ|Zx_ zQx)6US^P28PAL9NP;(ISUb<}wos41fbD%%LWG47vz3Cu{un+~5|6*iP)}%;7Cfdni zM!FxBmY2afQO!AT*dD6{sLE8p-Rea>wlh}&u+U4N11=9oS#1m!r#5OgJ9#^6=Xc&K+J#za$`IB7`+EC0*%gL&r0JR5+yo*p>gY-Bz~mB2z-5$E}MFk1wd>*_mMLFV;{y{;w40 z5l;7#Q*fM~4QsVAmi47n?O`=>dqz%&A{Nw+*u2+%BKR3lt2a za{H&2$K}-&(^dBZb7h-nj;CWgizB%mHq*uYJ0)r{o)K9Fh;l1xPX~0R_hPoqGaJsyDLpoIB5S`Ftg< z;s!)gBHskkEO^vE%FCxNmTVh{d7GowhxHE#5E};9TC>M)B15G@=v-KW|wa&9L3;-`KuRotQ(3OJ!fyguOBMC@DtLV5(X62OlD^@GVAcJN0 zVFw+UB$IPW-DbO$EhpV)#t!0^=0-orgXr+O$C;dVG2P~RWxby;Za$NfA(E&XQE$<> z*i&-FXWWSDqsR!tXc#p|b(fXi8`nK1XS<{wjxA3B6POA6V1`ymjOZjwNukv0C0?B6 zu!F$~^iM?HsK&{b2%qiEoB|S_@JZJ+xko^*H40as zfgGS~G*)#pq=N9#Ds8%(uRb5tP=~YAIf=;l^vKBQ%NTT?iGcR}LPbwkde1E2>y$?z zWfC3vL4Xh8aC_FM99ectu&)iiya8b7n!lO|1R9(9bBmsgJ9V+kaJieC_H+AH{U>v?B%ZeHR;C6)K_g1ksg3__b zB2OxQuYd*kJb?fpqd|Al%|;KIm0Pno6026OQ=oYLy3JGjEwZjB%vn7%MU`TdzkyR} z`{ac%?Ibd>NiYMJ`2k4-p$6gFwvy;h+BAMclNqXjQ*^W?&|w)~DsXLz19?V0bM+n5 z98mOudkwd#&r_=HOq*XbjUBWIr1Bs}}b?L$1DE6X) zA#$VvTWIH`Cam6x7chu?uQMqMTV`msKXktazbZDwjUsFb9P7ze9S+ocUiw=&?g)ib z4T3*Ia0L#wOM6WNAt%`JPzjYz?`MF`CJcEDx#5zM`Ph*(Ie=erAjsO^!f~qJfaGhENuS=1q-eim z^xL?OQ`CDiw}Ci>y6aA-!7@~a$~xQ2jf}5Yi+Q^3feWN&6h1moWeLYPsep4SpBRWR zT8k!KmuZ^uNl+=-Y*Ir*+N=^i>OnR66r|0T1(aME0Jd=+iUq)i7@b_^YJTe3FYgcr z=q&oU?F(I~5Jdi@x1`sLH3=^ODN_L;wx5$r_loWfNb59G;aH*B6+^^_ESv@1?j6#u zIVxb%s{dMAyX&5|C<*u&gV9-DqaNZ(llz-45|6lZJ>D&0CZ=6uVwW0kY-Nv9c+rsg zE2n_=#n&p3r3xeNylpp$+;bQeX@_>@9&wFbDx|I@&n-mf`8v!Z;qHg}I;NYz*wzFL z(e$;CoDF@^uCsiDPXdG-AE>i-w0+UCfBgD%;K}XR))w2{4pquPV^(swWLsO#aC z!_iIy8jJ|rB&&f<=ct=B4nQ3J`11i=x@%B$M81>yal=dc4H;1-+!6;BCxglLQ3inG z`T#*mlW(AU{{3xNk#+5&!^^p|59F7u37MZ>@XO@-PBMkp(gr;Bb+zrNs^evW?({wO zY~b}}IyOcDe7DI$w;2gfI_Xn#B)7yCK5VxX%Z&+->`CUU!0|sgy7nxffIKM8gM=mG zc;}%I#j9COnSjjrq?72{T5(Kfhx=!(2en%>!x;un3MtqsO;?a_hZbr>|NcDcPb|Pb zt}Fj2TZmiGCyas2qfHM{Y#Ny!}I9r1Bf&gE2rB^JpjKn7o@ z$6W+>k{l{OCfzKPbkBR=S)P3SxXY|H;&w2_fT*Y(L(F8nQuMg@Y9qgCaAmyLQWPgo zt9vCuuLvNg~ja!upJVcAe_rOPZ%m!ZbihbBI8 zT7Nf$!NEw4Imi7z^f$f7RnRep3m+OieAwxW{U)Fb*6V zk3ujw5ZingW8ZQ2I{}H+lo&71E);I&EMe5nS`C?@$8;xx=;MDGX;J;^(vfiAYrtr>N2N$E%Lt=yVW=)(BuA!`n z3V^RiF{vD}OJgWcx`YE=99Do$Z-+A<+jE=6)@juHwsTQ~z`>V}yskXE=mAo5&kma3 ztxAF?UKSzHZ=JL$5P3L(X%DOJG#Hb3c~+6~%g2@1N2U#V#^%fBm{(SB5mda%;Z6xH zu3~)4_H!diWV!g$3n@3fYI{-dmJ)Au&DWsl_|CD~*w@x$6ymC{cwnHHI=M7RHRrcz zUY*6yH@k5?N>Fg{Zt^~@Lu4s`7hsvRblFnmnJ9Tw6zK_Cs$Rf=?`$v3Z27X8&g`k3 z->#$rQ6$wWA~UKy?}~pf@b^b{Gz3U|SC%Qwh603qdgV_mHZ~|sodv08p@83Cs=(MR zb7H05MfWbc-zMra8MpMt1%SXLkhQ|?MC&nB8IRmzvzXD#*)Q~a1r55>9l^9c3XEkm z_{P|wjX9Em{$TapBSp2=)h9*K3CYTJ8N$#v(`PFMEUjN$@bXS2frXL9J`=pEJ5Sc4 zpF8>UOG&T8*65B@`BKCU!BX`Dp1_5dMD_BkXEz`2rQy|SyRh+KeO!mPgp?Rm59ow- zQ9bf`N34KaZ)QN#?lD=t)N=5x+q^^9H@BKTKoj@2YK1H&?(6eXFgiBff6q`!Y+ZJv zd?P1X3Yj!V_7eri5@*HrfeOqj3dGd5T10vV(Vz_OuJJY`FkR>s0Q%c$Ym}6vtwjtL zW=Y#W&og}a%>ZEoi#U{=6^FY2UP-u%HLLkec&t^fUATh9B3v08zxGPX)VjFTa`u*0 z%$OglK-R4Ca{=BeN11DBv`VV{+~bG5!=&}a3$=#>d$F%UUOTKfp&Zj9{7b{lIi{(h zN9?JIqwr;s0jp~zqd~sFKRoU~1Wl$T-W2Iv8p=pG+5(hruzFiy;}WKts{kfVeJY2< zEoX@3&aAI)lG3c7u2bETB-V^u-ot)sU!AD*$k(ZzJJXXqn(l}QlK8UcD)n8xP~Gg>qVB)36Z>Hb!&5K%WsIo&$- zldR+jAxL_uAOwJ_MgVx`i)euG(^)~lkJuamg=b}n`z0K2pa z6ANp*>mLXkE@YBLcL3Out6R4%EQ6LZM0sv^{k{HP+|{L%GndZAoZ7|ST#|r5Wp&Ax zT=v{=z_VJ^N>#$yEC9#CEorjFTaIc^aaTVV z5wBy4#N}revhw1Cj5OExp<W8%q`SxY-m<6^rJzEOgIt^;<{r_hyh1<_ZxpMV2Cv z+6<4)m&R2tFV{w1%${h7O6k4-_;yJ2GuQm4nXqxjnH)@`<>*+)v@W0lxbymj_Hq{4 z*8Z>LhU7N!(vGaH{det&IYPyJ?e8GNC3rw^lJE?8K}dQFf$|5hNAD9}`*L7T>DydNHf}}bP@iOAL~t1O z3OTFTHelR0@E-Nee&co;giPFs`F6Gawk>+#ZS$IWV?A+on@9EeZ_6uYP4U-sqBU4| zW<6Gi&m}eQBU=VQ?)wChMLJH3Ol#!!-8i1SatOBBR_#fyGMilgHv0Cso(4+iRlTmJ zet)+X;f^gplj#|XPP}|7_AW&95awgkf(QxP_DbzZ7D}-`Jtv%rQv{{=zHpao(g!KC zcZA?5YiOq8EH`*~i+Cbo{%|ilGUCKM!<-n9{Q=*bW8OprkPWU^JsOwRFLDcAxQ5>J zb+>vWX}LG-{;3uS&+d{=W^6*PpF#vsW+say)_#^Sc@seDZRQ>Wo%>JWPoA~=gx7N3}li`V#V#Ac_CD%ZLIrgYF)ph`BN=;FiwLve_C1;fwj5-7?FROerhXiMXFBC3 zCBMbyUHrpMML^YE#xB(#4(yO(dc**NUu5+%lse&cgXQP|$mbm&ox({s1PE#5^QPF_ zK}Cs`FZe%0Du7}su-sWsd|U2!wkiQ~T4swH4-HfCtmWox4nF7upp$=}FXNLD+(Rzf z_0`!mtO59~T$}7i7RH{pYpo`Q!*{)7t%oKp_qr%OOz_qDIDYe}i8np=lxDsbfg9DK zQK@JQ7^X9vsckqDmfbuI8Z#ML=u+A*fobIo>P7LtyFBk>%up)5+Cw`M_0oyjE&G(n zP*-{&Xoi?!H_HwZqvqrav->v;Ee3WULc@p@o_{*n0=)y;}Sg)bd zMX6WTn+^}89QgpC%R&Kknd*9cITZlUtFh*RmUtV_UEqTznENbW>)%-(-{(OBfm872 z>EP|{)g;_KAe<48+-S(yommi4`+-En5-=$vi(&KkwqoP&bC1X|Z9>8ILD2wuI?d@$ zG%Fq`=1;XIh^55pz~&Nt^mm5`g}4lW6DYJHFuv{%#fw$35stPh8i0?nI9g>0a1vSE zUoxIzkt`VX;4iFMb$u(I*3%JP>>>hy=#~3S^|{uGrhE%yvdXOVV6A; z3mo^0bwC|Q0?wXm?Ura!q_QfoW~rPPkT;b$png}{!nR%)PDno&HdK_{^tp)*ym`3S z%6mDj;-%5>>+o10Uy6q(0^s^KBea%CRg4(<@vR0(zI#-(gJmKzr*-YZfV(3}K~maS zelyP7w8TFE?uB~%_(E!;0%6Y>Koh%w60D|@Df!|j*PAY><_ynhJ(257snPNp3oY1< z6U`N&?-6O+U|u;3Ab~Zw5$7DfdAp_%281pB&Xy@Tp3Pu}mj6;o+VtvvH)NmN$89mZ zrYo=a13u>yAf(uEj5boQ$4vQYtn31yp*fQzxTw!(Dvpwrwa~kH=|d%AQhSe$nPFnNLXS$(FknaX=Z4=+B=@@ zfOR)E0Nh$r<#GEY2+Xw-88nSl_0SCuf|sl9gOu01*(6&9m1*o=z(?gBjs}McV&PPY zPFkZh?Ko*jyV0uFe7;LRn7);8!%P2BzUjjJaKe~2yc1d2;1$*b6&N@PaxjS=@c=OW z(Z8|v<$omJyrDL`JS{GP(F*3Y7l^4zbo!vBoexkv&hmjDEK6zdtSS)Zr+97hTz9&j zJ~*A^VDg^}%?Am9>%iw}cmW!4N{t%MJHW!?Zk|g&aVJ^|ovqETRE)na;~R{NfVn&%z)Mu~-KHe!#7FJx_XM-m1II=;c4^q8XQTrgl_H}I zvt_SNj?BK{;YAM9fZWtjIFxu{n%AbDc3mqw1^ zxi||$F$X=IR=w?v#s}-i01@hqOIw~tj8ao!{$~fkA09MR5y8K{w}gC^1aq!GKusx} z)-7WQzW1o6cLx{GLVxo--uZ(93z_dG_!|X=DW`1RS@ug%@WFgLIs_zQ)|E zd#7+e8YY3~j;5LejnI?_Am3YerUqU`CZzk76cQI@7cT=wIp%#p*VtgESGfy_~-z5(i%-`HCWPJ~<*FKgW* z^*OQbHjONARcBZASgnGHvEAPTAO5Yn_ii7E zN_F`UAgXBI$-GFzHoGC)s1lir&pEuA9*1-kyX2YFWArL(izlF!`IifJ9RJYZP{&v} zoBZee;Kl*p^8n?zMC{)F+NJ{W4P-^Q&U zr`*{TzV|mU08s9Ep$+gIAVLb?m_q0jKKV?eI_0g{)uIlH z|IjM`v(494xo_=--x1CGuv<-jK*ZZc$m@sWbJ3aO8{k;`cfc`&!;eAN-yZTW_~tRt z@KsM~%g#A4E2Qh^_{{?crGRt5-3Y9;{}3&NL9vDPk4E)(CbUEWnfMQo_WK?2 z`&Rh}!udafaOQm#uj_p84?{@B&+aq9XUF!V)H|wKNH_y(9h$JDziX$Ks+9h=Jb34eCJ`lU1k;6yC`e-i|Y>@;-b51aIN-iYaEY7Lf6;M zeG%K+xdCuawRq=j&}0b9n!ew!%y8y>dtIKgqFC$ty=eY~DjJ)GtAMm5=M|j>U5h>h z5bX68wppG2r95Q94M3Ok3Jf2Qr%`*Zu&*!4^^qCD-T3Ex=$QUYziWT*hA~`?5Gy;& z^-dro<`}Rj5P-I4qD~cDevHRjn>@Ns0-VZ&RLsS3zx~+o1tz5Bh-DbNI9+asAiluIe zHsf|zQM>Yi7P!P3m$8o15@%qoTAx{6>-B$-yZ`HC4?Qew^yV-6vIrxIHaVfBk9zy7 zW-*2tv8~ehD~;m#2f)EAtIi25_J4$u|50|{hZX;ST|1kyw0-bTEWkg)-`~8+KT73) zIMqM3;Qt50-^;Mgb_yfgloJiO`gDx4z@>|?SUx_y$;EyAS&E`|ng>@GWT}R5>Dk3+ z7g-+MJaRktBE#8&i%Ls14Y2lwBuM!1f}?x5vdcIV8>^{UOl!4J^K`2ij@jzi=g*(- z+pkVV2#QLit0sg2igWMXvNR<$E0861vaX&7oIOt+IQZ8;$it52gRLhIebz;4HVpWR z#cpv=%!P($&enxa@|K&FQXW+BJ#g&lU;jK+ph{Euf7tuWuqd~;4H#F!qC-MbT3V!} zMH&Q@?h%mgW?%q=MnLICx*56|kdzz{knR{7h8+66aXd%*Ja>O?xRNi~VL`h@X-2Uy>c=x0pF;AM#OA{|xJ$r%g z?UNA*)r1;kwK6FUe#>B5CGf0C|Lw-WKWPq?e01BhZRLAJe3@DQ92lSYXgri>e}Lrg zV+KBpyw7}_s7)yl2-T3&7T%V|2OU$2N}q*ORgYOw+`0AW#&17zk{v^Yt| z%nYa~E3na@-}u{4^Zi?L-}uTWl>fLNM^^|msm0~^y#&kf6O7-V{>Hs2Ec7zdm&(;E zy-G`!kK0P%w*G4Yq96V~MK|u9r@k|@DZeLkZ^6p9xrH)ixP7!SAneC`W2P1t!r70% zU;M5aTk4US9%FZ;NJ!2p`dP1P>-&y>OpV^%TLu+$Kc*Z{mJHn}k}2oRwnv7GYrBKm-z$b^1ykq#Z&Ur~^E31YqeQ7m zp+M+h1g4{b3K0mGQb_KKQgGo3w%Xm_FTQb0F614Ft5&P1f{xRjqJOL(`;>Pi*5&4& zw~@R$j{I?`(;iU$+7A==MnPf9SCqe-d`yDuf^S4H#x^l z2OZy`$k=PiWg#M-m$z(Y8%GTZa&i8)5PmI+2*&q9c6|~SsyU8X$P&2Tx3G|ZSl9jY zyTqz^N(wyC)&REvKeIpOjOXfO%4e=E|3B`RUMzK|6+v%^dDJrQhxi(4*!S2)r+@{+SFfWVUlnPy#%O+ zf2_ObKj=TRn6OQ&HYYbgOW+YJN#X_Kf8Q8404Q-T+r~79|N2p{?^Yz_HaKE6O2PPk z{kJUyeCv?~fh;kh1n$)9=ID-AQK#EN92mdbdrth8?S!eT{c}jA#LgXjQ1vEi(h%c+ zu>jxuyO<=-S!n_@%R{d4w#_f0+#`R`>l^Jp&SJuY&Dd@)?(9~;U2rpj%>kD@MB#xN zxh~Cb|GrA!@!zX4NgQ7jY10%scmxxCiThtB`E|*h1mv&-l|l56J$wCBfGv)%ZSn7e z|L)`ewfMhF{XaYX-#z~S0p}*a^P@fFV49@BV!s1!eoWmol(#4Y9P%3y#Ua5cpUTB; z@;%9KZ(1V`V+zbk^ccdq46A&^pb*nL$oKW>KLR8k-nCD)4TfqK>t*}HiLm2taB*=( z3G>Amsb*$YtZ~2U=+$PHNr80pdDciiJUq5H(K_?5u-P9)1^~BAGA$e@+vb75(M2Ji zZ0xDrBhBf0m33;37M{B||DGtkCr|kmTT1?j>~0$=!ER^s=WinhA28=n)UYqJ+j|a3 z*xf<8TrSO5czhGOb4O{c2oDrGwBD(>Vg9mu#UIzIYE#~XrP22AsKep!0YEPB*U%*Y z7#hif)zb5DY5_?Ra+rLM5=F@BTc%HAfgFJ!{}I4#$*!kPP7yM~D49QOk!xE&bAT!* zdWpE#Ho9s@83>hj#RFxeDm)|nD-VjGy;*I|EmZgjHj#E0IfCy|E_0j#kL}|hW-u(W zoyTj{E<-HsYA~-JbS+l6ji#<&>AXj+-}6?x^?S5@^P-s@Z8l`V>QZPsJ1*|6^ZdEC zp!d>~WFe<+=l(#-(T3y4M8RWCy<{V?ms2r#rq958TI;;y=64wngYLmcH@H9GfuOi% zdbj@?TLW;<$zG)^8enW@jvT@uy1NtE6Wq?{yWdcR+#M!aVRSPqpVj?tu}og=a7>)` zc}&!J+~?iqLpKbd>kQ@oN7llIXW!F&oQ@K-Sgwu|RMe5$ny9oom<0)$Z~Zv7xajs< zpFO3eD*0$y^!3?{tY^a7z@ZZ<;J2zXh} zzRqVxEQ7K)ho(swT>vH+SEoppHvjG7>A02llJY8kObC*iPRF}%BO%MM8C!$lN z^$O;3$YS!T=Y_T-a1t!lV`*HGLpES9hFgH>E6PhyHa*qI(ICYyAD)`^iTmyN1j;wO zlB;A;J6TJ_KIk(mFdq%G3GVT8LOdqoY|ezo1wA$Cl0LzhmOI(03D^d&a&@Pi#gr@t zp#3)M%{}N-Pj?$RVC$Wm0oYdNFH)-4PJ|IHn^2mji$_x`hI-#+ha8*R9~^L2X>(Xmq)Rw$ib zpjkZdV?5G)*>gzlQ}rvUPi@)L0k6N+Z(}}Vdk*RIWW@%%sW&?e1_R9zN?FnSrjNzd zLHRt^ibMu>xuOK`!F>z0751m3e4pWdn;|ogM{uI{#XPQTbk6(T34W$^8^fc_2J4mI zm9k+m9dA6Jgh)d4(-|&rCRd;q9Ru6W2XmL7RaKkTe)cR_ZM-KmKm4G5a27tBaKxku z`!MXXY0bd45fYD@D4W&N5pW~BI6Q-AyDYy+tZgk>i(=QLYCT?Aure|6SywO$f|gh` zo<>R4|Fv40$*w(-RS~xs#&vPThIe7Mmh;8H9RpMc%j@ki ztzEpMT?~|KCTh>{{&G#vn_I6Jc4Pv|<6ywsQd8tfXMyDDm&ZanK^l-sBYpwHV_Bh( z;JpOJp-fs>u#MMxTj^)F1j!fqvuUaGq})heN*R%e$1l_5Q_ibj?=Vx73VQ;}MTIE; z)wF9PwmL%$3M27nE3OXD-|&lT2gX-EQ1Lb3x~lra_n_&Enz6`{X1!uitiz&DiwG8d z9ZUsy1Pd*Cfvb(m0`u;7VgC6M3<^rh(Vp>epFj!R)O1cUgY*~2>k3tRV08o9`|Slp zK1g-+Iq`Bd2?{AznB}z}P+2YpU;-9O)DHFX;i*wyN5x4`B~LLz_oh}RWw@AbN|}TC ztO7~@P;qk8Ans-~%Y15u{Y%FPHi}O>BNO)CF`Kok(H?zy1Jk(nfs2F{E3LH$TO>c5<}RP8n-U$P5<5&{3-QR}gGgmZ4ky1Hx*l`ey&% zajad^o7Gt7nT)XR5|={lFr2M3_>&KZLI=wSjL;4r;vy7TzpZ2Uwty?@kxQ{Y&7*}^ zbyyGcQ2Rv8vo?O}sz(m;iR|QJImT>)R%@nJki%Mx<$8-PjEOpn}h_KggQl36*;?K8Jr@*XVrX}yuiU* z17}pswy4eG^mes4^RbGU95NjAzmgPXf@^!D{lW_V*bceAt&}5K59QpMSOI$<_`!gA(#E_1nHwXKW! zquDLyuHH}>iY3k4s(M?=V#rMGP=4~QW*I4>F;OkcM@wsj1V4q`<>7q%VH5`_%&JZ`s~1+b?52hT)i>+_^w*g!`8kPSg~2GvyNrEm%FS z?_#bun`uis=xQ1F7Y!V>>BQ%bdWQgHp6el~B_oHDNCEKU2{?GEwR|@6-~nY z%av{uUYlviDG=(q8lQxTR&LRu7lP%(CPhAN`Ce07C2{;Z8 z)D~bHIxoLdw9*(DexHop8C)`6*tv^>pSNprHQephBO9nNZwI1>ZI0*=@tq1q4*Si4 zo0hs4+O3t6Un#kKLbLR0Oc7$ehDo{y4=D${%kf?&OZR00aXmYXa-#S>4X4dY;dpdh z^3GAcEV>0(gRwE8uz%4f%CO&(;G0lj?9END+cc3VNSa3@$}mg=5;|-f{BwIvtw5M^ zs(kVZ`^D`pd#9fv3mX#3A`0whVO%_3NFhS=@l82@c$>xpR~;@wP)~zSq-E7A!Milw z##pol;P893V#w{`QqzSEZqCuKI(9Dn!_is`tkx6u+$4VkMc-B5zlKTDZe4>ckxoFS zhsKf;Iq8kk3Gym)(h(MGFBmG6HP4*i7ft)*pgl6$bXgMQ@3y-vKhpiNJvCe7qVskV zELB3vC2v1{e*fHUs>pH3yd4A~%4oWpRJgSstN|&SsSuI#Rx779XfAt}#%U_;^kl*m zA@lTUrOxqJGZqca97rkTW*^xI|LR%N^1>P(!+u)L z#>zX!z46kne@i8oqUU8XX{=1vc^SnlqqszqEntGC4>xBVkHp8xiYF<7(4(?;wD?lD${L& zsvT(>xznrDSR-b#QBIugKBlIvuB4*LKq88~fo^*7b8Rp#K#ZhoNx@Gp3gD=xrVArpM9g&I z3;DIPLa|rqW8Om~<#(-V6h>Rb|6U+Inv?pUQqRk-<;|({AVn))wo+nP2j{f8Y$*W{ z1@opKgibrrdBPHqy48;OA<#xFV^b+e=+FJbxf!3lA##}Fkj|G}IUsa$L#oV1sfQac zS^eQjc!2)GaL5BG7sB@!g#6t78Va4)B|g9RA1CMd6;%55^a&zOE}GHidt_{Xf~*26 z92Urw>+UPh)pL@-Yi3I(XvWK+kTPXIGis$d7b?vy9%B)z z!@eKXbn)^cp~)20dmlN%W@c4~m9&3&!D}dR+ZYKHu(@*8G+%IEedccS)RNp(w9-Up zs~R@{N%OqHWwRys-rusa8|cMozd{N<%=@^#lPAyTZe7Qq<3T%HrnONZ)RgZ{g)2NX zVShvpd05+RH?Ywsba%>Y-Vv5CUX3gKI#>rc?oF=MNW;T}Zxyfo;k^I|4%8xp`ibOW z1>OJR+Z7pI&aW zne!-G6ZxV2r3uAuk(T)&Xq(TKQJ=J0d@F9oKV0B1z{3Tp{-S34a?$P7dQJG$@sL0w zMeU(gsnDH%9dUafw@hRUazpt~dYG0_6JGjzlGK^8#sP z^!vml5sh#v!RJ19R}63XU4R0WCjz@}?7{sPOTbDmld)owpo?G%S5IHxIu?;d^d|w$ z^krIKIklZu(twhLZ2f7EF5Rd{35t8Fiml>sx~@S6glm9p_~O=j6OXJ8GVI7CW-n6_|5%C-DmDH`p88p8&=a zeO4ZJd3iz7t+}dXf^KDU`q{R^Wk`@^PwRf&{(GaSv!5TLxSWWW6(Q%}X>_U=&?Znp zTqMOy+u#Hz)Yt#b)>6nr;wre2kx|rz{lcoDAiP$fFPLH|9lnafE;uzNP=L7nO1Qtu zD>@<11tdm)nDrJqrUfTEhh7P_(Ks_8R>fx?34=%6{EhosPJ(T86Kpb^*SPbI%a{8V z{I)p=25shvkm_W~?`^)R5V5H09qDx!&!CV5b&$1)G_mOQ%CEtd31p%hLluHs6Uc&j z?$VQxvb&qnteUc$vrcA8ckO``q|fnt_p`=VjZ`LGUqTR;PYLQ9%#lCxMSr1DQSQ;F~^LTk|WIY}xqWwvR_go*`s7kq~YX zat*%C{zItDUSjt)sPW5c6pCf^X;12dLLjMF6Qc7Ns*A=fqE+t%(nxYK1It>R`*5}w zuGwtkY^fj7ego^8t$yJmk6}6w$xA=3S8XT~6m=E?5uRn#v1R*7b){;;ieu3of@=hUQdv~Zxcj& z9R}|LN`XJZATU8(w$+f6hdZu~)@yIM7V`OQTTk>X6R0>3*-`}Fi+WsbeQwE1N)kC{ zXTh~h^Q3xGWCdUx7V`&#*|R|bwiicU7F!X=ztF632NfVWbLWbZHA{WrozdV#Q%FULGO-VQ;RXP?sjmj!*`$zOs|DWhbWH)+7%`QHYP>}(= z*5o#2f7=64a*1b!zun|HZy)JIau^jB`_qJRFBO?*G|vDc-R6#>(b=BO1Z-p#Sx`!N zrqjaKb9E8&X)w-~{aIh1$B;k-5VtJH-3sNKGB{0Liy=)GeHaORxeBk+gmjQZd_|h6 z-_1$>M1h<8US30#IDgOmbCgrQ+9dKb@e=+-}u=V-Jr=5{{nXC8KIYLrktu)tHSV71wc5KcfTnYP?TSus9vzfd{(g% zP0Dz&Z_r>L=0vibc@``pw`?_xIOhRdyeZHLVuV@R;ENLUTTHIlsZos%^-BAWW5q8% zwhEhK`Lrvc9G!N{Z-;A{HtpNnoE!T|3p3renu` z+N;}y0>|MXpG+~Iv!KiI($FwU(dJPxaWk-ttcc|c$C2A*xZfZ zYdS$707E!~p|mCP6JH)MtElMUq_Xm<9|>2#`qpy%gDxv2hXTCk_sdq5A(N$^2b7;f z3jnOjw-wVTZL?XmGd#bq#OO@+lz6<*%PV~h=M27k_xcyrrh2k1pz>mg` zR11b{X@r=@rgUj}J8bF<4r=H+riJ@K)IK4^Yx7;ALLV0D!5~4RAih-C>05!V>5EuF zHm1}w*}Sp?NVJaiSx%-u{2`snZ9>rU6v$^RlaEy)A$^D`jGWJTqaD~7hIYz-&sF`Rg9LIKLo#^#6hI4L$ywWp5A%(0S^G0{)IGQ@h)3 zqlsR3zFz$;fua0Gb>F*&mHuCe@~gfXj6f*=8P@Zwzg$|g(62fK@4615YSRo@OZzWT z$?l!&g%JpyA-zg$B*FvP9X)(^`IkKAo!c+MoaV;=iEiijWPJpi+2p=)arZC0Wak$? zCUN6Bfa8HEvjhH1x6n*}?RIHk9>(W`7=O*+FM!GJ*SCi}uQOb6h)Wz*KH6Wm{^Q!~ zOPT{v>!C;)*1v$uUn+|083wqDoolWF;8Fhn8-cp$?~oSo`Fd^u7N#-a|1y0(6}Ns% zd7ocD6;K2oQ2qPh|62UdrSb1l|96l7<-h)QZvT4qf5X|IQTN|C{%->FpZx9L%s8|Aq2x-|uU2l> zc^qOK{vnT&Nw30>)@9o5Gov0{x}<>pK`{I9cdC!Qc&7HFuKy`EYV+YQcqk{}AG$E` zV9gbvk>M$nkf?J#WXS@cA)uMl5RXf(R-Dx%y*ETo1JdY>yyXSp&K3Kk)E1*hpf%K> zg!4M;wR}7a?{Wi!*fpm^5Xt>ESmFFVu7h76xmbpSh|b`jL>9%vZO`(28W`jDXIs>& zgrK9n7?D>x4uZ|kByOtC`K<2vU+n709_Z}Bn5CI4-v!*o(mkz z4z#=HRm_F z!F=2X+c6Q3_5EL-W7Wo{u<3pYj%HB+WOp<&Zvh3;>LRY`m-DO5HPt8qE7!WFu}HNv zQ@ZX1y6UxB0J6xO^*Il_oVg4+Wk2`qggX20&@g*tb7bn;HLPOy!h?LaA= z_r~xAJy#xNDaO9v+w(8zOyQ7GZhRGFWUT7`hB56vP>{+c1A1&2$yRx$XpB-7^@7y_ z5AOCwEDXTVv*Z%P0Ir14ODf1YJKC*TvaW(De|ecCLdt>Jge7?%yZ!vVoSgi^##&_8 zm{L!>>x6pM&{28v%Kc&UHbvRFYWBRbj+b40D<)N3@M#`&oNdK^_(jF}R8u$nf?FTlB7P?Rr_iGR29X`Dww_n!@cwX3&l}=5s$?uFTonO2JjGfC zK?KH!?LAbEcaJu+`zyNxeCb=R1u|<@^*e5NOdnMCQp|*0tH4|)53>P@2l_A`U}yz@ z0+2+g*@M0&jvQy`6^4GTqn{2gDb&}gC@|+J>SnT5^LN?lPIx1S&6f}?YjuSneBk@@ zTm_Kn;tij)HnN^K4f-3ovwr+V+TyBrU9&q=i=HdiVV4dUwh@4!2))4U*_gQ+$UuNy zc-FF0cK&Q^hyLBfD#pAZD8N{Ky=JaHnnBgzg@D9mFrhsy^);@rL4;d+m`dW;-fpv= zm7%0kv|(vx=B~)jJtt(@sV5n+#3Xj|i)+Z%KA){liF~-dqThaDqb@pRH5tG0MgM1j zfXyi0I+A9L&Eev}z4d7B#vesL4weC}#{ADSZ(7&2I`xYU+(dKt0O<#@>`bA0B}aAo zqw9W!4QM(=tjtuME9|@wj&RGt(fv;EC{rgs2?mWG1&d)q-ivhDDTLcpdi@C!6K%V$NRgoNd){OPFEld_5iE@6>r z%}wDEYlo*?$h2+pPrsm-Pk&0b3b>zFN*QS%cx5g*;j6EkAP>L4_d3{)^H!_*PN%lp ze>NuZm^@1<(Ez+1NK?0d@v!@hN#RJ3$NUIm8Q-%p#MjLhVCV%MmheL3nqoko9Z3`L zw$I*dVKcxCc0^|aN9$8QOoo1k+osZIR`ntor~S(esXadWxfYR&LqfA>U9~~s`$cRG z3=Bw@*(M*=Q9;hv#kv@|v&76p0nUp8^})f8z8?cpStJ0>U5HAyQU?0O2g zV(hCcg1y=1k5BM(33 z(J_m8u93XuoWzPR-?cOLaglw7JOosFt7nI)Kaz2UUw|>Eii>H|cLpWS)+RYMh9(&H zqe$7Pde~1t!Kj9Aj{^Br&3k~r2bvff-!qLU&hP^8PRb1pKmSa6n9??N=T3t~)Go^{ zHjH~UQa2V@+eXb-Wyl7QF!*YGWF#40$2Y!J|nG^6K!~!8T2NRfIBCam(3=`A_lr z3uFN&O2CJ|-Q;*V9LZ^1U!BPg@>1$DHn!LdAb(c3bI|e}@QZ6r_A$g~!v;?ha+2_T&o@Gzf zF#NDxQo-rOx)wp8U&{$_kwoQt(hLCZHA&~05d{4~eO~}_Y5f#%UZu$~2Tlwgo_Eq* z(g1MI4@K5Tc?-!)X88;u4bX4^Rn7D#6lsZTKD5`q+5+C64qTZ3aka&;ek*NH-k%U8 zwfp(pSmXe)BNcJ(`c>7*8MO9JF4sCYJ@uH+|+NhFBuBD@biRi@@2Ml=wk{4C~YV z`WeGbC!)1)qR%OZT~D)c`V*jhq8d1QHJiG0UfE-+oDK-Dm`>{L-9OjD^a|qlHi)3oL5hNwx_J#GaLCmH($Ok5iwaqU2TKE1`A%)s~qD+$f(TvU%usEet zbYt>cVR}SXN$|4$h!hpOIG}pvA|lNUrG0+>{Z=b%+GnV=^{$$CYhmqP{!|{ot>chY zJslIqj6t)UuNbUwoN-nMH9ZU=BENc?u1a{iTpucr+7>k|&ML(sf~l^<+dtju(<$aS zobmm3u$C@hX)xkZAmjyTL~OJgwZL_Nq&Yz(dGs;Vs!Q>lyV|%XHg&K#d2*H1&bP+? zvY~s~eH5)dSMRW{d$G3=om|0(Q+(XKWUq1U;CM%D|BypvTh}hb%%E7Dp{L1G$L`6D z>qc})mlyrM$HwT!BBJQiRgX);U==kCN6@a)-!hSL`)&A9f5t3V1tXrAy?lS92`GDym+X}_1_)*o zl=7(Q=G8fpTz}sNKc@xf_muk%Jqx&=k5eW+XJ2tSchz78XN24H@*`@2xK9(K-`f+L0s*B+ z=Nic;TF~*SfP3Skm<$8`tosUR`(ahZ5s?TR$90~)d(-L%ALa@6PCBVG7Y{>*8@#`K z8s#){zgWP4f|86q4=eJ%x~~~wO*-X`r6}|r>B)QJDs>#`0l&~3PpNM{nzE=ue4_C) z!-2Q$kB}Bg&VK&EnY_#!ytq#110Ovs_^~If+_Ntng8*Z}_c4Xx%KO4iq|Oalp{bk- zGP6Yrik??;-<*pTmjtXp(x5OQcel+Kik5SX}IsV}uG8L;B0F;z&^OF2UeZj~=Os zY9m{Z)2+^^b6D~xn@v`=s%{I9{3&gjGm^d<3+$TS8ec?pYL2=Sa_t!NPc49=dv=e# zI#;xVdwQ?7L1Nv5*_rT_oQ&J5Pmzt@DMJR7yjz_IH1QUTofdiLRyklr1{$8#@gc4T zrAXamZjbq!201~7lcFLvlS%S@i#grNX&zh3ZKEgf{s#wZ0c&GaUv?rL5xd54)K44+wBQ4;Rv8O0h0BD9cB&anD%mZ~A-t^zV3;yC z6o=M3J_o`@?wn|oQ+X%V^a8V^n&#h=KIW%Hc19w}3@i`LX1BErIu$xL+}k$XHNliS z4IUh(-`}P{b^2q^tf#vx4le3=)(r%FIHaeUzI1W z9Z6?6Pr!@~Z#jro5E@dZ$z0oHQ1TPLm&#gI;Z5bDM?n{blUH_yWOOHu0u`JZ)3Xt! zkL}(ou>5%5g<O-ZyrIwYUi~l;nrpL;B8-gfFM?) z>G%1Eg)&BQI6Grn_us3u2|f1e=sjNm%Li$yyroGu%()qpwz(n|YS2(alQlh%cWz|z z;6a`J*!K$xQ5AVC;Mmfk%taV|!@-pL!D~zYxANyL0hcYto)t!~54eeigG+Xi1_OMD z78=Q}HdoOeN)Qlx6e{-3Y{+Mk)gr-eV{LgLok3-7BoMr8f9C^pv`OH1kR4vID!5!! zQ|j9{gOl!IbXU%pk0|D-H;HBctNQ4J2o?MLGsAsP(sG)56=^4bS+LbhLZHrIH;H=9t? zrWP~jrx%(PkZqocz7XA~*T?KK63bOaM4t`h_B=XNw&BVh4pA3qICLPYQw90M&(uPr zSF#GdVu3E6%E9N`a?FmiJUhHUhPfw$Q93YbkpzMC>V-l$KC{bR!kI##KYf_BW&i6> zmtk7{xrV-0X&0OJKI@@(9WsRR=Oka31fJAcIq|({(g?_2_*|S9m*sw29;fe=@-+Xv zS>?Qa)V$+SL@l1;#U|CnP+Bc(Ue|wPO(eu?pU-b%;1M~0#U3E9vgFcYEu%3uN_A~0 zIiw~BpG7_AAV)ipdNH1jKqMNVhnj9nO>$niq=IV&`x2i>27T4RN2(K>SJF1g6|qFW zqVa1dRNEF{mBLhK>kKw4@cyEeD7GiWV>q7uuB62=Y-OC*XB2(o&8UkCB)SjB5#DXr zoEbe^Ljaq_08yabx;{Ms$lg8GAw}(mtIGncM}e36y3+e!RNw{2t>IvafpfXmVY|pn z5#<@Lx=$%YZ$Og!o<)6BieC$mrZScIYpdTbEV*@GmXStxO9eN;-5!c0lC*e%*(wRH z4D`~1%RrJ(Vvcu*DOj!^G~vpgEl;PZH#yw=5?2@u-2=|Umr^-kTiPjm-DkZYH}*90 zv`SN8*hDqgcZK9kMpC7erp&%A*oBoukQPQI1o!Ebw8aJ3}Ltgl5*M&+bT?Vxof2fV|Ne^vhGkqE-MK1 z83UXA7voGlSerarY@q;j_p}|~py>%lG=ZsR%a$dBOG1G)yuB@z>5ykfCA#y@C_VZllA8s; zKqoR%zHlfQc0qgK)w2*E$iFA!E6iLQXK{Q#V9Z-iJfptLE3S7yOd#8|hp+*ExKk38 zz6Ii0WE_n)37@Ad-T>H4CN| z%F~rxyF=+Us&nY=q4SL0`20M!)_wk#9W6=>;Yn5Z+z$JhB_847wf~SAC+@ZGz6#Y~ za;IFynzVChz*$PfEW7!Fymhmdb6;?PMzxe>!c&3Jf4xFPDV1F@$5$(!zk!Jsrc66= zoB(-d&_GgZZXPnq8ET4-I-^%)!4SAezz4ZA^J%w(9`|X(L zU91J$X7q|(ah^rm<4NBchhPW9V1T=m?T0v@u$^!NwwFEJuf#^Pu>uzn-fUvW zRsI{Y+@NBw;M6minQwF%+8a zlsiwLi?=a$p5Do*qZjtf20e4Ek2Yrrqq-;@eN)W)MuR}^Yb_-Ps6MEX*sj==>_wjySqFz-<)C$A(} zhG@LyeqB!^v*=S_0WSjj45y5dByAwuUt(O><`;iKQ#QI=+RJ{VY@7Bx3k7WS#}+Dz z;GZ!@bh@r<-5f^XkK@40U)UhnDh4XFs3XHPC2E4fLQN1|cK%BdiFN(%HV4<7w=-Vth(zVU{y5X`g*MopUnT-B2jm$>K!QfrwbC%4%-tf(ie_3up5l0V%JyN zV!K*PJY<2U=~WvQWjCOe`lX4n&Lm35YHf0Mgh)Llk)$B}f`r<~ARPUed5t=#UDas~RZrfKA7RmZPwK-jrf7)Q$!(rV#@(CqfM&PQY`!bv2X&uYR{&6e1P>E5sb38_HGw~|>#;Il zGzf@yhXKt_dBS#5H4)f2X>FSKp)4swXr>(8fh(BqI9@zH**f3zhpJ+`Z?j$qm6-82 ziWJ_(Txlw(Wd>RtD!K1ytb9m5t#shpsCGy7lTL`diR=U_F7RNUFF|S+{1=?3rTmH7 zC|);WLzj3#vUdnkA9J3zp~45A-S}HY;JE0S?>hr;rv?VV^WnVL(Q_r(==%Sr*U(H5KVCrm_bn`pFN- zh|kIGUriUZ$OGLqmq2&TsJIA8;vt3Y$#{Bb7F%(+e!~nN(Tg}vPq(M!7vK|t-BOF> zJx}@%bq}~ZP9xKaK+9CHj;kQeXtqZ_P4lYDhxFw6OKvcrlR4%rz%<)6 z!PK$Gvf%K>M1HUCY^m|*kDD-6=!efE+!$YtTe=M2;lsyS_L8w?#J6TR4Y*fwYJ@n} zAzWSJ=VG@@kHVJK@x6;LYqB5X4hq(%B@a5N3OzC?yhxExT}l?`v+T-NIZ2Mec3yk> zR&B@qG?!Nc0E{@=l?rAm8iQZa?{L(+T4z{%tY;yvwPTk>n?*|H3;GBLnH@b_{6%R z@}96*0r8C6mTcF)aJkNBxYq_l3L*gOyQKN*5L-RL+il1kt(~uMo-Yo*sbrjvppb~? zlWf&nl2_HZ4<#8DWo0{QM+At8LB&q;U~SXyKp9~Llx>kWW%XecL6e3d}$P84Y!NM0JH7iVc44ArAdxy$v{D zbSfQ&@7~2q>32OP*{bnsHJ|dyN2^B6?Y-%h%RKE1WYBODIosNRUQ&u z`n}&Eh1Q;=4nYfI@((MET`Kb97|iCiKnYOfJZn7YuwZ$8$>uqP>2%b4$rqi_Duc^_ zaQVW0ULP^XTtEUc`Ghg!5`>5gZ*q@^Qz8~QFMuwc@&=r!n9=v{SPEySMbxdql}gOY zY9F@@7iWH;zlcAhofxqO>)AZ1X-Oon@=gx+5rVuZol(`RuZk{d)we8M=RAFEJ8b+vh?RB@IFLUL@Id+Ui^%VDKm$3U+op}+h2QL^iR zz4G~~1Pj>=4XaG|JxW(CV^$}5KF9BIA!b$XN|&8*OvM`gZ!;aS9Bq%K$F<)mMkj2E zd-GgYvNJ#zO_p35qF6q>lOz}*0c#xE#3|GqkDA z23KbWND|kSQOue?+JoQ$ot&-vhiuW8AxxK(txxwoA9y16k|KcilLS(@par}_Jbt9+ z0eG`v?`fV9<;Und)sak_VW1j+^N zk4EPh13k~aaH3WawT5<^{C#|uG|4QUZ9y$RkDnx67fduJQUfu5gauM93Bfs~n~-CY zk<2jjHcbILY|J)VvXGi#&-J_A1yw#F3Eh*ZnSj(+RPf96>S#V){HY2l#yn-ob%X)$ zEGgc)N7*Ynp(T^~Og>Msz|K;bFQ7lwLUu>Cb9sF!joN+NQg7N*%o0_2{fCS^H*U=1 z#W6ZPVK74`xWA#AteDv#p;7Ar3GVtEV8BX|c$T!T*va*Rm*+#9HZ(1ffzQk07+!pa zB^lpTAs&4j8T0_qxA&83#bdgz$2xsH)b(|~FTzH2p5+s)*AHIlv^8Q=Hy2PQ=Nm3p?rggYz zUpl>*(_wpeOXANQ^Oclj?=|QdKs!?WuyD?`%~wQEbZ?cUyDw9?ABgP|;d?7~Rsz4B z;XKv%<}iukGve3vlQja_>RsjO-<)uB5#Euu5+f@g-!SMP)jc%_MpKHD<5B4TKZ@hN&QZ=6oV zx==u&+hi=p^13)8&7d^olyBdrIZBQGx_9Q%bNmykE8Q)6nwBq`xI%7cvM{jGD&uE?-%*bd2kjeIz^LLq?1@NF49pNw&v zpkdQ=wK)~P?gpn6|hG!Nf)BiO#E&0aOFM? zsB*upK3ArdLcYX${P=Os`zIbjcdT*(OXvhm4Lvz{C!IRSQxpO*jMdEt+7xDm#rpTB zNWG7DwwT>h&eFLK6CFMyz#n2~nJ|a;>pi8HFD6YYB+)Uj(jFG>oNmBo7SJcl-y7uU zG~sS9Kk2tydMmW`bkYK5pQST6I+VN5n`#TBIQNzp-`hlJ>|BR zBk1xQnU8b(sw;9=E92wF+hn0hKF`6v@`E8z0!z)6&!Wz2Qp7V8>hQc5bMh8bhon%n zhn@Cag;;>U6%X2`g>8f+169O5skiG_N8&^nS|FZcT8dEp%@B;7Xn>rISLGUBFqsvc zt6?~cW-1(N@2`>7U8)@_e~q);uDjsJ-C*nff!`x@#U+}pd%u+1GpSpez$-`%LR7Wj zV(`_z#s7W*g|A_(pa58(S-UwKa|=d12s@5yZMY;PotU~31YF`^KXZJmP5z2nO25k_ zr6#$}fm90bv^!ytuU8D<4p|U_7loMoj6yzgU2E^kkX~MXqq$xi20B2hEbSETXGI88 zKK-TP{rvII9lG-L7lH2jU9`L3Z>Q!>Nt#M=x3tr}b`=X?NiZ}uK~5G@aU<0~ z&5X`$KKP6|$fDjA&SK~t(0lUu&1*?vIETTKzA+n5{POk}t4|vzF$8jNxj?7OpbuV9 zKeCaQ(D+Kt9y78clx3BBnq7Gy=!c!pyvm-$kyp7t#*nbfA#TiGuI1J!y48JL-G$r6 z96!w4HSy``8>*IE+3gXHY&FCPVVa>R67igbJ=}%<3VD3AK|n{X-zOj zS2+VY_RP|wMk3^hwRiW)&^Y09N1LGMOhOlj?ZGHAi^c__M&x2v_CkcKjSZ494wdis z>~PkX=Hm0^V@^l!l{oMZh5kvuA+Hu~dSF*lTVfUV`q}R@t6ezSmU#ELZCOUWHoQ7W z8>(!VL^eEtVCg^deSHG6*fm zwfCi)Ird{64UYEXZ@fpgHdQ_ZU@d1a^<}GPyfp{rqjJ&h8g(K7sIz5EYD3X!CO(-h z(f)0DsQcN$JjL`vS}|wjncLFtQo%E((Zyjy)WO$LyB6>i=iWl)_#N~MIi+A|890{~ z@$eXz^Pz43qUW{ydIHEx-tVXb^MRIp+4%gD10SFM=?9-dZ$iPaIDBfIpWTI+hn_`2 z(1-wyHuduKMtCgS6-!8q*COuVObEkPptYXjmmegS>Y$2a8vMlNY6DgvHI5EJwGZe-t>l0ziM7 zFn#?fyq{diSj|Ao#xgLFeg^ozkljYyvlhGV4-FXVy%Rm(Z{LmTjME;Jl=OjrvpP4>nm)%;^T74Opu5PNB zJXGV*sjHgU!Xp(+G++9Wkt=dK&fLJCSM*E|MpiV~ zMrq%V%0OvD!{2DuqBN`VEjBb;+$tw^=*g1EB8C*D@nS@~$5o zLxWy2aw#~9)-WyQ@qPq`ejcuziJw2|U|ELF9u|A5kUIEM{SRu_)}t1)R6N01Rr z-qTBSu4PX`Y7GW~zpd-HIp+x8E*O(`K2LP(PAmF#OIAr#q}vG0TIW(FxsrI59( z$)07bW1AU8W#5^x&SZ}<_OTD|SNDBCPuaVegAy#e;f{D&hK@d*Li-<^*X=W z3H0NU-T3|mf%Pjh+)0+uB2;aY%&YTT?V3sQ&$eqBm&u3sXdUIupE{sd_IMYApl|f* z-rc`kIw|Be<={5Pf-aRDUbN(=~RS2nGIj_NAVW1KNi&iB7>5O zUU`AVC+QsV%Sv+(z(u-7o-WJkE?SSrY3gJ`J;Kzf4V#tU?MG<9wKq|sJgPInSvwO{ zCJf{|Bj>m_)^}ckthb4r6KAR}Z!_7r8aF59iJW!uCPjPlhfRK#;{z&|O)qlUdAG!6 zPHR)QJE+)4c~nsO2a13w+4Yxk4j!f<=5d$P<&qu;C^5v*jQG822ubXXo$w~z zORsOyGJf-2F4qiKP`4*WJ8?Z{SaQT+XFoDzZ^w%H(>9_a8sm%MW_*!-#Hj(w+W*zBm)R$@;4E$oj>-=W@?g zu5~tyJ$=S_QbA}uF8E2m`zhjAoH^OaPvKcR)k*r%q8gF7Zpn2;%}!F3{Dh*(=JOC~MUKG?rj2NPc+t1?}Y{@o=OH2U>B927JpGbRxuI zj7DbS0|N9#k%6irIFVAo|SAD_KwCCEzD()^xEy=Ln) zHQ)H{pBytIDm3-9UJbwOz6q*Q>yO@icdBjTboI?Gk``MXV4$ANQ1 zqnx_wc!bU>s~0PyJ7}fP(J!0a=*iO1wy3=+RQA&uGOD6-yLT37i~%m%vHewHn>+m= z=PQnu)4!=?<#noOx-n}}9^orLX7vf}aX0 z25*Dd5at|ZJrN>C*W=e!iCg?q_M&!VwA+tXpdW4^H(50cRG{u9g~$$+@*2Mo-on$@ zxM$IUFh@YG&TryUUbJ79NKS$_9n!>R6!1s{A z*s-1@^xd~~30Q@F5z0(1<7Z71>lMP*BDgYGIi{<=G#~WR_c-vj!xWuef;3aNmb;$X z^Lb#H7^@aNFQ$cZ!`|Dx3X*qcfwvjveAN)hLT2RQ7Ep0rpGz%nqncF)aABdn$L=)~ z1pMIbEx=oq^BRbq`3PXbjEt4JjqpI8!lOE}hsjF5+69^B_M!1OUcYX>x>I`B*i^S~ z6>8DTZ=GJWXU})vb4-63@@z7s|K-(|#3Q%19@H)0xy*4IVvx2+@u-#7S?PPhtp4$q zgJJRcjPFldh=MkAbM40?nFVNE6RBC-54gkdyNC#bJnHZ8E&K1hKyTIR?+ND2p;h^ESs`^jRqGh_fSL>D5 z^+g`CRVlv-OATHcEhEx1wt$p|oW6Dd(H=%!nr690Ou%@-quET_GC(PlhJ@baqoWyl z(r`41(QgI9)!b57L*9%!5Ro0>L!0{#4;W{T66(pscR=^KK{&38zLZ&3@*20|sI zt&ivv65k0sIg+?Sqi$5hlnmOFk&xoRdM`2- zs&z7S`)q_sF45^S`1)f1143oq`e}9r_;vd#%j$)M{DiJo>7yA;;Ac}|JwrU{)z%wT zfhU2HXH{N#xMXY#5}7E~vX zrEEgG)tpp;$tRx;o5fTn7xfXB>x%a@H{^7Z=N>oN7!~v|lLCO@B$xZF>S3Xa^)C_0 zwLV=1Q`x%R@tdqF==f>KbIqeeN*V7AWo;CBr^Z-};iC;A-8PIg=uJnT(}3ISO{DL4 zoo$t8nyz)nSo`U}NN7T~6RsgnLh#MI%s2Gwmzu{v_^z~;eRh_2=)6_dMHoxXSzMY9 z^+Gk@Ua<>L(EKd3D4P5MJQ*7`xxM>dp9w$y6?g3*k7QI`gnbA}A6CdoW|65z*u9=F zOg-%7%dO#n?>bu`7-08w$oC468$@LqG0+D#!vwQ>o8Ix?yi zQrP4;xD*-uXWbxh>`@ZW^0?D;!WSTYmO^mS7#C1=jCxLsGlA2RF%W)?dn&o3Lf~2^ zP8aw?(xTt8SkpiaA10tgR_KK)o_&6-H%`v~jy>b&3Wp6Vl(1VwxoK{6n%+veRze6d zviEo}(X{fW5L)zd*;!3hlxtfY^TMnk&G_h?jU7ywYS?3DFja=*QiKDU;Ju-OoH@JF zYs|npxt-`cTXzBBn)GfHYOXCvBaS)$V9(-O1G#n1YpQAyn>9wu^%#|h8%Yv0-FbG> zXInfG;QqFq-AK9ji+igiW(Qvgf82_6{BE+&omj`dt*XRl#=uOGewrkxd}8i-K~MYK zWo$U&^PI#40wFqZVW_$?GXNMCyWmh30EkbPd^LUG_Ej-T8^@ANfYw(f=Yo}iSa-G= z^@Uq=)LHi#-wF)2$XVN&u95e5A^ci7abkNJ(x{Uew_Lwn7kKj$kv`CJaZ!gi?CcsS z_i-X_Sxca#(|KXG;SzCCGY7Cb9*!+fBta7D8REHDvNu`hjo3+LTD}^!S(MV|EQ_28 z)Lnn!Ps+8Bie!E;Wcafnrj)r-|Dbte#7ckR@Wq(&KSqTe|7%p3^wC?%&oA8n{w6=^ zg{49OnQBoIl8*BG=5cSdeAa(?B}F(ItX&TlekXSOMaj6IC>)=&%=NLr)Ril2I_~28 zz4VfDcs?)!r(MQI=J+G8trZ?e{jrBUSJgYXn-W9u>4w>#xp{nc zp`>pY-aaG3u*~!Z<#Kl2pZwJ4bgFPh4XQZ}F~eRO{%Fs5`FoV+nSRzvF}L~AOx7>~ ze74d>_1uPg>fs6D0*fSIEE;gD#Q*Ypj5A>yJ#>s~3K|Z`ij5w;UpD{iu+Jt+Tj@iL z8(LIKI+z*}?X$bZU!lZ-@MBQBP`^B1IA8ZzMGGryGP(FUl`y1|%ml7*i})NM!u?KF zlIfV@_vwT2U4f#2_yr))7riBQmir5f8@6vT?D@coiyMqNM%Bs2Zuc6qv*}+VBG18!LAu0TmNO zswN6xUS8O|8pa;KqQ9THA)RI)T|7d}RIdBf*`I3DHt@pWdql+GlL@E4gHqmU8#P6MDn0RK@^p8BVy zI{L49deTSx9DrF(jV~`;qG`2NdLdeO=n&VFWy(0rZ#<{L42(~;DHou*! z?87L$a((9VTgE^TLBgqvP`6({=CvIi{F*FM2iXc$b{2oYsmhPp+t|>Gd-lkkMlFJC z|LnpJi!!X(j%eBIR$^>+_S<^GV%^HC8RAnuZpqH(`_QKij^Yb9uf6X)Yg|Fw$g?6wy!s$; zVf@{M)JVHE`AyyCM?Y`*J!dM%9?0`qp2FwvKclKMNC~&Tv;ZhB7EZ_jZrgl>e;Yr{ zTF_+5*lE=~>ntW)LQTB6KULiyt*ol>g)5WNy4&0k_ahY;TFZLf!=cH-LkHeJox8ix z`m|W^^v{7@isc8l-&~oN`xg#e-q4m|X(UWzXr+IqCE%=#y1)>u#5we0nSBZu<%##J zcoD2ZUmZ|r1Yp`-Q^@_9Jaa+7LW}J%Hclh2-NxGkg8@1VAV5Yiq=iNy(qk&1Udn?) zcvj)_!`y!a7_Pm3ANo*ilpst2uE6mN-I;cl>XUIVwzT^*Pgph6Y!c^@hw>5HVb3~= zF1>ccvTOd?a?Xp&x4y3{IDCT}!fmHEOqkRX7OWl%-Zqb0H8EJ8;yr!!(s8G-_`vHL z-s_1et_xxUTeo?%bL)U(PVptFv9v)tnk}5#)g_m8I&Q4O<2r$)<7ciWtJ&sKySU0n zSH8A*Wzhp^cS{)|mSKk-y@mVxXNyvSG_?$+Ow=X)LaIWe$8ub<`l0S{EpU-mtGJ?V z%U3D26OUbV_Ag%?#x403s&E6%DvhNl&6nnn!a0w1Di8xnzz`wQh)Iu@%s9~!zjP@* zoilL#;_%z)Eji`aSxm^yq7TfHrBT)9918)ZqQb_b=b(+i-0I%q6rUF!mpSz_##JEyBz-rRQI%vX1NCx?uye|A(vGb- zXhyAhQNMAEU7(9-#OC85@dfARa}`smevtNa+-ZyWTRow$=g#?)awAE~DBGsYQWwa$Ff?@Q|nYQ%t4ll z`4X|$XX_T` z7EMm1I3r6I#*N@)84+x}AnL^sUKMhh6tf!*jQ%TP+Q?|%nk%2lpZ1?yv?vfWZH>FO z57g?>zGJd8K*is}YHvL1VyEsZh`WCZpLXk&Pe{kO?3F=Dk*4wY9-@7tVu=kkE~DiU zAH^J0MIIsSpt-nM~jEW~JE4PyGpLb(H9a)=lp!USNFPxsoe(`OlS0i?c;a zXW#oweB{q5+egLiwopv^ZrNP&y0>?eA;A>JsV}WD&U3BOxTL-|Df_dY6KJQBysf{GxmN<^MvXk zaf0XTM*nu9V?uY>8E*vQ@1&U*?Ncg-L@xe(Jrnfc+Jr>-rsKvzYs?i( zjm{UVxr$>j+H;$Ab>2~TlA?U7k;>U_m=fNJIvhTH>KW@ry<9<6<{svf32Tp!&S&zM zj2re#e3;F3E}F14(l0C=qdt32$&L1L-DC7)?WWgfeVE6UlB>>eFMLssHUG?0rR?T2 z2-9j`zg{MTltJI8-zCBpR^OzgCN6-$LM063z=>B4wGi{dcmT8hz@eE_d3RUYH&gg85jFdwJt~7A=J~7E=*IT`LaQT^YjGx=uHZ6! ziO3qw=>Bs(of66lNqy*OmCi^if)j0Ogb5Jw)aF_Y7)H3NY%Kxv_VvT~FgssWwx(}K zRD$v@o};PNE;LTJSZ+=aoZ#-MHG!eCfnK@thSq)~E2)b*`SZ4if3Z4BB)Ld*GW-_dR;+PFk_ywKa+PP@Qr+z{{-8p>in!1 zO}Hg(VT$@@&EsbvYI}Qo?Y_nCk|ZLRxreRsa~tK?#_w&pYOAebn~wBChLI~QG2b*s zkH4O}OgcR3xq8(ijmP*OM8 z-ct+mRmjeeU`|HcOR&<*%w9}!y0vx(6EeUwN!P^&$I@&`qf6%EcwOkoH5GnpBGnWI z2W)8}tT~hoDQzGOJ}UtyyWBH&eJhh1XfQW>(c4dhpUeJ(-z%D0vpo0$l-+;q6ubZD zj+5f9rg=T-;i*kGCleMu=}>Go;UE^H&bV_TKNxxmdxl8U5MX534rTj&C{-nFmih?R z2@5T0>p=!iEZ3Umy37=x*nzWq7+N}sAO^N@9Ja-l5>I>_OEoqtUdH9$rK-iH1>}~` z7w6Cw)*)_nP0btHH9nwm6S|MAjwdC%G}fD>1qjTcA4^0r1Mfi(l05RR<`dyzy@8eX z44g#e>D_KK^Kn6fu-L(X-@*s93So%_hV-W(q%Grzu6GeJWzdn2z4wy87gfw}&%JJ3 z#HPwVvjinD>TTBe?#a^1Mtb2qAho2$0EHp@TkEy62+q2AdGta*xy5W` zVs>tMCPE14PKhi`HN<7D4;MB*hSfr9CL>fw#UKUITEp380eQQ}?|XmRUG6<4z1@U8 zQB)X~MD|>5nJmd#8;5RiMt~)UvOqrv)vvN;MfX0rawk&(|CZSku&NXV8WJI z(;DPN#`hg(t~WIL7}Tzcc+>;aof<7}h^(u-B&(23QWoD|cv{VqWzqZE{qyVA%!g&K zp@i(W$DJ8HV=t#%&Vxq~Z<_*GE>B20*RwbJaa9_PH_XHNvynHdpHqE9(x2{@atz$h zMuEzlX{TOF!-W87M{>tKxenT#BfB6$3qG=S4-hvpj=CnD+a@=DtT|NVmEbjouV=Cz zyjYB|*R}!XXCRd6ICGTgnUUv_X3oz&hxaP-!&}ux4Vt>`f){KIS(cQg{I)(jx&(|3 zrpA_jGq=Pbbnw*p)nkUZAl~eFKBE^>xY>lQ|-Moxb=71l6NC%o(W9~ zwsj4>LEkE5wwRg|8`*=%(LwN<0(rp*WODodW+ELAiIFB9hcF{zxM`D+QT|>7;z=SB z!wm+*@~~?S>n4kD&2LANDw%4AOy401#v(P@wG~*J%#Hf1cv>!ET$g>qsAWKo&}2DR zGU;1NT#rpl&HkVdcCSwE&azTxmu&W%F{c_xR14<8fVh@TErbG5s9Op4A4M9Mq{AT8 zgr=3Z>(iBCq6JqHd;2lxTpbM^SL>l!&ReI7Dho#8<(;KHI-E>ZW8Dc2t%ir3=|cue zdS6N;22p%M)Z+Or@$uAK%uZAqO{E$rHbs&*>ue4mHfT2jzE>Qf*4b?1T-uK|kS zoplxiZ7s)*JwfzY1X(d6W*pm?S9fpVZ!|-{4;*FjZY#HJyjc6r@oUFnoZQL>m?WQT ziA@dXkZ@dN$Hr(e7X?k>2T?Jhj3e!wcuxUuz>bPg%obkq$y^dkS*_TpoqW~m%EWeg zpko++jTJBU?lsi2HO9Z5-ogF?8tfZPEMNN7>3QX*rD#W;Dw4$?$4-7StIoGKpFwWjgpJ$89~V(GM;AYI$t@n zBPZ_j?YzNj3bv%HZ-->ftzx#bM6(4wKV15zSK`jz@nXUeW_+Knd;od23IN<4pRP{A zlB0k{ihJ&hkp!Mkh!co5yccDK$z(6eTD06LVN{!saB zyT9~97_FR-j~5PhebTwEWR2BC`528Z+MU=NzP9%jyTkGZ%>-eg;TH5Q-Z1P!BIp<# zR38dQf#A2&&RDj+l6C#>9>=xv3NAHdm`faR$zburfV#}f2-GyaS}cA zF<7BtJ+H^c!=fGaEl_fl3!(=~~F%`+nJitf=! zIBC|3AB3gv?4T1|{I548w>U<*-2Q+QZYOim5I@hOMPBKD3OrwKR#3=+O?@S9TFvg{ z65#$2liId^<_WEnDdJ80YV9>gcJL1)&B&dp5#ym7ZAp#W1stsn9tt^@PK%WmP)HTN ziQ*q8@BE?i@tQoI6&;?_tN3-m6|zO;p>a=yeValUOfMSuzF!fhIB&s-Ff_R}=tV5y z{sSB3e7lM9U0v<^1rSmoRshgIs-$TZX3xKDP|Y7_c)yW)jvK3hLD^$&4{DYN@4aX5*~>(CrEb#QjWh~Q=4@W z7KDs^u<4R8n43SR%pfhCTCpbMYxaeerV7V_Lev?wCMF-KF%kP#%Nwi z2G}!sfU+FpGg;{>|7L55Bdc`)#i%)&oVRl^0Oh+%l{LKDKSYY4(e4Xx+j8%EoYOpL zFv?NpkCsv-Wp|@R0_T;8Ph%f99DmuKCFS2EIoc@}+e6CFms{t6uWA@*ejLjea#B{? zfGvu#N7xtaEF0@SYw9?$+equ+EH~{O(L~~cS3oKjd+cfmLmPf^>)*;KNVCeLYkVyl zQ?CAoifV^B;>ZxK)F?19IxlGCyo_!cSs%hF2Z`F?`r5>%vc(O z@|s#-r02B?i&D7SkHf4)6_*TZ!^p=n*hzNe0Mr@2{jF>zvO~lcy;#U=o390%p-V}~ zP`02n_Qy!TE9#68%(W%`J zc{+Lxi5xZR9K{}Oyqj>(E8zQEfhJA{z^OG2x^MP<5jet!%aM@Oss$!`RJY*M0h-IeVF_U+A%f4R5(} zBZM94@E32WnPc6~=snHHEeLR+Af@gHaR}l0?j{t{h1C=@r@z5lBV55CJ)h{knRZE; z0R5{MfHF$`0I%GIjWDA>MGn@?fuQ?^=C^rvXy2Y5O{JNCtYpUgwh!+RBfG1=w{;6W z#tI~wMbkRMS&b!JDNJ{a7O@P0as4~YS!;VLH_(A^GB!lS=P2Q{!VkjdZKlx#=pY$} zhz51W^%1Krp>maKh#9OyZIL^g7J}(C9Lq=7?K4l2S*uZPNe^#c487aI@>UR4E`2rx zv0f9R7DzD!9d~jaVTnei2j(fH_=-8?6ACP z#yWPaq*7(mk*N1}QJ?>?1z;rPd+Y18!H5Ff+bl9`*&WG8{wo<<3x|*Gq?$zDc8atK z*Qi=lF`^Y4Hf`hKmnUte9vP*m8T}+v+*a-T5Pq2VTw-#j#^Iy@EK)mnz(# zD*HuEPlT~v_dkeTYf^rpQpb}QST8igk)h0>r7`=tqR)vI9}Tz_OUO{MA^jsvQEPss zA3-^V-Nf=PYT5J0fSC30u`>hm07XcA1jm?@H1q~uV{=aQT0djn6S92GN-)Kf`h(cY zT29?m67Q`+qDqbX;i8eyAEj;$>IEF(he2NYg@=w{x!19)5njr}pjC5Mhh8u6hKp;{ za{fYK2l>>%!K475c$%^7*2dp@Bg!6{A5++s}=;0F&|KA4_!N*^|R z(17EV_cKvo#|UjJLJM#oWs{OX-Hg>z#DH1=0OfB6a`&8{B7eyu{%m6_sDDj@6#Hy z0)qemu9LEQnSK?@FvTawK>nNFTf`iMb(v<<`>Evo#i*lJ!PvvbHu^j-p0vNS@nA6F z%ZdqkRHSb6fr3rxD!24x^*9XQcfM(H_iA5dfGvt` zv0TG;I7C~7HOTYXRSdIAuW()mlL%z#vIJ zlk}&z>_<$ULBsx$vJchc1sRWT6B`ke^k(w?mfY2*8%-K{!l@-wz`kA;-r{@(2%|H1 z&|t563#^mT`NvkT2w0dlF?!Z{$DR&;OCJ-dj7v;Xjjsq9yI$*p z8D0Pmh&uYG-+1E^TvJu#8|)<5|9F=4pKAV7*A*`vJ;+hleAk1w4${fDF-&!8C5{JMz)ihy@K2tDwQ zsloo(jKA&i!XJC2y}MJ!^xGZ}C?;ihIJ>bq_y^ry5O zAf)1@1B&7rcEpoFZh{+5Hj144_2WZ9=G4}6N79zWE|5Bn|9z$aS|3n^#CKo;g~?Fe zoGat#=Q_ZxfBnzrGk?Iu^`W$iKThc91OK2nB$!eN#96A5OZ|3oAC>?3Al$ z&C3;>g8ohs5H*tn&H3jb|2fG2w;DODwkhr2<2e`|Gf{TzZ&EwR_W)&h!(kZp_^y5` zvFugxYWlxLn)=ci#i0A3q^yS9fC%Ef&tf8yFt-{|BRGvQRzex8x@ETox z@X`9DLx1_hySPMv)hnxGzp?sI&?VC(#`vJ6iemT2cnS9o)-R8V{x$nwSwchnmUCx2}Pw5eKhtwtbDm#e7O-8n0k@!S`W%MouFz*#X~j%_rJ4t`@S81-gq;GVzkzu% z)!d`}%VZ}#`aX4lTJE&D&9BtV*{F{#4;8#m(%dwDP;#Z!4k(r-*`dKCKBu3K_P;Q- zn47xqqp=!mUlB`_=G3-eRC~SpuOP0-Kz(d$sNhnP#-{O^!t?Q?9Ka^q0uuyd{KDh2 zi@yp!{THbN=(k2Fj?U?2f!ZB|@P~>zkBrI|T>Ql#P;Xv}qQ%+fxm+D{`l!PvTnRX} z$Z8G!qIfA9rsnK+6P*z#)9nW zyUw6&J?>HLsp7Wv^BCZ5tCN#I&V0%Duiv?JMmDG?OI@9vL^95B zSK-qyFkI5V{+ac$f4pA%*Omh>_b*XYJQC!L5!b*L2P2w;Pi=0mGhyz!?9F)*ZDwD_ zl>+|uV6j8l`t#BGdZXW&d*;qz8Uu~^Yg}0x@lh)_8XDx5xV8O+lj9$md{e6A1+KHW z{$>_WpLFiT*_PY`TpN5hat1Jr{C_iyzn<|=IW(_5dkJ`>wA6p`hJKOLKh0|CohM8E zGohE^WR}?9ZTxRe&CfM8ojnZrJ8Un3MLhW57I8`A7~rY6Tm!yvJMVV~>DT@Jd`OTW z!@+{nfiL{sefwv@|H*;BK=mIE|0Bb{E9sv~m-{{#!G=Wu?XTO0+krcAyQ1Eif@Y|D z;?9O5dLQ*;`1>92ky*vEwpg&f1RBq^rV#O|3G7aPC}F{B^# zwb|<7S(#E@Z2GPc823UtXC5#bHm&jPI_SGXd9}LAvwaY-)Q=@An++_fUhhics@Ejp z*<^}AYRuTP{XUK=jZ8-}YmW`$=;Vidm)LLZ##c-UUQW0cTpEfuaBUxS^%YB=vY?1J zQKt~9>Cj&ka}m0Ba=Y$amhbZT4WiUyu(I`+sJePY+_i^Fd#NQ#dnqy|AlUj1=6fPW zs0WVydAuhsdY2~$4vOhy=(AVuZhH|k7uiJa)oE|HymHXZe3Ylt^TEbRYHNPMee*GI z<1Q*MX7A16y4~+1+4nClVtuAwPLeJE}^!l){sfWG|J7)do<&R(v=k#(432zKE=zzA!l6b*qNVR)uevYB7is=#aX7^%XngYFTIXCV%yM#EBL3 z`WOF7^)%)*_2xQTlJrWKhpYWtG+4d0?9BhTB}KQNmNWc*C<4jXINxh*3%vHbLJ!Nm z_MST;F2ZahJ{}?i40=uCou6XenpR!JsKIjOlcII_$DAm~Ut2HCzdK2E>a%9vn#YZ8 zL5l+RryAXRZDw9l)#sh3xk;dG)29%T0e$0s4F<$zlZUXRZ>K0ei2ge|FxYRK!EgNy z?5s;1tU9}}JLQ0L_;5Em`0hq_WYhA2UOAY-;+{PaWi0z zdMl4G(^I27lPL#Aw-nep<*%oBUBExSxm209>i;d=zb?9H=i-IEou_^tQv?3PP0|!u zL~K77tg~)cx}>Y}PIL6iszh3AzO-|3NG50&x&OImCq?ALTqhd_Dq?&cm6q6+CZXy) zY{f|9NG~~mEl;l!v%9XWAo1w2^^JrRY5sC^u4pKJF4PIh^=iX@_@Q$D@gm>`1xamp zF?mor*d08(;XugUNDkP`a_{wz+(P;2{Ljj+`0y0Kn1ZC)rT%Gu^my_)n=Vd@?$Ax5 zjo&P}vh^J^DAjeVmyV#Z*Da;T-5MA{L9D^&7$tBH)+N;s!e)01N7rQ}cV`S6=!V0Y z*Q;(rM?8Iet9M-HxH#)pjUV|N{2-2cNg=+>1;O@v5ls(yQ#r^OeB?$S*K5vK+ru*8 z{i1Zcuq6V}r{Wi8x8S!Lx6(a~m=>*`7_*zq6cup`tmHs7&X1~#yW%f`I*!aDPP}>z zO{-AL>&ZHYY^-i%zamawWWXxsauvMG0P~%QWsMK{57+ZIG)Gn+z~&?%3lI-+$j}v6 z^jdj`=yo|^imqn~p~k<}X70pEgeFv^W`}vXaFFrpdLLVSp7CbN208nKb3DH9t3BN};f44|c z>_CY`zEby0y@6|t_}o(Mvw%fJ5JBXo<&R6GuRt~XB#7A~T&)qs1_^l<7e)cT6@|?DU_5yrOYD0o1Mn)npam5tj zXRvgg-FWK_-oHLDf)NvtpYA@oOz_JNjD{khtMMOXe z&(n`i6*-A?cX{cwcq7gCABuiavi)E5+WV(oYXQZgRhPt&l8i9g_)h#HHyFO;jS&jm z+>-bj%7|RCs_UHG-cMnvXbbGDdEgTGV>b5dPl3CLY_YwcB-ltA7XOGpr=?)90A2Ij zkRUQZ6^VEG0$uP_kOjjeWvl>zNlA@fOMO`nmHFr?M2Xpe6^4g;DS?9(?)`-uDwOv4 z5*uy}>rH9cw_CV#IA}Ff`L3k%M?-Ck|KkSez}ymKq(E&eF73{SC3>$YY_cF6TJ_bE z442z{J~6qRK+rT2)Zc_|(fYKg7>u6RYix7Sodi(k4n^oD{B{3o1yF(2sW5&Wi_HQo z#>$UylZI*3SUxTH?#3%dpuHNXr~R;GtCuYGRM>K(1qViMn^ha zpQCT0(H`Nfctv)sz!&OUwr?3%JBmrnIF;6*)hVaIg_w*D7anYQpFjl>o3I$hXLuY%=~?I=SvRs`xBtw z^u)fAb|Q50WX{}~dR{dEuCuJ0qaT_$$Y}z?Xa7zHX4^*qL-(C#OE(UW)uniEf~2+Q zn{sU;x}b~x<6m>Rppe6CJ-%O}4)^#WC=IPUESPU|x1p|xgvo)JjIg^ArbcWBL`E=ZLESi3uw`FpnIH*7ki z`2IQ7LA*()v|00=P(4=2pWiTqJa7|p^-R=buko$Uw7)6g-j5eDLEeG*Rm_Q*Y4_-4 zX@u?L_lHHM5*Ko&&N+@0gwXg9C7dUp;??$F#ie1kOudIQ3T0QP8^@+7r@=Gr?|2$X zIDPQ~FlSMnb)JsEMrKL`s z5Q=SHB5$PC#c<3dzxDKY!WKZ_m^s{yD#a~Q4!WSihR>ikM%>;Bf~qcMk;*vg zz!a&gR)FPGmw!ws?3wwRV_Ae~KMW(K(6{8)ooMv|!_QZo22@D1?f55GmQPGc*euZ?>wI{hcx zXoP#cNLM7xyoJY{deBk-GsDek4j3+f8<1-JmH#b0b13LM04KVN{7sjG5meolM!(jZ z{v$yHR6#ipq-Qw#pVZGkxvD=!<)3WepL+67`tq-o&OajYk4XH@c>MoCBy#VEAkLkD zlUC#Zmz3Y3BZGWhmWYg-rNA8!`7$8$M==NKrB#Lkl^1i9Q^`86@0R57@#A@cOpurw zMvZA^^L-1%PmT`YsU&fbMWMF!UsADA{ztr10AH+X^yQR&b7*!u5NH4`SmN`S3%HLw zrqb$fz#~ORPL(L6B2QYES+}c~>|1N}`m8<4BzFkB6e@89yhoK*Q9v^5VHspWwqaI*^q1~Ji`mmf*ctX9xT;?SC(aE8EXGr zTKy?tX$2P-e=TqZefZbX)(@8U&P?iGY->9%u=)RuZ7`;Pv#sBaiZF4y@-`|m2XbWS zx>2d|yn<2BgMzETMxQ9A0}{uW7$Nan-IL+ufY+%p0f0trLI}c@0h4Xhl*G_f{%H_; z^E)hgR{uF=>;ILwEmd^3sXO$4PGV5JRCZ+MmNCmJOm=rv)w-j2@1cM}f$vE^BjnRq ze(*zn97Ir$jl?fgp`h5Jmv453t;D{E+UO?a6)>dp&Rhk)_MSXuKN1shoc1Y#!V?l1zO0EzbDY_JsFKf2h#_!Z0Gf{+w&APegK=?XbV$7hVR8ZEy;` z$|r-Mr@@FD3RQ-WnLjN&-sRD$LDkV)?XI>b-Q3E-l|%>`Cg-95D7T4cX*X+)NyaUJ`rp2H<4Jl7zDwUQiI|Z3;M* zT)W!xmS9EPlJ@Zp%p3gh%u{-?Xe6@jjV2M>RKplKeK6{RV>3dDCtz%jIKug?Bl zg9>oDrs&7(tz*BT&)X+vIcG3e#kt{{rP@ZD+uOi>^QZn~L)IYklgR{My|%^WrVaN7 z920w%$J3>Lq^(jN&Sp-8vxb%kZEny(I}n%d4*ejt+P+HIvf7X4(Y@PTGZk*#kQk@) z(Ud2zr?~pooOf&=zR=@*ON~u<=Kj%{E5Xt~A}X`)XU!#UF`@R%D-?GtUf{$BOZN)w zY^eqUZxTCmxDQV3JQ~2u4-(VoehqrxoBt^|ua?A`e>X+;*JC?D1r9fB1 z3e(U!%98@4n_#Sv$)zl>YZEUk*zPV@D9m_EG&)<%(Ob%VG3Ir-W~-q=$$j-te7-MK;U2 zcHpkafguy2DEBbIRLT4NScfY+rZ^cCnkh44>jz&RVfjoR*4KwQlSx4T{`_Lu@j%{~ zx2~riMb7Q+LSH-yc`b~`~7@$t`e8Kz{C zSR_=?uF-p=G~nT6CK=g*6901{A zMEk#VRfYSfA$tKi367bEF<~%Do5r^Y4;)p@*-m=mVr~~KF8_GGRAF^$j99+M*^=o> zaC7o+qNG))hv@PNOl!pjX_KQ$^_-dmDueNhj5Oe!sdYHStZ7N?AtkQ>d^G`(xp$_# z@-nbvfZ@t1G}QgWI-CToLo(Z<(l6>7^x@or;lh2v94M;-Ge#?G#;Mz1GmjbQ5mM9R`b z-1X&9mF3s))^vI}3t560vyQU}`+rz_51^*DE`Idt6;Tlo6;P@bkg6a=flvh%DGEr3 zh*Sxoh@pf4Hb6wcLJ=Y$y%TEaL8(fw2{n-#AcO!R5D4WRuJ`|a_kJ^P=Dm4u<_v?* zaL(R)t>0R|RrcPaf?{F7aSu4iTJ?N@MlI$p+JsP-1}xR0m{h0tzE4mp1Rx<;K^cDf z4~2Uv{(Ge?LjegApcCYFqjpxfuu~n-RLb%qJfSdb6O37=6S?Xum&+SjHbe@Zq;e=J zPgNMCS8^yJmji(M9I?_d<9NWPiMhOc0@E^QH73Kg+(J>=)4cc_yuF3J1c^JeG4dU( z8nJLqX|txnieXVPe!#O4|7^ja`oUS7b2g*)zU2&tB-f+=hm_*f)Jd$!@FqYio*bKi zas}MZ;dueOuw3s5eH<9oTAR#|RiK`u0Nf+A&O1~x{<@X;z_59>^^TSAcx%vBe#hDE z0)CUq&iD$QnZuoF7-;8YmF%zLy{dXaU#ciyTP#Pt$|keI9^u9qSr{1d!e$!nDICzB z(gbWH-@{d$;GO3?POL7j3fzFN7*Yq^F6AqGx8q-S=QlP^Jo<~6>5ZuYLic_IAo&uK zeHs7dq2{@&Z_NnVH*~NNiXUrY0BrlReZA+qY(;6Vw((v-q%V4JJ4|ry(h2?cpRk14 zMWu9;_QFjQ=lQa0vWpq0aU$oBQ&|15Ban?ZK+CRC(9^^YX)XX5H6};|W*->$?iDBf z1_n0(46~brf3-<#UH&a@_&zWK0AO6LvU)&C3(oZ!Ui0Le9!u9mnS2az530BiUluv! zed)zLZ*zXYv9G^sb-;2cxlTZ9XwLg;A_wtwJ$G2V)x>iO+=E!y6 zLgKQidt8Q*wobU6bnM~r$ga0~_Iq~=;$?SgQuZjVce`Sx`exdkG9Hg78d*Mx{R?6q zfB_xi`lp3I=lnOres~0!U^%dD83oG!1VDWXi-CFc+*Q{ROR{m%iiJbJ{E7?>sR^5Q!lT<7* zPNapP<%0H%MS^Gv14HGX%D@|i$cPVaJL>M^#%^MTmYDmq96L6#`a)NraUG|~!B!fi zaVg7!NMl$@=m0dR$ao~jyayz|wACrN^(r5Nz3;MwlTRCcaC#z-Qcrlr%3duvCB(N{ z+B6!pi=0y~rrOwFu_-Bw7PGX`G0hi`S&$!O6Fe4}KSnvBS?%#gP-{38g2A>!@AUi=L@Ks9RGcyXVj>wq#iFb*L68UL)`R7UR zQ*cImN)nnmrrmAI24Mvk%OeZ2L4!)xyXmJTOeB-jgV8}79KmS+aa~5jde`U6+*Tt< z7(gj*04!^jHwSZ1iCv%iV?05C(plZb0+Mjty+{k$xg_Ca@C+T(Z&M&B9jL2gnk`U1 zbCFnU89Ud=%X1i`JcG#Ta|Uf}$qKspHaK;+9S%6x$OhS{QK}}9C&M%y22dM!ewJ{j zPhF0>yJ^9`ngd|02PI}RI)orO4}o{Gd>m$}!tJqo{HNS>^w{fslvc#AJS=?N1G-7v zV)vidHdnE$T|{Vb<`CzL-WBXIqLTXp0&5qMzn&NZRu7xs8run+o~xvpVugo729hS| z0Xg`*ON3pyUlZFC(0oqm2i5u4NA~JVXn}i!tm(iJDsJkzt}|m-WS4esXQT(JF5Z|j zHJ9~!{SLXqamA=AWD(xa@0ll0(ap>nXf%s=FZVFe6U3JE$HdkG+OS(c2g?evmsaS2Kc%Xo&CJD$Jr zS+h&uBdVSC8zYR16wo#@6W-*eJMxrV=5wdCu{$n=ne=|$2U^7jard4e6DsOHc?S&J ze7E4m+MFyuD`imgz@>s9O3W;HHbI6jQT@laGs%g%!@DffP^>h-YiijbL3$kb^DFsz zDmQz1tClag7Yk7vA+>ikN6VKoPG)Fwx()@6uGS>_X1wzw9oVSYS=znxHcniyutB5w zd!2L)yq)Kb%C)aVa$BKkRt!fRxKQ1l8=r43|?@5b9&ph{w#+QPJ^?A$|8t_Wij1fvEzT#8CW4V1%Jvs`A3f9M^=4u~S>+rPR zbnF>l3_r$$vu0Xi%PM!LOLCzEyjOU|MQPz59n9w}4=pMNFUxyExd5k#l0pW!J>Ff0 z&wRQ2m^W5oz&S>wAbl{tqACem{B|6fL~j-4tBn3|*k2H}zTvd_&Qn&S5&w|~OVDAs zOpt%sq%4NIATdCFXKwg?Mhe%wEflr4>}mcU z*r!WyzuXqx-&6L{IJ>lz@NMLwOU1ZBo5@kIF&8@D=4v~=&q@In?P^nx1HXSz=jZ?A z*|#f82O`v?Ak&0us&(%~-cD!8(w9Q7dTEr3wnCxBEaHuM*x2p<9MvD4r_nG+PKHM2 zoZc-7_H{-51R+oL4#MJkHM^$rUE$YHvKy03My@CUy)m96lMIR+a)(lQpD{ecw+F;i z%x*~37a4v@BzRHaNqU`yU5pez=tom`azFpz&Ukx-eD?U}bE8+=@S|hZ<_yJ!2&Ig& zU^M6?_Hf|~v-5y+mp(L^NM3atdu@>XA!u(`T^uxGJ+eogcn7PozUYpwhsuVuvHsKX zU;W$hi|sr95nrIrTw3AUc87A)d^{ML(_!ZX9(x-n$xRQ#`9r(Dq3tK;R2?XjH>ICl zu`I_J6U$52KZ1Mdh;~b0_ia7V|E+89c_Sq=&Z0Bto2oNtQz2@f>I2PZx9J%JWMt)a z)pf&orDbSh`#P&Nx=PxtkWl;Ty2_elqG)-w4j=NhgBvFbMjZrafH+~-W7nXr5 z34C!+q#UKFQdJvw#t&w6BKrMXWC9%K(5;BMb6vj%vE4z>u)R;G3^NH1cx2z+m(Jd?P5~P zXe!<7$yKlFKLp$DM$0P>e5NR2_8vlU+L~FX>^MwOxfAPHt6We4#wYX~aEBG9cp34_ zj@Lz|`Jtg+loG6~PATJI6#QK|A*wWRuL&G&TPF}cf2|VPnfR=*duPl?k$P=dMQoKh zHd)zT=0xEfxwp!1Ugv?~Pr#I{7nY~^iM&LYeWW`=`+yEe>QU8wDs4Qp>*xssvz zFs{G5e9F15YtOR|Rp8U6Dh%fG8Fk&1HYLv;v*3x~7qcSgb4vXp)D*-WiYC-hUf z-JF}K^;Dz}kC=44vQ?DbpALgxeim+I2}-yIlEw8b?lsmK^n@PcYu?tS)?(Uw_dX@A zD=XIO+C~uxdcS=6ok@H&`ddN&0#L4C7DbM7{*57mBj30_eqDE5Zr1tm{A=`jm3Nom zN?XCw5JaQf%im@%H;KC1eW1p6#xQ<|L{h9^_S~0&gCWs*Hl{9f%6075*P`Ud=sW>8 z+67B*?-Vk-Bw;A}67n&p;}F)4;=t)MErk*8=HE(G!i^SvT(ac=<3`_4ms?OM>(*!= zBmq+_^h+*i=T5A?;1zxoshDa-;SzSI#5o_$flcyUIJ|yqjHM5!IV8lFH&Pk1i0qkH zglSA=KP;cv#m~#Gv@Izfswv3h2CbE?_mvRut>3>#_!vfgqQlC~vro`~o1zz}L&gnw zt*PBUa*c2!$)cvE)aygy2JBFlS%z33eu$KA;r>U$^M`+@`ToiuzEB4Gaen$7ZNOjB z=E15comV_wX5*rX2%bFyTTP}lX!41WB5fMn+M)t!(!A;B`!t7~^Ms~nQEZ!Hk?0k7 zIH=}UU9VeeByxraPlKZ^36Tg&6jLAm-Hv%r5*ZPgEaeQ?I?-Jcp|m?uSere_rLIF` zf*dq}4cQf5&wmyH29uVVNCuhw{FMg1`Q zj`NDHmq^BpCn|S23of6we`3+*HccwK4x~gA<}AQm#k?8{6QzG9^>ho z*w#v5AEU|Wb{>dDG;sZ$<@a^PRyW(wlQxB#?U8;pSwdO3%4(hHD`>A)H;;=vW;xld zKX8b5ol1uw(;LGF%Qh9M?sL!zx|!1+#?G9;AH(xDGJv$Z>}2-Usg;ZEj#N=;-^@td z*!DLZ)MFMc+?MNSG_E+ZGN_?|j?S^@56(4_)D_%VdL@l4@9}K|p+m@}cpvbJOHTi; zn)z3CqN=c4Vt@ba^>Sbv_;3R$Hj9v*VjG84`X(l*;vFC2-5FW3OwV2>2ZIB<`~E=u ze0}~g7^CR=+)@K96a>#HQk|oKPNO=)u z^9zAhP#HjAlcK2sAA*X!i@b_s#`Q7ZJc!vdaSN|ZrU?FGwbG8xmqmSq9s`{Xr14Yo z=_QG$Zj-Hg+A|}M!9FhinNbF+OSFUumln!!i)c~w!5A3Tqy90<;+<{a&rTMrnn zL{yD-yY6iCaH%X#96)cUU1T-8QEMTrP=uEN9q)uwViGx(ALc5+rgr=3fjA;7zp;%zOfkY2r8|VIyHhJqt`NZ9!_9?<6j&PQR)Ld&m!Mk-B3xhknbkGuOWi)(G)ZR(>#9sbK2qEz>26J! zqBsaii4{fe$bpcsxN}HBAk(-ccoCVHH<1@y?cHB9{DJpw0ghRnq7tXOWhoN48qH=& zhJ$r@peOFD`+qlMSfzMh^l9{NGnJX&4u6$iNm!&J8p#jpLKhFOK&mG4SOjP2b$WYV zSmSK`EzO~r!nlH(l|pH6T8&8(iWr8JLJB7EW4#B829>1*i20yAl(0|L5W54~jG=e| z7+eAfIf&}O6?$uqxm}3MH5-I65AQIMFxpu@jv;nBj(06Om+$0lo*F-EV{h+Ldqz0E zosYdX&x|u&v+L>iYWh&=eQ(D-EFZ_t+!&<=6Ra5R{6YvJ`-2H?5j%6Rc*OA~LBQ(j zDzDEr8SyIQU9cGcGsbLva+L;f|7gy6sF#mqc6KX2|KY&ip#zG#E3~DMG(n$@8_dYh zL?%8Nwq>1RUG+a-lrIV@HHU+q0t_TpLfLRIz#N5HBf$U;qzk}_mW+QiuG zm@IyFIZr-FdEl)1;C4vQ398 zS~{gkYR?1iOxPy1)7yDNgpT!jxH$=KRqQ6U-nl!jFM(S7Vt0{i=+3G2M+StG)EcnZ z1tjbkq;B??P6&S>M;CTU`@1kr5c$G67$WU$p8|BSisWZ(*T!B!+Z7euU}( zs|or74{b+OJtw?ncT!OomMvL#NjZ?m(n)|~+@*J&ipWQ6j4JFJP-T9N2%o^Gq(Jrw zST<#0ONBh+!Z|XeX@J=?i?UVPQ%rv2mB$`qgRy{B)SL8e?&b&bfB(MDyj9O%ON*;; zm+vQ_y@kp`4@(|TS{wF~j;`2g?kdipkIlsZnTa2tK}$~?Ieh9S$}5QjM6X6XhhGC; zH^6|+^Uj3SJ`7t90WgG4$0mF}Dy+e&NxK8bdm ztdvYn=V386n=83Q`QRLo9s9R`^2q1F)`^x$?q-@;}!sT25P z*0tkwB+TD)I|;4nAB?~*!EZn7#^>;y-#n_q$EWA4VqLI<2yAJOMw6bMd&JHe*x5 zd;-c?1cC{oL9kn_&)muv{)FCxKjU|ZUf*xG<{?5=+Z*s7WI_;F@2d0FK;I*0*{{bf z22|(llBjjry7Mk8sbwe{o>x}Szsmgiz-$An9mgj{{m_fnsoyJtdGw+dlU$9wEAi+t zSOAu~!V!9+hdpvo;i63?F8GVWHlkh5U1lJ`px@}U~jjGKRaJqh|W-9Ld>Ww~|s8@umxgT1q3K6dQ4mS{az5@~|%Wz(Qv-Bxk zsMZD50U_VBdxTd@2J#l({dUuxEZ)T@yj_JU5jii+OYNlL=WXoI7I^tc|wPsnfGWNTytC#avDEt6Kdaz+pi-nlL`mkd!;M}Zb`ZM+Dq`Xbq!u&Ss-7H z)T66s8o%|4EGP0%r02Qivb!t|>_^Qw$JXQK8!fJd=;^Q3oPQeR{*=K|@~Nv>z+Cb) zhIo+-ubero*c33;*u`G&%IrHskq6Ryapb8`bkLaK!Y1ElK(UVO3Ro{g>aR^ez}XGC z^>$K^5>+LEy_xR(==?38gm=EXE}+^#=DnmI=r#X0V%5HQe(nW@Uay~@mYP$ytc`{c zuLeZ&|5-;e{u=q25!*sWQ#8_h4rUNKJYBtRKX!n4TxQ? zj;`;U_#dZutG;GogzPNliZib|WUmK8D%K!ij>MeSG^58R*R1Nh^XUGG8!5r|M5jF0 zA5-Jjzc6|90X)~ie6y&517Lco)sMAkwo^t|Sl9svK61}rPC+kY0om>gGKE%v)hwnt zo-9g5>o6>OrhI^XBlgdoINN<`@tGwr0)x?ZlF_)TrxNl*4@Y*XTpPXo<@mL)q20w{ zwiz@!`d)mSW$5cjE0B9ou}x`bBnSA&^{Lhh2b{YwkC>S7x+^D$UHwn?@B&jq@yn}+ zy+2))9!M%{Z9!U!Pvnh9QxQ|BYr@Mb>-;sg?)99tAKm_99^!?PK(+D6p}Xe>iNz}o zrqqOzNR?j>X|7G;4_?ga1&rO%(4yC57SRvq=$Y~UBh%#l>_7v4+W(K=nZjhgdU&Lw zCGul1`eLB>1T+h3qf;ngca&{K@;>xg5#43Sy+hHpP#G_GFZ9Ba_b79MKUKkaWe)EK zgh4fgHIo7Vz3RO(fMxH2XJ_R)EOAoS=YmwvE^Lla);UvWk>zYOzY$t_0$o9A3WKDU z-+WNul*dgrw=51UMYikijwMy@J_vPF?xQEIwq=&*IrL}(SxFe2qxCJFhDMDymD`x( zj0(azEWfxEFT;oWvCoYk~k&kIQ1tsrFf)8=!*xCet$LgN{B`3Ln{i$Yk|hiRf*MY zWd>|icL*+585629I7D~V7BeIJgSxBxqsTP?crns#nWlPj;Lld%em9)9zPb|tqvYrf4P zPZL3NP+pNjYuWAy?MJQee;8_!COeqFqAHefM*Sgi8#;=u?m#YgjDNvzWTK65&EIRp z6Q?axbUw8m`Uuv@G=YuaI^9wEO-wgVzwHJ9;OtjwIWMo4o8Km z{%!lBiI);uYwx-&@gdyd_K96A!hTC-MPBccvI>m*cd}zuZrC-pkiaBgxH&&NvL}*$ zvrlKES-jdH(=c^ZWZiFg;qF?^_fcZrQlCxeFOJRtRK-x^?Th8NtGP$_T<2UQ6a?0W zQ~+7~3G2MgZv=>S=*f{M-D3sd=89eEAsn&a*wuW0^W%}X`*yF6d{xP5*!iZ-+08z7 zw`Z*U0Y4(~D{XJ%r^bpER~6COf%wyzgs)|+BcwtADn=GbblL+HkbvD z+a?i;D!%4H&%Y{6sRD)g^gjB2FvQHEyh{|c8Vnt~m+1OtY|Ib0D3UuqP`+G#FKHS6 z2&hDfs52BLLCA4A1BQ#{s0G8tKntX_UV_Xk^Q0d;yLu-Zq%SIHbM_&ZFiKSU}3wi%kf0zM(i8;xNuJd4cn0?2# zN>e=R*?s&YU$pXf4;|`g7jLzN!C}Ap*3nH7r_R}C7k+-eF(HwntI&-zoG&kpR{pTl zU=2{q%vTXRl{ci^*#ZJ&QMf(J@Ika$Bv$lGtoOWT zsV%plips-;C}V!=b$HkrKN)1fr*p`}uYd_xUi{j)(IVmfj(-C;?v|}4kx`x%_$1(K zMr9Yji^EiIh8uR|?MP#3r&h}QUWpaVmEYcZCIn5H0&*{#JkRvEDouUsDDFIuOkp^E zm%K2RxMH1JD#j-g7X27lnT?ibgR_DeGv+BbpHtLRuWYG3$ZbAI0bkHADN0#zB3sQW zWaXQ-;j*a>W^A(;N%huQ=QoS8G)Nt{n63nPm@6t0kDb(ZeDm;RBBX9-9FagxZFFrF z=b7USSgY;Xns3zg{*wr($rR8vq3@^t`c1x*+AW3)ZOY>3^v3l;1&5n}B|UfKV9su2 z`7-=PQjH`MCdh^52uQ7Q?YROolv+RWe#foYa6i5?a?d4qKmDr|`1gB44>oqJ8rFaPH3Q#Z%C3(Rs*}G# z$32-!;Fi4(gj>$vGYCE1mYMWXKCCwIW>m;@2zrG8HLz?QExd)nl|pX;^HqOq>N+%g zmRKipnJ$85;o(ty*ToK@6}CppB*(D?1)*)Vq=xAOsMSraz%EQl_;$y7^{zwSOmbSz zIp%m|^`g8KXqqWR&E`=bT{45o&402faUMVy`e?U8(?S{g4m)X)M-@4rRq+-3-ydld zu>p>uck{hGT-+1Oj~6|SRlqElx8rP5*As$aJR;Jl>$;?;g1pBbm)}m~M%7f!_<_?x zaDp@oF(fP(3?V;zGfo?vedaxpw|*EFq_t=}4AaMd=%)8fjk)HerZ|*N751b&O>4YC z-F{fC&h}&+e7&~NbP{F+n9I^LpMmC^(q(y2S@>Y&`pxiGcA$q~u(RufO->%z6D9B= zXlcNg2%`!BwRwH!rKL?5vNI>^a8+qx6N-8ojtCC5Mml*(0`@(c*+RIxR`$*Y%+Rezb6g)7z(5rn5lqK zd^bAM*vnHvcw;k!pX>3;*d?^``bG(5G;Wx4wr??aO+$q^*jrH*l8HBrMW#8F#%3^B zAd~A1UKClV#JhQc<&?&UL>FK@9&WfzeaobvAV%2C4;IND!9=BJkK#sQVp9b`!r{ao z$F`13&9eM;cWe9NWjghC7d&X;5f}o%pT-*W3C#gSHESuQC$XSj@5=`xjG`yC&ew939*Se$GrNf6EdB3q&3y;dHsOp-qw2P`6ua8^k**it(pU6Jt z%VS^P(VLQ7kl0CVFyq&#u3PWGcmutv251RQTUdeYIX2VQTdUdIE}S2aZfft22`_4|Qqm zhu??y!*3-mVGf(mqcnKboB7j=3|Q+5LQ#g%rO0=`x;cs{UyM6%UN>I|qW+_T_Q!#} z-zBtSQNT^_bOMYzt z46lJ>b8ObL-9_dDJ6^_gbV;|W1TeM;BPhJmEo1Bd`?mb+LM5Ct~ohTy-bfETc)qh z0r$axf__xU77)V?Q(d*5c&@U`?iTcX@fN&*v!k`pWpDYjTRy=zBIHO^TgALRe_2um zC#Gur2MOJmiW`%$w{YYW-{|G|w9W#-ymx)5576C2ad?f8R0L=J)!H{~X(t8P zE_NS?CS@WRBllOZ0q!4ItL{9O=bECNxXCQPs?~c zIdiz3-adcn7DXXy0@1R-j^3lXuXSGsm2h|>^H#A?O-?kRf(v)|RnUj+6)Za9$9Z}k z{IgTq9pqkhbZ;+jJ}J*M)_}iYf}FTiaX&Q)EsY!uNQv0BSw&zi1l6~bGAFxa0WJK7 zIemxjlg|bTB^Nca`^5cN9@G{|K;xYpatC|KUr2BQV-lMms8|m@S)f*n2rz#l{`dJ; zFZN1(gnA55Tk=b(eK;qgrglQ^n3yA1$6Kz_?l0V4fwRjUX9H%xTy^TWNy<8N|Maa} zw;tRod2YW6u!^KN&yN64^y@!}S588_(tvBi z?OfiV@!D?`J_YeVt~2N^hK7~qx=TafYEA|9sha%v>2#|3wsAn5d<;<40%(PePaZBu@lg zGeCBFyA^uREQmDW(WEf6^@S#UkYl$jaSZw8{PxAoZ4t8#K}*^%85jj3OR@lPG7Wozo;F!oxeE~!L&~L5$TFh0VD?BG6W+c`%;MEWP=2Bz z$d7hFX-UP7r92ZiwYt8U6u$CGRZey2DmGeQ;W}iw>McLp&CvUKuHVe1&tYY8p|q;Q z65C8x14}ACv?F!>?2^C{*11-SgTH-L`g4Y&eG;SE@!YMa%Uiw=Icli_q@1cq?p_{D zFV{pYKxbE5^vp=}o>tkw_`;?IoXPtaQ4&6aVauw($6C)mZjMQ~`3YN6Rsl zu7BCEsl;35c&zI>?V%b{>v@#;E$v5geV@@pxa!4$kSMJV$1Z2*Mi4r1pgeZ|@dtb= zwaVsIh*$8a6>QS%g8H_cSAsd^Ih)n3!);WYO;EXayL_eT$LPa%w?d$#1V~+-mq7Q~ zqzB&{X44ONpXymVnx>12D7acXxX0Ew?YO&M*zc-8q3yWiCDb_*s{g!#w#avWrCgx< z2~KB$mFvU-t~*y=q*II2o7Iul*Dqo9cXw{n#;v{T+2Sak?;}RaEOD`i>o2>I>oR-D znPvXQQu}}NpC?=AATsL?vU8k%5@ILB-h_u@BWzAPvaT$f9w`O2O4iPu%zTJXboFg7 zhjsPLF6H)%Mi$#%smJ);f_Mp;N#<9~W5bydV@M*=y$+WDzS^Vza>uH|hD5_Tr4OP} zjPJA2b;G))e$3@Z7CX*lb12Iph}k3I0jEL|Suu;4-23txu@)MLSl6y#5ZuCd@>v&f zT_~rwJ99u7d9wE7mDl{co6VzA6=Cv|CrDy*Ddw*JJjhXdmDv9b$#L*I z_NY+HnVV(Qpw3FF%F`gXJ_JjD+pGx;hfNcd;{_ejc<1D*a27`M=*FC{=GsW;YKMdS zL$(rYY#)DQjviAIf})FSM~}<7{Suue#ku*AB#YY+3Imngl7C)ylbr6M3fFoRGODd* zEYVfAp=X^!hsS!QU~`X@y#pd0m%^5BE>@w&&h&Yg23XD6LQaeg*?<4sZ(C2h09t)_ z8yPSuPTFh}`IP;V5juCnf5aZdlV8+d?8t>`{Ner|xYz-Doi+BLh$T9;|enZQuyyQ=rhJ1fStC*sau{E3PURI5*JlCv1Wu%_Aao zQaoyh7fBN2o8W!~7Bh|{?j`%chAbPH`W0S>rxe)+uM7L!eG|yao267(yv6FcJ z$B|__n7*B|9$s;?kk4Rr*yO9D=NEQE-;bDz4U4{zZ%#=qgu4%)7gP{DocOuy+z11g z0vbK5W=-9x1EEDj?XaNAC$@FSCsSoBL7$T$m7O7Xg?^3n&kUi)n2|Ee+a(+7OH~#> zwa1(Zep7cww--;=yy$p~uI;{VULDYRZN^vsqC~dpTfjT0y*+A=A~!sowQSWgDM8gL z3c57TvbJ4iO37`tQLa2RGieiIfPgPfJYJ2KT-f8 z`W<<|QzE1wxXz5?<}hiilnl=%cAtXaIl|Ds9vqlWya@XqW>$cw+WgE6LSjhqcrl?k z8m>7>mN~Ej-;|t$?mcHzd)(^6PaIg0_hsjKQ-ACY`T=Ry#k9~(11Wgb0BH>lXXr3l z#}KKOVjCx7k6jmXvBjf_D7EXVzE`t{+npb_y(;Y<=n2x>LkmOW4Isc8wW! zOTC`T3Ezm8fff9U^!cnK@}i$&|0~(9$<)1d^Qi{X`r#xLJM>gYcoDVy;+<3dum>>+ zx%#D)Zsm}p&mTXN-x}DW%`H*HUQ8%>lYd%Nx-((Eq0TVD?1A?~rIwGXN@kNS8j*s^ z21uCjNeHV&im=uhvH#Nx&=t4ECi57id;Muu`q@`v;TE2eT6%%B8LDc~wLhuK@TF_s zQ^zNHrWDTGm?}BDNgFnqwg@-j~|UqoMbbS5=pg{BghCnYkSt!rAS}e9_VDR0X<)IF~44P zSZ=nGW?}xF>zb@p$*{qm-y|6+`g&JwudFz!7?#g_A9M9xhn}j6UwOQ|!{nI45H=<9 zO$H&B!E_0=GP#J|eMA}#qoqprWVyzE52=&F<<-A8_lz>?FchqD7&ubw^=AA?Q_I01 z8HMq4+04ExML*BoC3@HtZkD$YD47SEY!%M;8XRfa9S_+xbD)-vXTJ0}yLBCPN#$8k z2%Fa_hboGH{AIE>3pi7l~t8*Ujjl>I*5w{&}R zxZR{_=Z7x;MvXOcuhma{f@QRt=_VgO@ZF_aYk)~YU<(5*GD+mIEnT^bs>n9Kcg64D z78Nnt$e$e)%BxoMKd*G^dDU6q!AkwKL8)(nj$jCWxW%3uf(ADw*5IzhH)NobPeD{G zlF%v8N|s3iO&L=9g&$w&YCY40x10q&5U5*5lBkc^Sr4G~%i-}>BDw5Fx1+As*KK6e z8@6zCiOI!DiNYBk(v_JqDG2N7Z<)0|&a07jK{`7+Qmx3n#5(5vgxyyCjIthA zR#8CHy{B>VP~x!P{0Dxe+>WC2V=Ll`H`XM`GdEho5D}JTrBkadj`MU(-p@usWf0&* zhQ4!l(vk4VJo;SvFTb~<(m}2LUMGK9bc>PRZ8OfwIZXFd zg@5(V_twaxUr)cWLjMV9Yi+&5l-B1l)v@xxlF*J~yQV(r(*aQaSijmtXo$HdE({&& zFv}n1a%w}YnGx|vIEaHwa65!c)!^68jfv2}$WdYtM}<({)Z(@N@sRTiRb`0=skc-Y zVmkx3AgXwpyxE+85+qgc*Im7~KW_OaPsztUZ2W_2vJmDZZlm!f6eKyd^ii+|l?gM+ zuWB-`$#H2o0cFG4J=>~O?a<{NSu?9S*1?(*-9G4t( z{aNJ=WpXL4K9Kmh#t!-^Uw@dAcy(ktL73U8B>kAR`z+nsvA?baXhinfcB7>jFec=b z0wW8#(oM&9_Q1AGYehV*oc16-2`%*PB~t$$2r;j7;oQLsMUReL@-s~p(MOh3?;&}g zDPIEA3%@4AjF8*P19eP8>MOkZ4{bQje$Ff~y?$>V=E|Huk`;Q;%jt6qIL0K(7T&O$r%*94~2!Q*fJ-%xzH@&;61 zqz4!=<$DFiKRjQ|V%TURcW9+?W0^TK2%(4DEO3^;9L-beF)AGM=L zmJ=_m$b;^Lp|6F6Fc5>FlaPDM$_pni+*dg0J11Od=VZ&`y4Lrn->r;J|PX7ZexD21JiNv(4DOuO3X+HIP$s&OzTRgr(*9k*6t4376JQDrk zo4L2_omGJ#-V>0xcu}k&B_#@CO9KVES&S@~w7dEH+TUvlq!pgs-PV&HV=Rb{*_^WA zv9&AXhM-10O66c@mg`*^mT?BXA*zWe&8LxU!8MaMPF2g(6B=T;^>i4z>Y5HR;+k9-T1z#w8i+;{)%vvZTqh;`gn|&0I{n{f&26Tp zYc9}U?=Jt^bX;Ag>Pny_FccmpzHFF{pU$tK;N`m1;!r_#0BBpo(ZXrjotb7!SmeZk zO!YJ^y{t6%BEOKh?3;08ABn_(koFH@zN5LL(hA*Apn0`j*NkrOxCRDLgybBW9`*k~ zbb2oi0w3EEp`c3mKZw8AIarcBSp%r)%JJW`$<{lVKz8|5SSa-`$y~U9*R#ckws< zzL*DY1feb~d3QULf#|8@n`LB05Xw$b$mpx($ zPYFL9V(5P$LEYW}O8D+La7`}r9#U%6d|>{z_UfgRo6<1vy zzul1?#Y}KixW268pN#5F`ao56)DB=oKdE-TUE1~{&VGA+l;5k_wBfjq%Dj+KP)e2U zt5*(dPBIBZ!lE4?P|MVB9a{`ckcWTVTu1Ov4$US|kPV5_Up$Rh@b1N3n9OnS+tD#z z=RpP|pkOOY(m((pw`yXmOUnM>jgekFF zTrL4}`~0m6+Ie?PfU}x%h_qQ<#x5wp1IdaKcEDEDqY!l6tSR^M$If0pdp8vhEq}F) zgcGvbmwdmN=eEzDrE3TX_ER1DU7O20-ZJgqVs^jf1Ra-C9rF)i5(oFjTSLydEdb3z z{@YLO>;zDm6qC*oT>Ph&Dn%Jg8^SRd+^ySjNs1Lv!Lk8U`W;>XRn=RX<-ZAr7#NsK zhEuM@K0W`(bNcpgG@S*|RD7)of3N-{!(uEgyHx9I?V&D9fBjSvgE*4iMu&5MWzV(Q+>qD1Az7!_rhq~;9M9A_nfy?SiGN{Y2B)o&a5dYD~ zv(ZN#dm_B*OiaOBB5`>57~UPlB}0roGnU$pwlx{s*ludgn)#L<@F{jHS7`piIp$r{ zplpTT7A!;bYgOTzE?nzh0l|qElYbj#O~5cK0ERiN2|t;1D^cp8gUGV5(%DOXk*om8 zw{+x!h#uemvAs1$S5Wrgm-tASv!J)rK%c{i0nV{jUK0Iop=l)Ol}YlD-z2ogb&@>2 zGRs6oJ|lTA1I)jo!yP^5#$%HoY)+}1zTg`FfE2(42D%Xp*LVoj#f-H;c^~!%dVlcH zb^Js4M&8Ol4etfO5d6+D$OBrh%ez9Klrefj%#8-{pmC6dwmS)KRXvB*KjUx$VhOA6 zMw;$9;dJE~v1y&Is3>^wj*M)Cg|rj4e)Eo;6YhBFYKF1IO2xRZk`st|BSkt!rv=^u{<%U=mC0RMd4SUTx^$xQk@cs! zn?g@ptyaplDx6oiFI*%p*tM8ngNMmuu11XJ+iPP3E6mgVEK8rF$roO1PLXB$n(`6b z)|tB+Y~4O%LxD00s)WFA*R4`x%xAJuW7N`2-RrLkilC)!2@c*jaFSImv+yjw#S>0g&iWK^Tf6&q;W7y4VaT>`5~*W*Uvwi2=WdtR01i10sk!`Z#x z9$F7Q1%-IlmyfKT>qqV+oo(Hb^Qu1Uf(Cg_CTDJj)O*P;kB(_f*dWWqw%yERKaT@-4Res56__IcVOuyeP^rF{=(q5mzTd*xqV;7 zjsIuafB!Oh^f&1c1$5i{H3RXTbP_Tp3<5mE{AG_`1bts?2scfUut-Mr&u!b#0JfmO zIb}Ke#(A&3ioich+pbk41Sd#y9sOekzRim^vn?y$Yz+~08=tyLmanPG!9O}ra!^_w zRO%$wBcTg2eFOOeGdpW&x|vnknqHl@v|F8`OUd82me0M_Ud zc2)rhy{~a{PLelILgbs?sA-+59MV60H)$&6z)HCT#|pdwTQhFZ z-R3l)yaHd5uaGSjiI6u6TyS3)?rEg+x=lZAH9TQ7y5wDL{;autwx)}C5eupZGa+(J zYM570>0|dl`5PEOfZFk8SPmNLUQcW`L0XqjZe96@A1hV6N*4JIT!%}5y?M6`hyIVf zx$_&yFaZF_LPAaWy*1|TD1a@hv7D;3-`NUFno3?${OO04E}wAAoDuPNf~_p(eCmt> zZ9J{77^hqp{pKhUgbk>tA+3>}LYWVvG&_A!GTxIu)M4iK6P6(@-6?NDJ=&80XYG43 zaXW$inV*mDKzz=>aeB8i?$u^r2+Yl^AGFw?O|~{bqjHP!gO2~28(_rt6-x&x z)uzJ*>E3+i{Q%$d6A8e?+}|jasKeCMl^^`26v$XxPkspAkPi7 zP_~JEMz83ob_LY)%fN%0#f{ow%uX)_Z>A9nSe^l?Wgn5)Y z5^OmMeQ+FJawHn{joVvRZ1!f-&IZJ>sgX2dmDk^?+AmTyD_Oj&pShXIY*v8jsv%)l zy78O7+@qjk!LNTVTn8L>;*yR0Z$tbIfRL!%f0EU|f4Rf|8$e6k!Ge*+7NXc)casrU z%l#z6k5}Kg`K65qyp*ABhvc>+?ILnay8)XN`~>GGS89*9h85zsipxDF$);$l8x zKWPE#R=9|HpEM=Ap^h2Nb2l8h#%(AyY!Lh`I`_(7WcU76KtEX0`osR^8qXapzx5aQ z0Nx`j(}b71jOA&*FB+~^b=1m1^iOc>%u?_iGta$mSSH3$V%Q*KEqdYa_lW`T3;x*e z_rLGc`akb0*?*r(lKFuAvl0NJen+jw18jjeQeNi!m=e>=?8MV8N|KMwi#;^@OQv5R z0U|!8opnmy-V0%Dh!Ow!R8G*3m~j-Q}lUWJUulB`~P8t{VE9${yW0b2b%vM zN0=p8a_<3&Z2vE4{Er6x=bzaUhYIIuR^|yMj-w(DRqj|&0jM<=X&~OZ0@+*lx#W?oc;DP(N&<~{CZV-l^GCSuC>@Zp$fyV%0L@cSI@$4yStk*9;lD>97V>8noK5%$HKs3G#7`KRulK=(a)>jKRnWw1H^$^U=lP)66oYyvng1;y5_B`Oc z&L9*`|81$hDjWe!ewO$O0N&b|xg%WJ9tSPTblh2U-TOvCG56($4KA;JToC(>*5d%W zntS@I{wv?VgkG+$HVIXJxG%A(f?K!Ws0E+uNqvY=x*za~fX#WG07xh$&iM9k2|Ydn zNGK$!i~D~h#11IRRE`cZhQa54SyAJV_!mGteWNcHP;2XYmj`s$jgHo(EB;=B)0zOw zySDbM@Lyi#jT-F4dj_Au!%0BSz7oA$7w%6SDszq@a2Wr;7w}2QWo(OOSmcQp$3(ws z<*j*t-4dw_FghwP5xisN)hR!_+Aae!P}xi$cF8LcPZ{R6Y!&n#`rh&RT4RrUjLlLF zRmE!|Co7?biXC0VGM8cqUq0f476x)06XCdPqpqb9RTClnCn0;G!hu)$-6~UVPQTae z2-;&3Z;=Gy0}qxq1+PAO{$vybOZ8g4c?3lnuc`3cn$PNVuitTW?E3P=CUDciv|FRi zt%l-A@TY=lOlmz!JA#k;HpkI!pT~nyr`Lv$ELLp_u*~qdA`97Kycdwd>oz2NglAW>WNxtEk zYHl0M-BHpGY^dMeBL6^KT+1(R;d3k>G?5by+V*U@mK?NORYk#!c4Q?7x@boU!4ec; zE*9l34mb;_RB1Dh<>$Kf5ebR#Rw4fm0sqC~rHu*#qCFq@&kwD;OyZtGz}DJNM50X{ zB0dj|Rpg?Pl7k3GU2H%2_v?Y=iGj?5d18J26CyTXx*TWiV=d=8bXCs&$BF)0KU#v_ z3A9`7iZ*S16o@A;YJWb=&i5HOt69;8-@UfDG}N&(;c{-W>Kq;t1V6sdVWqK*$6nhj zt+$gPE-U5H`7{>~DIp}2Qh}m%DRtNG?J_fsvh>52qP17Jxfp%-WZ>NoE8NCAG}xr< z*T=lN^EG5HEIXoLjwN8An3ASPqizVc6~_ zNtk3^*(RR%dAQUG1+4k-(W(f3koR6Bk2zy--D^#lXHL;e)de%XvS0P_Xs_ao>MLHGjWjri@o;@YO-zHMHT5t z2_5NOq>F^!n}9S?dao)S1cC$*2oMQHdI?2AK?SLbln^8|L5hlWq&I0HAP}UjJ3i05 z_V>KA-*3(AUu$N6GmhgA3^(_6US~PV$qdCOqx>7v9SO8@v(UZdqmAmef>7RtVAbV5 z(dvgWfvu1BImXUUu~BK|uD0YD4hBZpUX$g{`0YB0cQh|mX|ZD|oJv=I%I~`Edm%@UlV0@xyFn`08E5C`r@~ruJ?k14t09%b&C;gu9E3cI8&({I_aS^O2CJ z^u%$p@j%ZA;r|BWdm2JSkN-BM^>>008;qQZIJ9S>4_Pe?9b9j>t|B?r1T0J@#oNkC z<+t)&xFYe+_4}i?o)1UxxWxe=64LH6p8hN^?tirL){`!CN8ogCNNwPmMr?r-yMzOs z82H_`9(ROW$mIR`J$m zlKzuFKfHO;qD57$qJHt%fpkESz0a)U7|TCXBwe%=A>o4o`s}~H1VXx|ef?Luu9rXV zwS?+CY}sz!C}3?`D?wZd%ybCs6-;~X)pP*RyARO#1uB`aT{o5(Mvc8+!zOZuVzt_0 zXyB(K<{>szcA)AlG`RW$IOv%tUttizS@C(qA&_2<*6pro(9tdeFXTIgKZKXe@cDwM zLmK468IT7hmI6g6T_j4jL3$j{bf4DFFVV<oGTIP~OwsiBW5@H~mb!OiV@6)+XY>!q&3f-#DN6xj zBu0tZb_pwq#+NKU2^EHZnDSK$>SptFBcD<`!)phynr*o#jvF@wrEbT z6id5yH==cJP-fdLcqPzq+J(<2nI?=0D1^KYYDlNu020(h9IlI$y!BRFy7^(M+L=^& zL11FNkt50}?ojR^*|aY#`HF1dd{!^=vn;uJG7m!BG}QY%mR412oL#|K1fFob-BQi) zoX#W1+Jh}rwM0BBmUE`NJw`p{sUdtF9dXyC`lIzqW&K4s7+!n zgY&b~LLS5g7~OoFNTinC`bb62XF;s!E*X(=gE;S4^D)Bo=Iz zUJHd%ORo@8e(1APRHuc6|Ai)39G8yRZkWTV%Dq*fJmdy6*)51_a&kR{$m+%grM$7c zJvy!&7E)`jZ6Zm@;CrP-`)bG3XgJfOcFHua!JVWjfszeh$>)pauEUwW*s?GF+zI z>oNzzXKu9~&Uk&o8MGrA%<4Rcii|VM zxYL;>HF>qa`ci5N6&1h}ATnfz61c3I2J*WQPa$!AYlduQ=}lY@cd_KG1;y1B+Fs97*Z9vs54n!=!}-#zPUygO$2G0?HC@3H zX29tSaHW&OAcmw=MepIqusg76&yWESL0 z{!pOBo_>#W6QaFw$TJ<#7I``?4a@58$b|O6Zc8?B!Q@uXydONLE4yOiT~rISeNAnG zOpab&l?Mx=MZZo$r*)%W426iKAdnjd0krm7{~V6giFbJkV_LSdqbsXEG+B-h7_#pF z83zuY{yh#L>WiVBW8vFPJx{3HW8@$AECMa_I;Q*f4p@-!epO7a6coQx(8I`&{!PVS z^i@~0jxX@{Bi?&_n>@O|uM;}E+q`RY%j;Az-_zFN#_{QwofkC>(sc)Xg5Lvor<$|!H}XmFy{mJ+gkzd&=jR%NY+ig`c<;i}=u5`iv_+g4TYitT z)5F=l{X&-TBi`3|8(Y`+ERG%XIg8$r8Yf(GW8!edHLic_mHFrdfK!;^0`i?yIxrn- z1JmKRzeZi&zeinWPqia>_0lh`Tyka0!egRRa$4l~@zFqPftV%8C%LmBEm22=K zSfh?WztDHy8r~DSqD&50K-|XMKD>sWhTm{)8X?jA>4_!Ws$KJy8?3QL*x|N@lw~WF znc7+?cZ~AS$kT%@ph1n)IO21Y;l+!w?kI93u_S zB0MYJeE?}n2c3G8m^fp5*HU@4<%kq8;3JQqtgE<@dfm=auK5@%_?QnpPS!DUx}u@e z5~D~achg^zTQ~3I>(boI%m@ARHW0+@!(pSruGpfbMcuI;E9xR;c!mwRBAo>=`})&gjLaIs(d?_UEdQ2zRx z=KsnK5};i&LNj_Q_TM(6uc>iaU(i4TbLHa|VbQ|)FYL$TV?M3#DUlBG{sK^iB2UOO zK@r#s!FSiVUaDI1uO2;GZ6mFA6}I9ipXr3{%#&btYOc~3fV`gJBySW2v=a@^A+ z4f_;xR?Ap6N|hZ;iA6Q>e2Kz1sG!(B5R%wkj(T3+;)T8U>BE`O!X^U+6~|J%*;pW= z`sJ0Ndx)Vh;Et~E-lhw@yt-aPFa<2lfcd9ABJraku9AF&)-3kfvg zzc5yY&-aUH#T&4OXs_SMYHX4)3chRq58PCp_zeL~!bfJ~uly%8$-3b5dpthTROif4 zn@agbq6nP*5*f`yhiG!;c}=_-=BdZtmE`6_pp&~tqqxSS#cCg@cbojsfa^Pxg;o2h z${QXS9VJ^d=y_WD9lHLm3x2nhZE*50)G#C_c7YqDT1sT8xCL>Cpm+TG?J{gQ@~=$o zU{(VK=vk(Sd4VTxbE3Xbo#QU~#>tP+S%qlk==|FfI9bcV+sZi<*d*br^R6N@d$I=5 z0pV5Au$0mxKP85gPYd#!XpFvo5|*+2Fu^+;v*5l=INO6UxXl9DB?|U!9O96$K?G#rl+2V57j-hs`uGW{^9=mPNxw7=ls%s z`J~#lCndP$z4fQAsaTzAq#t^sro&E$_)V62fx7uyyKi57_l>SRdLFnz=c)Fd7rnrw z3%9{|rZa#yzwS@@wJ8jZ4}hn{n8h4ma1L_WROMZGOPM6O>r$PYL=QW)pDyK07`Zcf zB}|Man}R_oSLJYeIIN1dfUn3jMQK}pZ>==7*}rsYHt8C<#2A2D=)#w8!>gweAv~xw zvJb%6&GtcWTwpq=!s0E*kW-2U%xoR8%7Fz#UK|Ai!#+D^B3{R9x0P>=UXf`?PJ7vo zWE;jf<5?Wh4-9u#!q$LC+}d^dhE<(@2;?1QSg*c5+zJ+g)i(f?==-A*H)J#!x#}|Q zVwl>7B_>&D%m@6)gO;khM3Lx^GHmT)>#g)=sisY^YHH?AIhbmoFvD-fD(xrL1&gho z6sbn99ymgC9nRGB$LFC(0Y9K!mW{#N(-Yn;TMgSyR5{SDvD+yM-$|4_%r6zq|3VBM zB5w9UMoV<4VhNsME+#F|gsK89topc7C;LB;Sn`-nYzPBPtIN5ZbfxXlHAbbNmRi6(TW;>uA!s1Zd z_dsLdlY~0i{ruy;S(M?U*2cvLu^Q7V`{6$bU@Mv%hybZKH*h1XOD@vXk_;$SE4TIv ztv_d% zfCX#Wdb6oTip~iWA9(e5%G5imWpt9Ck_k`@E2`KeVF*t~MrP7+czqFG`8yZMh+C~ko_Y<(;n}1(6VEs&7;=!PVa}xJGvJDz1Kxl7^vGh&NPulDrBWqCgPQ+aw>&Fk`aiK zJ({Hj6C$6z%A;H!hTZax;NC_8LMw%63bW+Kl!WW2X8^p?>`4<|KAQorKH6Qx0}(U$ zbM!5e0TYG}6MoFV)k}^%O00g#5!9GHd#^ppRS$^Z2qQUs&8XN~8djELMX9(JyO^pL zqTmQT<~rEmOGx+CZeDPPljxsJfCg01)9dN@#PXvnUX^J$xaRq8D1&p_s-qUNQO^sP7J8$-oOf`*CM+bNBB4Jey37jJoQ5nW z`<}UR*#lD|VY-}FCNf;1NgO)nNSdMZ<)3v$P5BQvwWE)iUA35y&Ero8TR_;G{Q=%q zRIB`}t;ula{$g)N7c*oi1$t=wQ+(_3JCd1$dTCmTl%QMRz(Hb24&_Sp!pkx-4fx$+{9A+ye^ zF&n^H6yXMYx+S{BzeVhDVZ!NL?fkWz@G~4eDq)uH$&YcF3XEwm>2EAEUK2ZvcUemz z*s2D;wDKJ;$t-;0mVH1MlP_wClWEFb%k_>x8OF<2!E-d;C_OSZV0p8CC?tq$F=mzn z2?KBmfy{@ut`HzQED1p9CtWjwy7Q)_u03FRxM1X{cZVVRYIJ;;s{`yRoC)QR)A*zb zj$;uxS31XbSvnXW?{DPbq?s8Vk8Wz!kTa)v2IO~%7nwO)JyiiF=8D&%D_&j&5Q`E2 z?|3VsfCF=&DP_{yB!JBY{yimBB_dS4VpkS*^1RxG#-*4uKfOHqZf>Gr7K4L(@m{6( z&DELT!HNBvYwy8Gh25~$_WW8va;sYk>{9ztdwz0hL;FU@FxXN9HlIIZ{`Ss-=)P+$ z;!6(z16jH;;pN?h<@nfp7Qgn!Fj74eQ~qsNapuN+|FtmEZHL~YmL?!OfT`LQgT0=z zdQB_-vb23mNboAYE5W;Gk@FKDlD5l6V#sp6hfn6TuLL*O9{Zz&f=KBUJ?0SZeuLK- zy6gnu&#jSv;wigHVK&emGi54|hqDk?U^r)Y-yVg<+B^hpDD7Wi}6 zqApB}9M`pa;LhQCEAiS@8~axQZ=qpKJ_CbgYgO_)ZNE2Tl1 zM3L7(L(nvQKC62>8DDPwMG_4e6mMc%{nMli>LPcb`(JtRTpIw>#n2f;!pksZD>)N} z)P}k4(ah-2Yeq~w{a8kl$O2%Jzx&Gv07UAU;cGf(_K1jaffbcls%_Zy!bSp+I^Y7~ z$x!9b!BhHyEu`X~TJW)_`kT{Pg`q)VAm7-y0XY1`TObh;g9)zDBBs%H0zYDNcQ9dOf)VJu4N%LXLOt;eoLYMdb?l@0=}taD8-l-k570VDQx6lJ68%w@>tPRtM) z5x1xvlJ@bAA+sbN!tEy7cXZ{PXAKSLK0fWXYU;s-e$2$1q#MReqFh zb%T|pfONI*r@OMb^@(SKEW+zraPb8PUcwlGP;#y<-~Qkr&KUg-^eME83Q-NY&ubUo zF}`Z6C5f&}i{F?RG0N3+Ea|k|^;^_P&0*uEDW#lkg)T0%nn33KMJ%Apw(Pisn-{e$ zKX#-6dU!{wPguxM4mpE9^Bxt*5i~}3dB>_VgE{rAalwb)&l<&;k@*5B+CJU&2LNC! zHxFIWU(Z}|r4wZHqvHUX2R*5NG5&EeCsg}449=$CWyvQhtFAth9PvLKXA#xj(DwSLxSncWfh{fN3fzovAn~CG zKfe#o;j6cVbI?V0W0bs>0Au%v|L+SWoK8VfeDth)aj6E;J&lqOb^4z$Sz7IfMbfBt z>x|3Ujm@>~k3T6vko>=Rb!%x7$NZ#jDa|)n+vhs?5;a??S z5Y+gD^3X|#?qet3K$^pNl38JIWxjn%_J_?6f2IeBdH)Pb5v06S)aR}4a^|}Ee91g? zGBZA5^vlF7*)2zE{~sH%0HVVi>An{lY+XIpf5SK<)YObjY99=#2M6TPWOiDu-@UJ(4~ zW;`zEK$=^ScTXWFC(daNjFGLak7uzAF|MJ^6@PRJ3)B7<4sLVQZbfZoMwA0ez zF@X#_40ya|kcPKuZ0+1PnztOy8&L-`k2SFTmIkpV27CS8hX}QL-GtsHGcJ?~kau7x zmXyZg0G@3SA1{!}g9w5(NWOsJn1&>o6DjtdgytTe0QbiFm+oU{22wc>C=<}#Rq0-0+WzEXdeWkV@S^w0JEV}SRAce zp3o0MNKva0RiA^RYRK%j>>AVT7b^Fay2In?MG})~VWES9=v`&{?2M-ea9(Dh6 zM7Ru7Y(a<3;w!V#f~U8x35=Vs(UuTDJ&QdnjetA6W760Gn8wN`kE&*n#2LUq2Zv>X zV%dU@fBjs20al)~i<3p`#XdJyXrUVw1N)*Mc z5UvHz!z36vhYIhRotM*K{TH;vF5W4O5-%{%Ym8I3_9V2G=w>Fgqj?6)z(smDE^)W= za^qyVr}|c#ZcUw({C<(~2P{bgz>-&t3l%IO5FBNKmh(5a1U5-B^iI0QTKP_bv=SP> znx@AU5DRI#YD9Zi1_ z?-p)g7DRuco_#D)0w8JIlUVYn->F2eqWMsD5RS{QSp?B_pWyPEOXYTPI#J5*Gm#895|bn@JDDsR6H$FH+2fu33Uo7_&H zblb6MhxV8qk|*9gddF5vmMRS9cY&rW+7Mf7=EK z(P@eyxNXu!ARN1OI;mmm#O+Qv43mSK_vYLr1kH!xg{>GQ&mFb@!<=>iDM7ZU5IM9HTnBUAEx%gRjPSC+;_>CowSfMhl`!0UFcBMa z#Kw>Wa5?AyvT=~=A7!EXCRD4XJOzSHu+AN`I3#iZGLk9}z2XhjUp+t$S*OOK45!18 z68|nrzXKiUY1oaB*o$Beh6d9EDvVhypxb55(uBIZEWL#9(7?WaD&NuUn(O~eC#itC zTtk)o=0e592uIUE|UPT9rAri&#izCh73`7-7ssQ~@yLika zjp&**u^E2I(lv)^&F}jQ@oH#b{GfUd;i&zRJeCh2BwUgM*l{v012`Xy=q1mE+N$V# zE9Tdj_|WucSHiU#cV-?0yf=Q3(Y^sW)<#jSW-iO4<^avj-5>*Ch$Qk-fxP35$D6)5 z;WW|J*6(aJ5OJV$uf#cIP5}O00cAN6?GP=~UOlM$32%}x`!$TSUPKh{JJ-;eQr3%U zM#3;D_gjgSOdP`K>*$5(#Ah2iij$9D8HA@-9?fsTKgg6^UASkbajRt8+n9p6aSI>^ zB)-8}yic&oUho)i<@QT?$EKdd)|0t!=fp5&q{@9D;253P08%TEui;Q&z(Ovn@D<}d z$mQ_rJ1ABWZ(Zmwt6HP<&&%{!v zKz$J7`SS1$ZG7{95Ss53Lwfc>n1JjQsE*PGf88kHW-pUvr_p_jN=<#!-P04vjU4379 zkNhpdQiz>)=U4LQ<;||K8c(UiUkXFGTOONba}_=YyU=YH*h8?tlivKoULEsw=76`iAE70#BHc2m7l|Fl=J$iR zIlqNN%DaSswiK4PlIOV-J+6zC$ z+OY_uNyBhlh7UOSnrdzqRlvF|Q*!Y*Yt@hvg2< z#>9B%UOLrbHP`0piy1t!c{QhDUpiS(Q4D&CQ&$2=n1Q3!;^Idc9}#(Rr+^A$bSe74 zl^DrM*KBpJX{F`=Q8Xi&wjJ&ZP9SdG}e$eGeG%eBRWPmy7m4WPhMqbS^6oF58Xd?d> zAe<28K%xoEs?+Vz4f7M7j#cf{h(~+4UV0u0`rI3S3@8u>_J`F+3oago|BMl^8z%^DCZ4w5IH^E z_T4pYsC#={7rW{S`K%D`y%Z9$fy7k3secm(*q&`3*=!M{%F*kBKnUfe;uBfKsLsXi zcrdW590A|B%M@m-Ewf7z-(?BrTHx)9sil3%vaR1~ODx392O7q_yyKj38Vq_z*FeW| z&wP9@>k}BJb_OT{3O`sk*4*nWxw^kQZX*#Td7YLfpMJO zG7QnzIYQ(O*7}%c7*pLe+j?{;cVeWysw!JOL>-{+RnLcFyAVbW4_9VkAzjC8ATg`l zj$DSIfjD?Q{6JOQu0b(w!?=}vies6A1FZ|l(7ZV`FH1ZWTd=Fr_Iof2Tr+xaU*_Ku9x(r+ElJwTqM(c007>q4R;e) zulsrrV@LD|Gh8uYhGP^&r(BWk3;U2Z%$rMBUm)YZI%Lp_NojqZE%e_geq1%~jaPqjhj=h*wQLlUCTM+qpDk9?N0bxQEoc+3FsEe^(s}{?l z#Hd#>EG{_`_R}i>R*&<3QDaS9t%urtZJtTyQh;A?RikJ_usolAJik}%B1*24IU9BU z5_t6?jMr^D5xL3L2*;k;3@=hc|`aBQpGEWQ_e(tUd5*QqnvU)#>nHO=B@JV~Sk zmf2By?hs+@5y-GN{Y3~SA<5xh!R}Q8i+6C(x?5oWB&wS&3RKjS%I(OH4)jAkuT6Er zIj7*B>D(H!K;b8wgS|S+NZRhldjU%5?!g(lJ2WQ=%M7CW`&T_V z@)@M`z^c8GO(?NZ_YE+O)L8r;H3P}x-{z=cBP$-kw%TH?*agw84hbA1Z8)2rbgUZR z^~KZ=m%uWna=*E3Q$+u?-!lw5l*=j@wIE2}USzvG=yGo%DY9P(eLb81q&`3ZfBE`Y z4d^Z1-veDz{l!xWq9THq!WQVkJFXf)-D8#opGNI{TM-T|eq3Zo+GQyOR<`NrZI5Wo zbaH_VE^0l^xf-H_<29HsjeZiB!v*wJ@oxd)^i2T%dSeO*4b-3~dm0^|TRX>2wj`?j zcH2q@UkMRqxv6QtcQWI(rD=MwF8BWKAB_)>!e6nR$ij=r?$?5`3iTHbiTCn}x-iYq znJswNWCJZR^Ttt#`}2HyfQX0!U#oLY>vAEMWA4y@9cvXUXE9u6R4B~Pm1Oqv`-149 z`U`$LUO}p}rake+V~#>qEr(@cxq8@-TIvdB%OD zT;*5_-6i9vS(@?yPc7yptZAs~cQO1hFQTZy1g_-&FM%$Uu&bh({};YbQ9XWpxdKkA@ELk-4lHdLmdk1-pOx(HNm&I_HYpvv{$ZMII?C%*1tsMbk8cSt!FZ*PlI6c1k)3B;J zEG;it;nziWZl|G13Zkk*=rYz{J$E(CtzUgfhsRw^-Rjq^Qal8g6%@8T0gqNspf>ef zpPy=<6jL6j(>%M=eHCm|V+K>jX^J1V8|Esc-y?^_1kwx%E#^3843uw2&P2DENcO~E z*qlii+27*-1wQoP=o}o+RTrRG@q@KI!>_O2RJLK`UZh+U;o}Y(2Q>EU#E*U#C~&a% zGGi1WKS+EsRNLfyUZ~aB!p7NTJQ_2}+7EMXj1y8i)rNe~{Ch;VBzj5){mvVETj>lhYRdYC;~27$0!JpQPI7>*M&e0Np~26 zC4HdqopxhqvOp;^YJ?Efbgy=sss%}5*-6F%ET=dN$-&0BoP~a23(ixQKIqU>L*~~F zygmwa1;vuED~rT9!%l($zpmk(p1&x2b;7(WhTa`SH1-95lKjM1Oe>c}m#7CqYz6@A zP+u5?`H|#Tpm<=vN0aCGfOx8-nevDY z7lL8G{5%UQ1*YT4hN;*p&KK2AO1n(636R8KUL>7+s@lOD%B)v}#<@%WkgG2J68wS|eKT~tNpS$WY*Mn}-* z6ZtMkG@Q+nP7demFV3VvLvby7V6PMSHONJh9Qv#kI6O6gP2-Yy4|uq~IS>nvnueXV;`{K4T9jTQ zNr%b^v_sN|fV~Xh_B*_c&=y{@=F5>UiuX<;X9ipHRMtI@$8?jeNS<|D&8axzN_2MI zZp}X;pN4mOqsuwrwMV0cz^tTM)!Pl~*6J3~Op-A^!uV;cYTxRz1(8;2$XJ+`KQsEh z?*S(HD6JnYdz4yhR7CylzH^&C%(9C64>(<(qTDvFzeT;3tjE@07Q6&^?)>e{`=<9Sw@D)6ty%KFzQsf0H}+7$YZtgGmOvJS zB^}Wu1gu^;;6FO-bS8#={T#NRxe_Q1T1fOagh26D|oD4c`ZlpzXM`rf^z$oSm9>A26CaQxC~6 zCLLkYj%q?-OZ@v6n<`$h9|}XqD1-}4DSnjwZO2>EP{$gCef;+PFX850ooAC>V{g{I zWi5ag5KNehLxCL}th&4ZK|)(s%%7_KZMlb+g<9m|r=eJN{k5oF3vhGD_ zr3eLvAJqZ%Fb^Pm`?t7O6zyVvU0AyR8vumtBY6TdanG?q(P=q_$~R7r4?`77p$^jD zNc~j>4WNSDGzyWQmWS&-O>%}Wf6Y0$NB(B%%>%$UX7mGv@1yaY)Ve3GJxB2D0Nif3 z9iZo50n9p;uqyMv4~U-rOUrFdQ^i|<8AI0rS;O0A3Cmc(Sb1BU(0_XZ$=X=}2wC_; z82O9BKN{};ZARG+)jN{}2j254f!+_=JG9$c> z{4K#x$lx5IF!%UU!80Ym%(i-;ouG{7s#;oY0=!@YZD;${`R3cexEORW;fdr@c_g1U z#v+LhS^wU<9Ono47LMC2 zM}XD22(bEpaI{*FN0!~nTTs@KoAQ0=+M^pkLpl<}^Bn_ay-~Dl$hTr_# zZ3^54z;Eg05WXcl{L+wx024yPxXurmI8LX>ff9jZ))I8>E#y~Z&zpc1q0FRMRW=t;&SC#$G&PiH z>g0JBPZhICEjR51XjQpZaR#`B4ZzUndqws^W$x@vEj z{vo$E&H3|!jsBY#+%-YVRlyD2iEaWEulBwpfEVGrUHI1fs{2iyW6_t$6jLXPRKoMg zxS>*)x&Vu%Ps6wiqNHm~e#jDYMQ!&Je_9dX6c%k)y!*!ze3d%7v8n1t=-0ddCXUtS zQ6TW;m4rI*A9UZ@-y;D#orhxZ`Vz_kuwL1Y-H@yyC9$6Mz%Vb0gtwEh%iZ~MFCDFw&!t2$AC`PXHb6ea{Y*ft-_F{ zqrg9YfJc_s#7*pt{^@gg1|eWgggsCHF=D;dR5N$hE2LNlLG9|En$Fkdw}4v1W&jz$V8q%fE3~= z{7Ln2I%w}n^^odurNU-SM?t~{;MVoKBIoO^_;775r4&HTW+M&@Cji6i0N@6kZnLzx zFSzxEvq>KJ|4HJBc*dwaA^l{*lwXLV%@(y(~yAe0O;Q~v1uQ{5~>)od9QI4FnT<2 zh}gfo`m-7E)8<{GRsOQsv|8jba@VrZ1X?KLnevlh+6G3Vg2Sy!>-y^{Pv{++xc5=N z*DDIY7)IDOc>h#(z(IWgleaD)W&W>$_W$vm{;zS#XPn;Alzjc?+VBu{(E&SuE~^xZ z{&Qi3DLdiTm}lXy0j1LbD4loY|146tH1a=-6#g$+q~Jw_ljSNUYzO#1|A}rkd8Tz( zEujm(Vl>~oK##c_pozX*MRkDEs>5l1uQ>K-X z)uJ0wC#snKe1+lZu|w+m1JWORO}7SW^|tCukA8K8_Lr-r$Rch$xbsE!4NgF%_-MZw zt+2azS41zzrK%#c;Z4!soiy?zyQGS%vD*5DAlKsZ{xX$@>ZxS$fQ>47?Cc6oH8eMs zEcLokzJ$#}g(Y_RL5GlD>hBJ_ps#r=dnPk-u6pP98^%NQf_|u^3df%uX7-jHzj@n# zmcZ?4+_v^0)ZzOqy`$91$4MvCH_bm4e+ron61Ln5Hc$h95tiVpGlAgwm%C3C`hHn? zc2zQ3Y`u#QJJ~-UIYA#?*&2Z1T#XI~!gFo8A4x2t%xPw)PP3((fyvoB z3vJ7bW45_kI^);2i#j?49XGeN9xa=cb$!co?K_-}^f{QQYOyZg&|R4+cMR>Ze>D1J z<#ob#Z=-Q6DRuvXAy}@W`hAtc9DH$rl-;K&Yp6gWsDBXoP`d5xBxAy@VyoZiP_1x( z++NOh>O}b$*)U+O7@*>&mNFa{dR#WMvQi;&gWb3J zrS*r6?y~5u(xMh@#@#i}lOJ*(Sf7@>R$7I=f{33{t6ue;WA|j>ixw%XGNVp=${*YK zQF{aa^-H(Kca@d)cH|v^-BTFkWX4{ErA=s4SO1&B7B`gZ_3rwbgU-{?{CPEMX$Ik> z(1Y>FxU7L@c3*EA#r4i{05(afonJnpMd$hP_fqGa_e_UwL=ALEMmQ-}HyM)-iDU@p z>0w(3Q2TvBD>=?u1Dn?Mw~!%QTV>@w{q1_i-Zl*n?1N4smAt)4Blqr>oPBx^IUn)l zMrwr?x4cda$ipdEm6;$tnyaRvXob|Him=ZXD-(w$8~&4N$4EKHNU-Z(DtE;~C+tO! z-~(tJ$FD8RVx<=RSxfc6ni>B<$pEvQ&t!v)*W1}2n_+YF9!FVs%PqeYM0IXYyqP0LsI21O-v)pS#&jPk|S z8@Yn}tUX$Nk)s?`$EmXs9hW)2%wA^SZOu~&Z(v`Xtgg8|8os;zXpGtt8Ohz|HAdc@ zCz$=fchs)DLSeCF#^WdW^Y?-ehhX;n_DeEbBj@=37RP)h3y1f&TRzM^GF%d2EfmSZ z_nkmCWOJHvb-?uFF|gm9&oGGk^w7@rEk@L`o#2uwdej-Y!mm`t5k48V9pU+N^mDH4 z*j9dSdgoZ@tA|c`6idg2xBJ$CMTLS!;o|?e+CrzYUw@(J9_l&2ImJgVcA>Xp{SiKo zn%my{wp2!#{WhT%ltk|a{+>Q|^n@mnbXhXjN~mnWE;lT=drT}3 zd_7k!PkiKZR+bIi@&`M?FK`;iY;G*KxLOZKld_Y~_GtHd!~OEyH_3(`{Q&pkeU?WQ zr@B@$HZl5P%8scse!_cghqGW><@1!Eb%UgA!7aZ}h6hT4)3Lghl$ekEVlOvQ_|9{p ztF(SVy1Zz0aDQPV9_f95Qv5^A@l?{!;mVteozT5y&+U%i>r>n}Y!9c!>kf}2PKp#< zsyU|v#K7p?HmnSDYK(9gE~|~xzmsC+9aU8UHXrbJi#?{fvXjM>XNz>t8A6K)K{6%} zpek-3xJc8EW24)UaV!q<2Tz_bbP4>~C1g>xD$~%hC-0<@SnVx#yubP!h<7DM@n0)n zwK;7*6YoY?0&-$W)kVngVcPMn<7Lb1w!Urk)^%Z2d#l|1KgK`vdEYZ!00Am|LuJ-H${foX}h6P4BT8{UWRO+s3$t zsHI-diO`t%y=dErt^H})tAGxbUD>~Upr1At9eglUUwgB(d*uU=R~%2US{-F^V`|*D zk2&vBh0Zn225nGJ>ndx*0C7dhaH?#z%K1AbFu(w*LJyOJRN64hwW=SG0j|1A8Z&L@ zf&EziNa@yt>A|GBuZG~6K->C-YZZBU;KbeVeYcc)np%sOmB2*Y*@?hD!f) zz`+lwuDIKd%YN)MeYoFvlfS;U)ne*kv}$V-6MEVvC-kLV}G;AH)fCD zQX{ADy&v`6Y+4^ZrfMKxAqQVLPM!p8H>@v>D|LvDBFi)Dt^vFK^@kk*UCoo|^*}yT z9pVL{STp`8nW?JT_N|o2-)|$++~^$7&%*1DlT*vhhuTV38iAvJ#vtLqXtsts`r5s3 z#FY0s-{wNJ{PEE&nI96kG`+vZoh^d~3TgFbp4%-9s>nII`Zn(p1!1gN@;Xx0#7lj{ z57r*2Sma#S+H|o?Eo#X>w+lA!#tL&cl zcr5U<#Ipc?6(`%Av)GumaU|9$R9{%n19*BzZO05~0hCU{Zny@YNUrI2Y2~qM#Ow_! zxqik})*hC6t|$oQ%C#g{*wCW=Vd|`IE{odPsE+y$fWN!X`_FF41tRGr+kQH%8nKF9 zLa%$KyhnW8-z+N@p2gp|-u$@woTg>VwK|P$p4Sqv%qOx)@NxWY6|y-7qX!;U$V+eO zvva;m{xN<SXS1q}B8&o~n|8 z=-w_cvcW!3=Fks*Z0X&|^RX^qt7g$SSnWtbS^w1#baHcc2;)*(O>3r$^7P+2w78G; zUz$<*A%xLQW=GP_Jd!QPQ)2lKCIN-o_RE!K!yy3L8L*Hc9_8FRzT!|Bs(N<2+Zocz zzspd2?P|QPJz|4Lo7bO*NNsQJ)Vgadz~bf)pS|F>Us_O2XEVYwsDsj~Wc>eP?=8ck z?AmZ)MFf-<>28tk?gjxtr9)xJp;KT6M7q06NyQ4zDgy|?hU?P^)+D#Uc2ubxnf5>y?ieW!Z~=5wdnT3sU2r^o=0_U+#4bZep`Iq4@mERR%0gF~OQ9TU5{ zYCnPfIby{0C7}mnD&V|^n;DreJ`S?YHzC9Zkg5Q10L)oIZeT7WE^+-5Wp494k8^#L z>HMEr;syUy>epSHq@Qa}g?0zl=5`zsWHtv>j`?B(aCh6nH^0pou6>Ivgl+Qu$aNdF zmE0x|uE8Hwf6}%tf8Mf&22$!h&d9R%K=HS?bBj(t!k12^_rg5 zO6NCP`q_>T#Z(Zt9~8_u?UD}|@S3)SbIQEw0y@`?<{^crVv-X(O+E0nG1|Q$2_E2D zg3<3cZF+7^%km$CJ8qj*zcx(m+sW$l)iIE?ANOJUuwE~!UuMCH7r~S(aKGKH*UH1B z5h(}n;aBJsuYr4h0~Q|$DfV(*1ecL!HKO8{9ziHpb%}tmhaK*hy~M$$u(i3}AC`=n z+P-@=$E*X)J)nRjOe{Irp6B9_53~2qsv50%&(Fk7Ic%l%yu&XmA1(Fa<6fghU`6vv z3-`RV?TDF-qwL#TwyRTIH~4Y^YnyD&tbMe*UAwMKSJl_E3sJPz58fy6sqW*=U)jou z*NroF<+A2C>#&aVi2yqt0jp~%$*~*gy)T1-&TpN5vd;bmlQV(UocYfG?rlh~?bOmX zAwJnWo{3l_F>tb+`9Z|l5d4$|^rJ51!_@8Ztz@f7&zg+f?vKi+h6JA``~_WW@Ef^# z;LBy<>s-G6SJae`<2qwu;tPw@7K!o7&P+#uuK1O^cI2h&YiVb1J}-TQ)SKH!YFF92 ze*3cfD+bn+9|R{04yv`LEjogp12-)0=PDJG$u627+aD-}N5PbLWsYGCv`v_+WFxuH z+3TLKM&?j*kqjAQzAaO%3?vDx`3AAPF-IG{7d7)HJ9g#sZ9+&Yf z=bb6`P+Bg{^|AG{?LlHOkkCGf3Imh%h@r2Jkn!?XMABg3X_Qq&v!CS;2eJn{aW+JK z2uzI8(Y0^t&;DVVJ)~I$dQ9x+H4tr4T;{Xp;8R-fo5c3IJDDYjS@*9^t*xzrU;Z#F zFn#&^YwLYK&L(MWJev811s@YUvHv$e|grMr9eFQGmyTN9WB z1-WiQ=V94ZheV9mo15;#p|2aKW;;zrs(O55=s?$Yxx9PdPxWBJ$5lFQaU8;A%yozv zdU1>0k?H7pq+R2+a`!tmC%DjU>6umFO7;8=(vVJE0J~p0qK&4w8Qf6i&u_+|DLd3m zYbI4I65~gq%(*Z$OkF#pC6sOWFzP0ITqIkDfyUb?>IbJS=Ix3e6$=Q1gK-0^juWk? z^A;4MTL)UzHd}_&K5R&qJ*!uNXGDJtHmpp7J$Zn}1V~OeRDNtTS!fPTkngH)zOf@e zO2;F03aI++R{dI}eR~bO&;ilt9m0r)mZYFWWnWen^+xXm&LDP3Hq>5>ni*LMQwU^EYM_41Y&G1ryrz&v z7qC3j*_Fd_)DYy4$jQAZtdf?#8^VL(Qi4B=Zk%?xG4Gl_C z=ahy1W9@q$*+CyoWSKADQaU>=xp0YR*OgtK;Nv7-?zQ!nf4eHi54i*1Z@+GlJ{*BA zFb3Z&yW!Z2)HrSC%pQ%;?(u@Pw5OPg5WA|ueW%fmDy)mw!j^sz!lR%4FH_fATQzVsE zwl=&ofZi(}0zK(K3l|u+cC#o?BSGAeEB7|@ z!O1Xtx?p1sf35h{VR?*%I40(1-Y?k%YWKNt;e$PLQz>ZH)H_Dco=E=`RT6qH1g}U% z;GS5E6K^bJ!C?|vB*s4k=A?(aX;{Q7Gx|#9-7^6%<6yL~`L0(P>6X^4K+*0l$dj6? zMV081R@dfZ%O#5>v36*4sX9Yb&n|qmCB$77Pz^h=$TwSOv^5X2gRRY&OuWwK?Lz-N zo(^u;QK}&BBbO07NB%jPsYH7gZTs?lVA%Z(b|LA;HR&}*OJ1ooBG=|#MHpAQ-Q$;F z&9kw~1zR@Oa)z3Cd5kFYuLUN8L1w+=JKSDFtx1J2c_?)_T8S44h8l*M4kC=;UH2(1 zo+f3VC)}t`5qhiV`-YhL$Dck;WC|VWxV`4L3ie8NU%Nb6A#U?p`vreI$-noxm7#iK zDID8!P8ug}{lS)K+dvYW0TYC2E?;QFM?ciXs&9j;PiYqFTwMRkG+|CUn@|Mci>tKZ zxrHyh*KLhVdGq9>p0&^RA3c7dvyBW|AfcMdVAaUqhT%Uhl!x3F=yfA zxKK3x4RX%)V13Va-r&YLe4KSr%p_(Ao&CXjeaNn#En80R<`1-SrTbd>Pq&)c?W+{$ zwVAbtT<4kD_OnSJgVbPViGowcOL6XVr}sj4Z=Lp4Kr^&4NibDVCBkpU=`!|WJO}Tw zY@y>#Vf^8*fj-jFupU@j&p3wD&pxZj% zo{Vr8qK=NZ{rgTNm`8;0KrT3{2&1iAMaalbD>=sW_ND!x&b8iDe*Qa!j4ME!dc0<- z9hE6K{J97^wQVyx!B4-YdnU=`vZ0wNS6RJFB)>Y(7g5{l);zS^+V!zh7=jMH`AlY5 zx!a^4jQ%A4*4*>>+O^>Rjh43Wj$&ZgCk+IDvG>&OcxCM99hUXdE;Q{^XmK(u=@-d& z5dwThI9gYOEV^h-A~;L;(tpZuwz`*Q6CjOwX?F`L;lIQ(L{&h=Q-h$^O( zFBI}in6@pIl-a|3OIEKVJ?Ep)-f;=>JT1}pfn8Pc`RxZ^1b+hr4p4dzWSNJOrc3l<*!yY-BWaKR^boASSeVF}p zA_Q)d#z+VpCkD+t6MEZu_)Ikyjp`XyJPiT|5mdR6!)-;Lq3M|f5pr8}X8pOoO|9_Z zCJvD4!vm+q{dXlq1dcr8L-$mk{FH)J*LT^jq^rRx(r>BA3_Kr|S?@aRmwHtDO=-|P zsGE|yMp9t`3hZdpQPhff!K?MTcKRtXuGLxC%zA;m#GUo{A2GetMvy7oDLvFZ$)d7k z7!|c3licCiw5TG}EMa7BCqGo;W&5%aV!9e)7(*{^$x70h@ze?hw(-nf- zHVkpk0S7TINZcy9ev@1$Y|vpASvDlbIN|u$`abs))$8X@J*-;}=yBYS(kx}Mv0Ge~ z{t=Q0^}4%nEZ)lb`Fo68ea^k-1s-DCG$*g~np~T1vlXJISgfegEta_9ZzD)-2@$jy zr@*-YIaA+4R^1&TbJm*fQ+?3*GkF`Pp^<1<+W}9Z{#wJIcOk55ru{5dnOWETW(zN?~xatj*GG7$5*!;UgT(=q8Yf|2HSDle6PtxRgb509?c(u36;$ygw?P{&!nf7WCRpOv)=yc|nthzI_$t`N(Z3u_3|((LV{D zHA9jGi$kxZN5^r^y<;f|wN8TA6cqKr-x9+i+@r;uf?p^8gRjt49i$>+)9m-{}JFTg7z?9=Q86?4p&_ zUnJ*Oc=MgXm=iynI7wgkZ+%Xgd9vRLCAQursSS*!8MsBJDbvLWhLI4~?`T~s)yopH z5Gw61)8aP!LtEIvlLmB$(%t2RjkBZDj=NGXUxrglAF^?f;S$>1N38%Qm=zpZ^xkt) zS3wL7?bNcyj9RZzG4_IRfOh)8RolCqoP-UGrPP(xobBs*PeHVbx(?FL^#KT}9~U}lmIWC` z94q$OkmV$?F@$n?X1-siWk^+RhTnjBA~OR7XCcVO*v6x;sEAlWi{msy#>RO{R4uB& zA2aCswJei(;gcG?mcwQ07N(({NAfk7oY~4G7(t=TI|-WA&*d-Pjn1i`Xx&h$t`%aG66{0~Y<(F^ zd5^@szO2NNlv}LPz(q9_>GYx$g8tN)E$o?9XXZGv2XTysZxEJ7b6P;=cg!EFrx)yJ#IU}822YW@qP*oLXgZ(e%l5AQvgF5Vg_c1X2O}}ueIq%lhXv5pYUWvi znWV18cAd<+;SBKU1L}voV^g)HE?>tjT`Tj^65>8ix_?&>uSa|nv@CQWieCj=$Bb6tRclx9puOoY3k?8dZ9e zko>jO`2F|Vm%q$2GkmF_%U*>-u~qb?b}T-Vn(J0GWmY|OO@ZgU*(v?CZDBbn%njG^V}G~6h7+ImLbN>xWolEH){_|ik9ljpu*~uX!=ZfHQ|pWZ~gWK>f;v8G+zAPN`}S7=lJ?9Ky*H- zYR1S%Nzhrl$9?U-`6XpB7~Q8)7+4rIH_evEGjGQ=W+TK**xxoySXqeWD5LpRa-_T~ zcs%n}jIc8*9zx$=?$UNTnu(<%OOG)nUeX5-^!l&GY0~#y%IfhwEXz<$W5^uGs`Ws4SFLqe&Ecs?}qKbN0 zcDt&{foYC0Fg2${xx6-^5I;L?p0mekw7LNlnl^c{}vWKA&#N zlND>lK2_7czrpZfKy&_!&!_RG-4i0Zb{R|lYX}Bd<^;#0eWcF&0>uD_;+0I}kBQs6 ztWb}sPm@nYt;mu)=R>`4bZI;|hz-tDUL|?4tg0_ZFsOt03Fr113cf_3B0d!YlUJUd zBZp$IJ$|8Q*37H7$UCQB?P0uY?etk5{nKVI+V95GFgkbAw%VR|S<+SwewOx3D43*6 z4}NKpS;xQfbtj2pX>ie0kpl=8cS0!h{jD{u;*~Y#myb0pj~b$!$*=jqL2obg@vewN zir3SGLdY!3aH8WJVgMDr5k?8A)Bvza} z#w~%LyN?+UJu4C+A&4@LdxK8ZgS#(CmPupmAD`a&r21}pSSr_*Yz%6me5gV#K>Yl& z)kS5}>07#t>fhsS^6&A!SC)Wm$jqlCxQ*Oz2u)mTUa$yWd{M+_T08mq*YDa7r_Xz8 z*FHwJ6iwH2VVNAg2!q7ue;^F49AMt<7I)vZ=Yo zT6WsTFTBAW*!+i}Is_++T~yvbN8QIWHJKtqjXIC_+C1CSu7$&L4>+9eU;W=h*c+IC z_HtD6(;`OHKPk$LY54R?tFQzJHCPIsllmxWt#%OVfq72<+Y5jeXPoiU{d7sOpZ;Hy z9u|c>ea9-?F>{VZCon9(+oZtuheVNeoM70*;VUv-8`$%IV};z{)Jikh{b zN!(kS1pYbyAtop=L>8M$^Z_&Ta@rrq{AVdJ^Lxo2E9sPx(Dw*g0`#G2Mme0r@y5e) zZzu;w@hh}z&`Eo6C4jdf&kD__#|~IefY>HXOY4k{f~s5aTdGgrR_i+Zt7A(eV~yDz zoG7WN&u;LGVkNn3q<%H(u5uVpp;Hd^ybt>N)xFwsFnmh;?Z|Y!;B<}{w_gK>D6lCX zE#0S?Gy9D(>l6EsQ0fLb<|!t+s6KPg=8h;Ca49y2t$Rm~{!;&4||eYa5?@0~_$iwLcWso3MaVHT^)%e3YRIh|N7m0Ly5xAWyD!z?d( zMUUFymb~opPJrR2QyN)Xu~XslF}5$XznM@^(!nhrsi~&8>m*JqnW_diHA!K^Z2kkKP@v@)C8(A(BEe1yNCBxbnm(yv!4HNkPmR+^Q$>m;IHh+n)$7(h1lx|(3 zNGxo!M1P%Gz^~F$y#7pn+}Rbl6ihr6-4hV`(&-O}cEa=bp)F0rvZq;rXVL0i?U5LV z6*$rm%v-sy#w3(MszxX1Q-in=*n3MBmew^aFUgh;SqLuYRicOum@XCq|2_4!>vD$& z_bO18_vzSsr|tSdqDSQPxjxowGwYt)&b9Hw?gHuZlnFE-C$D|uHtUWC$kq~mCNzV*29e6R_)6;OD)q$Jn)~a%qH7TsRQC`%Ze^RsCB2jY!n@ld9I>Z?l?q znTf!L79EQF@9u6!?sxc*_*6+#tt@A#!VR=Do`g!^_vDLPteh=69uM$wj>`bi>;#{M zOK&q&XQpDFkXU9VEi2Z zRS~;~d$)Y&ljUyE$xL=;p=Bwny{^dCFsV_hEXM}KbnW3&I!q4BE%2ft$go9b!!&!K zK^TPnq3_aW4}Ju+^48o1s8FEClpvww1wKJXA^%mifz!v0ypf;L&} zefovjT<6_0w4-5abya=WnR&n1o#yXZDMQ$~Ro~(QF_`j&P=4q{S!v<|vExZyBBbt- z@|7%qMGm^9Ahl+#-LbOa631Z3K&LK)Z%H_A3Gq-r(|MvTEUb8qy@B5`$*1%xuUB2c zz=?}nDIt)D(9yJd#I#n==_cR-aiFW`K*LBRYUW2~<7ZVhaksOGpySR~mtEI(*Vlu5 zituG;2NU($3#)1%n%5rQ->wQmy}<1-tI(v8j}00r%wnrw?-uBysqTL8T~RsuQZRjQ?ATL&HH;DZSjECxsXkf8&doLgN!^16Dp3$#lctlVHw+Wc=U;vcmS4I`guPmix<(`zVs@ubzU# zqDnwEE-D?GG=t*^qdj)k@8ojnOvu$z9&i|>8knakjH;vm6Gp3^PrtR7#><~TK+w_< zUzqn5_q56NSHM@z_KdjuTC?z}FFc&mj};R&1%sbg84ULk7`8Hcdo!K+&96Uabg4D6 zbx^$wQH*T1$@5@MUk0SZYcc5hRc5;ozBudCBOu|m>qdtwZxGGdxZCZII zp&!NV-y=0vY`LFbq(%#kJ0QqYy+9X;|K3DAv1xd{LJ0!I^jdRDsy2$LOQfj9Y- zdl(veV-oclH1ARJlS;}6$@6Fu!B$*hv?m&QIbKL!1MlfKC_cH{;g#@#?%{3w*pIrF zC3PyKh1W2dh1274ah825{NL4I{ycO25zXOYv~X?w>!YU2^63T78vJT`TtRC2yL#nR zBja^;I01BgK;Q-5Q^9hBsl^-Z!S!L0#aAp={%Acpnt*5WRvvXCJsY2B-RiVoNs3tb z{)py09C90enP(UE#d2=Cg7cE_%us09eQMS(Qb)Nv^R0Z#ad)Ni!elf@cGV-yMdE%( zB}hEVN*7g3`oaJ5CdO}!^>_e+&reZH$zJy-=_c!DTq2NkH=9kPFknZqXNK2&r4Ey2 zek`m1BKOItcENKOgi)Q$ilAoT@WQ! zgu%H50}a5g*AOA;0y7^nPPR;%t^{w+XZ_W{HmBc4~k`DPPd3 z=XcRq8+CLR{oBi~CC_5hfMLm3QN`_7Q@F|(_%x9?ViZWuqXy8@*h<7dk#zu&ubPZ9 zT~P@4dT(84uI4VCNr0+{p#{f z|FwpP(;>|e%-bMFzCC+%tiw3QOc-#MAG1_SgNY%kixJ0)mzs45(zzOqU*4}2KO(;~ z&?-H|@ndn&rr>rc)i>+09Z=_2qZ0SBtLT?_=em*q=M!#j)%B}1$gdBL@Aa0RJ(QKf zq*gz{n10Jk_PrRKwtN`&_V!e)TH^c-RcQpK3HkcCT|9~%oJGFdV>#!%fwSO|%KGb2 z1`df3)rJn-_5|8exT^kR;4>lY76;GjhV%s?>^;$;c8fRLVe+|(?7V#+o!fv+SMd@5 zk?3&WLf!P=JQkV ziiV)%;ZHcs%3J$~99bjR8y%e8m&LvwdIiQQxf~RC zx6x^Lt&-m>ac_XJd-=Xens}{0h`i+deI;T}bgZp*pQJh6Dv-30Y zc_&Wsvs?GYiSlxKK`nA{nV9h~g;{_<`d4`&fMwnEJznl2OL2(j_m3`XxN8*l+GBW}y{q+yFIiKYD2iPhd8Z$EHea;lo*xdO9!vx`p^-rgvVW$z z`$`8|pTiD}i>=T|S5~)HDozRdXG^Qwmu_(1#EY~Rv;A7wt!^g%w28Mw8S6!L!<+ND z>S>E501)sw>DPt}J2v42$$`6IbZSu~51e%O0&>o^j`46Nakag<3{3Gj?;zY4`|TN-bh>0~m0_FY z_c)|2tn^!bmqd!v=aRYO_}jrnWo*Eh+NJW2`OW#A*KqNRkFup+=F}Bl`fSZS4?g3v z8lT9qjV(&UAJWg8t2xVN<0$l`3Fd&5;RAIUwqw63@T^4Kv2FZ!@`0f)k&xD;}s z-K#f%ZgNrrMf6Dg2vIHYme3?-7Y_T@ejX13;TCDt~4JCoyqf2KblGl>$nc&uei0Q!Luhub$>V{uVdY^T=%##%ifH*lx>& zCp6SZyt_l((w~{iP2l&Rvi$NT!S)mL7r5)(rsn&TdyNyXu5Oq6t2Tz3cDjogTu|H* zOa$2Ve|t7AuTm4C*Aw$C&jx!=m0nCufhHHrFaw{aj8o;((D}TjmrF^R=jePU0 zG+ub`oG=;o_GsB+Xm+lrBXErCqbI-2by(oSN}M9zJwt}0bht5ypgXtB$XrzA0YS?# z2fBe%XM77v!EWV`%dkR~E-3JZZb_kiEg~1!Mpyr+!CA5{g<8h%yXtfVf((3}ktKaX zOB^87$i}J|gOE-LAD||Rq$;kMj?gFT!D>5X4BogYI_ooSA_wBhWka9{^>m4o5h}2; znW`u167~A+aXRIK4KdMC_t7TwWpC2EyMqQ0MaZco97bz9F*;u_JhT&)!$k`dIcoTH z6$^kr(24CvR<-lZd}YJX)MCG_jm_~<$t8PpTrU3Y(US6K50)6?7j>qhN9mV*l7a7N z-6yO9HB%s~ZYN8C&hkVD){-j=?iYe;xw;C}bWTl0iCY0{CeSyN?{HPWAG5LvU}Dy@s#Qw8CDp6xEo|Xjz7j^p1~vrgycMGqb7D+I_@7MCch)P01kqaRK& zIJZ<**LhIkl3uG`Cdg8@)2N3oUeWxdeiLD0=hc6*Njb#@&f^}II*fY0NFGUW<}d`V ziY+}(w0$JR$T=5(74WsEvj-GJX?|yFJrCl|6oe65vwZ!gaRnK<#$wql4@UN9Xu_NS zb{Lgjt>r3b7A#6zC7I?SrVUejolxA@fSX@jr%TlGYr`gPwPT!|zM`vltKNL^#wQsj zBMx)+a4Faq%4FJPVgB<^zJ*NW-_{M_VN35PW)rka0;(kigtTwW?}Y$>h=Y;`NZ;#M zdqBCAfFVoTpWQx0HFM`Z1#ZZ3n>gPo#q+U)d2?_HuhQc|Q#qFi#B@ryb|UQ>sm2@u z{19o|KPYZ7`IT&tLj6m5GHl-2N9Ek@Z?ssEP3v=r+)6N1{_E?-Lhj?TEaDUdM+Dw% zG8af(^HE809Ik9qEc`hCov4Q|f=8wjP4W`R8#GBtxm6rnLq~fmUx5O(!K+yyWykBR{BgCZfkFNSz*DXHYnk9sT&{^$qeIXn3eP$r{+#=D24wp4ZM4E~SB7g=Kc zq3Dr6MF|CY^W5MipiRLPyAg1Fw>AvK2}0>wqzKEaZH=}l5qCk0G_`)ncQ)-e@+0a! z5IU9;0n387NUMg!zMqvqrBvlc75;5jciOx}ZM&$5;lW*IChV|xB&thCh{CxP9GEbjrH9Eryh@e zI5kf|HvYR_Q6ne^r`?k0@0cW5vSIJeS{SfQVWe0}3V3)S-;_U$_fES^n~ADa?bYvF zWk*i@=X22t!oys^=mn0+F!p<&dme-kDSgP!RmANypMEYX?$h}(_a^ckO1=1>$&z4o zNny=yN1Remo&WQ^(g#ayseDK^rT7Bp-#0(G7h+fN0?1PchK>pYKL|cCaAHg0h1j{` z&|`a9C{NB$+|PA=gn3F$@}G$jDJrxWL_VvJ;QG(R!%0anrc^=;|GW->9{JB`V)4PU z8bDs~-Ij;ahforSHM%S{z>1yYuM3P&`vJ(QP(PQ|#q%b?8k{GP!dq*J_}pB9FSfvT^h)U8 zchkRr=UqmW?LheA?;QTegz8DL24%|sW@VoL->lMqz7iAmEU>cwEH}>+U=ML#CX!*% zhbah)hY|tAj73haTkmzmqao=8|9Q7UBomWGg))a}*W-T|<=+Xp0MGm*1r(3}k5Zrl z*TVUC%auuzY9^S;s^2{PpT81bh!`L_>wibUj{zk0?j+C|i_1JRF!&+#bq)AaPHxNV zaUB*b_Iv+s2s56B(N3bep8UVlfpulj%5a*kwJ%D?a4Ai*x>_CctC zv;Mm%|IRWV5EIG2BfS5+=?bSn)0yHwr-mtOo96S{N50?QsR*Oqf8Xtmhc%Wi1u7PO z_CM{?U$Ig9uh=O5Ke72gicOkb3XGjt6%fDwbrb$QO#jm{{IA4Ce}C3)=&IGB3h0PH z;ofNgL8k^ay5Rvlf5L_ZAkq|$`^~3Dtp{hWamX9CcsgLS$)`O0&)!f?k&al3R}FvZ z{zvEh_k;t-f%xtapv=I-bMCjN)$|_?QDEtTl#3_Wret@it4+NuH`3QW`mBGX58%K! zMw%Ef{zvjYkYd+Qi@q<|Bmdu~|3R9B1d|j088J-|hp?!QYIar}BUk#6PMS3O*~ebp zYrMs4oJcnHs=kY-BJp3T#sl;tjiMgsuD1obgiP)V_e8T^FLGRzi8OmaLS@4hk<9A#z zV%Z{wqd5yU667u%0LH-SX4_1QZ?<3ZZ4xBs0m7agfMRYYWAlZd59r4OPHzCx57vrC z2DWaK%O1@erk62Lw^_1aOHqXYc!Be~02kCAz_RTDhPC@{YNUOgPDgUdKL6LyE70Qs zs8hVIb%SDFMixHlW&k_m_FJWOWbNMp@%fs*Lr+H4UE>71H_Vx|82tSZC`v2H5|PB8 zJLumwiT4vexGlo>Vz_G%3&(%)&U8d4U%Jk@G!lM$Xga&-DKD_{WAFQiD)^>kb^JTT z^bUp;lM^y(ro89g&)aA&mU7!#{OsuTFaKZQ&R58HPNVO5B)Ddc7fGRYYdf3=_=$-i7VuE4*Dr+;4*s&b&(`KJHOp;+<|y3)HM=)bVkaicczq8Yi?F4@m-Q0Yo-mMU!ginlND?Bhxh{sBcdtPu^_<;E$^mR*=KK zGI3yD#*M~)$bfBLbPr>2n5b*Z=Euql-I|sBzwFJIz-ImkX!-LVA15Wu=htkCHs7Wt zK>s_j{zYw2Wc=B#RJ#w_j7@K0q`4Mh1lrHCNqDfm>fS|22||AeC3+}Uq3gWBoSN%g zt7&Wm2wKON^{;&IypSA6qWlIeH8X#Wa>3nX7&TQiWRQZzf*ISCKExEotwv2WTNr%f zFHq}#XAhu?Z$+hO)0H~c9!jfW5xLS;xwY`P4=v+5Q5jR+^l@$$f(b1>G%;p@AZMHwc%$h%Nz&-K-=&1=+ z$Us=232d@wB;HmCoVYI=pBtiZND>}=>LQk(-)&k+uy_YkgDZV2?g~2S#0@dGk!SGu zNg6szDU`914?scU$Y!g7@YwQ2qpU%BP*U99rlk+&)r*3dhaRCwB8;^T3XBdG2a$(p zgnHZ5NZ=|)Df$vI$o!`FoA)5^X$In8jfv`Svrm&?RE+@qU)cxrp&y7qmc$-A?$1<` zYMvwW$a%aM1&X?qS6>V9@F+g7rUfy3K#^O>4&)AeXFnMH!YS#NJ;Rtq zLR8_$GAjJAoyKGOx^DV6?fD*NN^)8#f#=8vIG4WSix96SF-^Eqjk5PgQ7m%!&BeBW z{RWU|b+YuCe06X0VK<Mj9&K);svCk6m9{piUn6zX+nJuvF!g=H*6HN*X!O_^|b zx&8LMNB(^q^Z4f*%x`G*&j(J&K2b4H%T2QG-7U*S3n53z4YLI@)iC2l#OW!h?QYIPqye<$2$;kWRLlEdWK za#~rtg{#udwjTwgB_Ex&nP$^%QRg701D=n|F8sBy&S9!>yVoz{A-00D0Aa?xyP0QNpX2#{x1}YJDLw``MwFfHyJqOSxdNan7~%LsCjDAu9>4 zCV^A~Fqf?jP_q3lKa@t%j|HHzQU&U-Y#MrhX*we{=m^Bs={!_EdxY8-0!qJgydabW z$3|vda3?`{NTN9sI<6xgE)wqs$JjL``Z0JSm;u;56Mzo}k3g2nT_A(ZG0}(@hmRoa zYbPn|qYh59lqj{}*8>~GRlz5BSt%|a=wq1>THx5Qfs4yt-vXwAzT)oT2f39v;d$Ww zR{jJnWSfjDRd?0kmR^IRod?}Cv3nx@JatU~$DzCm4;wAlQq(fbFQ(szx23}<3%~^W zWKJ9kc3OfGR{X5Pstq$^p5W5?Z9(-)ycFbZx(>4K3#hjoiW1*H*mUUKc(^t`-Z~5gq+$)Sg^s*hC~i9oAfvVaDO;B56(ne zrD$&QAe4%;*fFWvY0#j7la zz6^k$R+cVNVoYIdsMg^FTi#)t0azr&Ha=2-$vRnMw(0S+2FsjhwQhB~C`X`AwAjXu zkXGQe!WXX7y#;AKquFH|?^jc_Q`U$Hz(Te>CGDtc9(YiZahx$Hn4bdm92A|ddaXPm zKzdW9xsy47uMhiLry9YK=tg4ONvMQkhA2r=Kxx{p<2CU&=;B3ag8Rj0c_ftK zQN9}ieUig6P$VF{zTcka;N?w)crwJdA>@(cJLXctsE|=|dB987OB7_XxY;mmNehbb z6XYBBl8pUw`{CZtUBI;<+Q-57wxvt_ed#fh4X$Z!F~|QlAUSQ_quOY_Z+~02W3PIV zuouJnsHr&efX7`YBTm6C8_crl&o3oyFFe)!_kuf-!DXm?fYa;p{3yJF$A^&UdOjAN-IekC7uw_#`VqunI{=-($lgDw35;asN14ZLb zmPN{=@&rmJ_gE;OEzBq>y@Ot57KVLYL=2P-(9 zzvf@xQK->;;RN!M_3ZWgAi3wx8i{cJB~HY#Y0^@$SsMvAvkp#JQ*Aw6_IBz=MMSutrLg#Wz-}1 zE3Zdoj55SMx&-g)S-_7HhzA~knI)7%n$lH3Z{ zJzZI_hz7EP510dZ9350XQqy{>fv9m_6-$~Jo!bdmo6wkd_vK(4O5_|s4;RM$;@a9` zy_SIJX+_|z8Ta<((Cl`~_Nx?#ZM$tgTmQqAr^`%_<>wGa2s=>7b@XW$O9$XFj8{l< z8hLGWA_in?(TZW5X4EKduil(cvNGE$AU*08$iJ!{i)DNJoyedPD7ie^d{{Aj(=T37 zzM9;N@^aR(qkqOabdJO0*@npC1+KxHb*C5xa13XeZqmdO{L*~bgF7ZY&0>XN(Co(m zKJ@(saehn;A0kIhlgT?a>#DvX3ePa=DILMrbj2r%IYr8bU=tMgB>`AoN(}S>`=OLD zm}g&G;iTMl3 zt^C>3xz3kEk-?wg>=h`Yot!=nnz*5l&@x5BS-10gH98U*y3h%H2{lPo+Cx_#`jvdW zrb-hLLPBwE>(M{-x(NKg3%+kwv|P4ZH@L}Xt4bZ-a?A!n)fchw^WK=)ySFGkG;2yit2s;{goE!?idtMy1S&MMd?my zh8kih=`N*1K|lng8wTkH0qKU3hM_yo<`cjF|Ev@1th3I;??cuwduH~&@9Vzm^}g0* z+H|Q4`C;;IrXsYY^bZZ(35d#1XC$Zj?m+1!T}%LE%)|Mb!C1q2`debC>PIF1v#*6h za?i1-z`-nKHO$&1=DAi$L;T%(MX%|D)KxD8!YHr`CzaFUuz6=dUs!+&Oqis`5izdf zM^#X;auFIak7#0UWYANZD9`Qg&|cmXz=%-okK-&-q1fR!?+M$Y(QhkWcmN80RG3(3%7sstDRPHIjRk$9Egwz{83Sj)FEyp6 z9v?1SbHNonfG5N=CVP~aK#fOtf{J+0lX>8Xfzgd}7{JBoNJd77OBW$@(bMWGW*(fX z81*eG2bo-^+NQ)#2Ts+Enr)eV*-YqyTFIb_8^(nkM9@gA$`<-!%6VEfvFHqnMfUq; zWvFS)Ddp`8{!OwM>5y2vA^q(1GOs1gNvdDQJXlP9#Ot&|%jI#+E5EJL%*J`R-k73P zhiXLL;@qsg^(7Oe=ir1L8m2-#@Ec@Q!|i{nx8IQqfL@us`DO%2lMunW>S;s}FNA+s znb{(`IXgtjCgX#a+ZzM9rZU;KRYal>(aoN|N7@!cBca>T;66 zrxyW)YxL9*z|@;enz>nHi~>%_TDMn73#v;L9~}qKNQ5av7i=;gU2lfytO&9uyzHkK zT707M57RwE`F@uTMY0^{)tD*mveP=*`w=VuWf~FrWmXife!npDOd;iobZd*pAOh|* zR{BYuj(Ae4O;4xhuw1(@u7^02&`DD_8CciFyy=M9z5L>@F1iRoCp=4=(RM+)FFfTb zN2Rk3U-XvB{mq&r5!M`hE}6chVxi}qOV|iFR%o_fHTh-aaPXaNBmM>wj3ZoW7&IAH zO;oLwMJQc%4I(kubb=VI%%_xL~2HT0feRZD}JGkP1ra*jz<*dqaYpUc{R{N z4fKRw8Is(Rxcgdz=#Kb&79XORk8-f|SIN9C1S&tf^;w2UyB`yg_P+V}m>-RZ5sSPp zISeYBAoZn76I)imvH2{ zAP>ciW~oW{`efeO!%QXMI{Z4Z?2wOaSMZ}L>&l~8QfC*e8X!`YAj_uX9Pg)ICRX`v zkvYldAiBvL&xcnk^Jk2c*R=ByOSl+_s{xH09oedKp9W1QTB?~PmP}3!1d92#AP6F8 z4_HdQx9AubRKy^={Cn@x9PuH1`qPoG>_Sbk)XCCI0tsC@MW4sWYsG~#i)V;n5JZ!a z<&15oja!4nz6+sQ^#M*cxaHK=b4m#YIwzxE(;BHyk0@%PWDwWj2Q$3F40}Q)74G(B zLJ3Ck1tT!LVrguYt96h#fx0A!7YharBMV64lfD$Q+aFTTu3i7LfDiQH-Jr|>%x`}{ zunL7!-R$ z17|UpNs%egNxHnViN+QCRNbe`#%SC?k_TGi)+jAG1pXnmZ8ySZt7GjvZBTlfqQSH^ zMHXiE&9bchS4;=Mx8m(PK+;hTJtSJ8 z`p$u7x?^U5UY1_Iio%D+w@2u?seMjDd3}8d4n+`9asJBEz4FQ1KtJ&w-=GSpuiDd3~%)@33(u$L=v`8r(+~+&s_iR z-^!3z+fhv86&d%YGu4IuO-FLAm&o3jlS|TVl*&l|%SKjBGTySINCK&uS}GvR z_>(yX`kVkQP8VmXFD5C7Dbl>Mo1nis|K1!NL)i~{ZC?3@-dOQA`5p4({3PUnbiB>C zD1c-E^S);pW1lwF^YCwN?=5361e*C3tH=wH<0{x+!6u6p>UQQy_?T;V1Sb&t|LioX zVXn!}AzM&)m={&LLV{kgsY2&KqZY#+`HLT8qH$bEcu$mUiv*8|gC2#_L}S^E^c`=P!c@18 zL-W_`eshzUCu9r&qXcGCK8k-h4%ewtzu=pLMu8k+XRr(?-;KKpX{}A?F<7Qb+AGmdj$6?TAGZ2gXCEyZJl@bgtD?-%N+M!=aN+myjTeQs=boY z^x5-#eixL4P;>IpTs(J~vhq7qs%kBZeJ(23d!TNDPlZB9lK=IJ&~L_P zTv(;?V=|yFjp2!PdLYw?V&cXK%8uqLrMc{{?|mOs^+e7a7e!2(X8)Xo-=QyR9Z>CViQ zpL`om+)Ea9U`9RsdS)K;NYWx-cA}@qu{fa&@95(X6R2G6pXbJ-s14{MdQ^v2g3Bo3 zRWW-GT;c`(+OFMwAY=NjQX**sf*^>b)yl4{y4)lz^bUC@R#{gr(e@bQzcA!=S{KYh zm!{-X+z<77<{Tz$0w7M!glJtAhX0y!AU3gBGWEII_2{(hW&ud0-rD+!3Hif>y{2+}Z{^HFfCbw=B z^v__IK%aQ1BUN3!f!eB3O?JHDP1QgYjbnjcNP z8TtGTtkd!HJtj#vG6v;XUbO3y8t=X-ixZBKVrw8Qt)6UR(<3&| z*}4O)8%YDszD$V8$qKNR+nFPS$d5yA;*YSUg+^&7BZtLCy7O{=dz9$&_Lc$3Qul9I z;x?7ma2Ad95)p7mLWt`-Jz+(x^!Q3}Cx0gkez65>yI@S!OZ-y7cxSO4d^GM!Xr@xQ zy;6Z>-TB;^3XJY$)HiY(w7up*v3Bznph z!MqNg&*C#?>p(%X!yq^#Im-vc(}a4aon9O#U6_%D)iWB_S=#j~eP3_+F)7Z=dV{9X zLxFo}3QyWYaM5Q_u^fjmU|ge@{{K;7rAb12v647&)}>aK|A`{j^=2~Iifj@illoWR zs2%;}v`=_{=V72d^JH(ptA3My-Oo{<2XtrWJm=S$i3n{E5xbh3Uu-l|~P&39bu})%0)gyd1Tu-inc{X7mxQo8|!^5ElL*3^` z0*?&<8t$Fj`UV+*GIaD_9{|7WJde~_46#CcqBV>s^Les;ymzWvFJH^A!y*L2879-n zJtfFrelDGy#{@l{(3UQ}9pt;_6vps=e7;(b^5Xqp!0)P#Mk$~lPbusLgx46l!{h*u zXgLYpdy$5MI~fP;pP8Nrc12jhX(^K>KoLb^Rj2}%^l1P?xhv&{&#;ljY#v~79;jZj z>h0`2zZx7Qka5+7^Kj1x03MHnedg!4nM$5zldX2^t3Dj zdJ{Q$k$}W>v>o#e70sdJS4-J6^ZJtHMSZxPdl9QG;5kCw{VV#^Vo2-WEj7nlY^o&zSHAJcrrqzJAkF%pK~e{IsCdz_%eX9h@PL1FJj8PUsW3 ze(<1SNq|{i!Sqll!rn#Gd+8MeK<==Vt_;1abUzDL4jEgOo|ucx$g{C{X42>gvLH^V z>F;( zY#pgXBDCMuB2P>r2HHi234WR&*O2k-5_3g`!WyeP`8CIkaj!lGBUVDZO`n$5tf|ir zAr5JF&H@YIE?@5ImWY9=ZJTKGUdU-&S&lk*DNZy`S(dhRv?DwgE8mXqK8cr+Ny}?} zPDxHh=gp~3xFW~!I%m@^vy`s03Iwn38H+ENz$^_d-a|hnpgEg|r`ioMoYCj@+a;h| zYJK^)7htsX8G17IcdZ<{H}t=M;ujW|XqED_Tsg z+Y2%VcR%BER6d)oqL)-WAGoNbuq!Z4c!y}FXtUXh0?YNkrgpLki?BrvvMtq`UE!6_ zv-K_hwD|iZwM6-`99f|yZ^X3|u<}*B9uQl8-}qhWcoVQWuDM!C4e6`lT!Dli(;$oS za^*8>U701tsDHrPSm{jvZNsRNW)xlI>9|!nA&N|bZ!=W#X9Y23OqOFv3}a(B`izkl zz$YHn{I#%caqK))`RyYBpEN*$?KO-fp_H4mS8wjbXM_GB3L5?=3Ys+)b{f)-uJYv4 z*kN_ly2g^0r?|95ADxa8o-vH-m*U#+<^!6}LghYVTt>c9X%WGZqS_3nDwJx^eFi>I zqlxF7{oH%xlb#^=KQ8YHoC00Gq)Myr?}xPK5aU>^T05MIFp7rwH*?#AnszmDpvQ*cc&?MhyLmA=@RKVKA>pc-*lXV$= zI0G#4Lr5$GMv)5JO1^D)!?_(|S*eXOuRbc+Tm=O3#d^}Q30$9~82ioSGlJP}qGF@O z`JaxLK!7H0tuQv+z_iY#H(j#={aznjQL)!3mME8ah4C$t@}Vqt`}9Vf7VGq6{m+cK zOe*BAP=^7}yn!6#p14aT?_FXw@cY}bnu z$iP{@qK+>Q`@potH=4sWPxypVU_IxXEhu(IVC+qNiVExlYNZN_#4TX?YiaUjb@bVc zGl`~sO8-2_KTP#z&%Zo>rHWU9)|T*0HcfL_D}E|7XV>0xi!g~C%aeqkEOR7WaLyzM z`(+0&P=|5LluhI-O7P#Wc>Cw~r1TDugYWaV1AM5unZ#~$5CZ*gu zGmvDbR^wGF=Z3GMV!<3tU)u%0dYC4Eed*LI+dWtVjxk$Q;$>Q5m|hwDi4assqfu}$ zgHd*)jD`d~FV$P}yI2W}r|^F0`*`trkJ9=!#FH^99#D+Pc>f;G;F!Elc8lWDs#jHo z7RWyRZiuFX2Ye;7NcT2^vzh?4k6G8gN$?52p%c(BU%iWV*rYpNvTwTxlYZ~A-VP3E z;ZMs)W8!%FyW(x1;s+vYt~u$on(#yV6q!gKT3zg$QLv7`GS07>ElSroR%-xBRfOY- z1%6XM!Ep6%2jks-a0jXOEdbtI4D=5c)?u6~wSUkq-{uk^+3=UZWZHA&D?<#jhx5Zy z>hJTT*ddf^R#D{r>kO~aMA*x-u^!5#A$-(D5*?ge`&rhH6TR9>32jI*V_B1&k`)ch zS*phB%d4SBQl%dRWAtlv9`Xpv0MrD{Z&2d(8LL@l67YK zqnbuhVj%gTpI9`?8wqGhx}gL5VW}-7=9qk@Y&E4ZsBiG715R_b7;D*GlIOi2)z7+!oVlp zm~NmqMHTF0oMQp_XpgbjZTM<|fGyb(r-Q-{kXriy3#q`U4A%NcRLW=6FUW}#U8|U9 z`0wtGUU<`I7SCu#$Df{IO1}0Yv^v)%!hmqXB2>L&_KmhQh+yJ1Pm6FYw5?aRMfXZ} zfbvq$&A?!ZRk><0OAhv?BZm10E3##}{aiz+6OkCkOsx~Ny|enXVRC=d&z9vGu;Aj&Wok|0q!5b8{|mAmWZy$&wx@n2o$MnBIPD!nQ_ zi5C-+$6&!{yY%!8<4ox8&I!YInH zzroJ)THLYq!*Doa;K!iyD)(2Xq#-~{%yJvrU5FRzt7OBg5L}c@|Lp%+;($$xSq{exR%pE!o@nebv5d!Qp~S};56N=Fmj3LTP7EiGQ) zrxKxF#HL+5GJwI~b^D=h1^TrK51w}lL>=Rg{PB71y6-^9^=*?#(BxTP4V>)gH?*c# z6DI3iJURF{bc_O;GCf3x0MF+VJzlEkoAiFOJ6_&)xVL6cjtDtyb^_YH+RnPY(Y}P1 zJ_3;}gr3fFdrD%y%4N7uW!MIf(A^6n~(B+>m|J7)Xo& zVQhh`=3>A%LpTw+xuKcFD>Y`IY}HKk7n<3$%e8HN{8!BZ{hyjc1L|`WWKgIkPCV;J zW_wT03a>7B@dU4lt0k9ZpscP%o;>;TqZB|5P#Y5yC6t}SiilwUk279j;Lq#0p$t@- z5wPf>;NWK&vyUKmvKR2*!IFg_bs)G5i0weZ6qgeta>Ti%!Wwj)1-<_6Mbkr54sMa{ zXaY`lQ096_R9kytEIz!;sqU|z4_$d!)JVWVW~{TnWAGB;jGXdMA;G$WDS^)->=B#x!$W6(&S-H zjsDXAWzA`C68`fbEISaVqb(C`T0J;UR)%l()pgBx(v!Vgt2-!=uAv)XAxERaf13@G zb#s0;1yVnes=@n&h1@K!BPS`!@{u|uyH>I*4wlGXZhNWex~em3gqvcT;|jk_fJMhO zbkv7WwM=aCAk~B$nC-5Sll|iQN}t5ha`J|gJ#PcFlnh1^f(vg;fz&n!$a*(vLs{H5 zKybTJFdyM~E}!wJX}lCxdghZ0_HdGJ$;=5l{-2AW$Thbdx$Pq?TO7vIc#x}_pn zs<#bZWdm1^bAKS20vG%`z`(PgpPXV&G0htj#11`w`75}pRuf)hU$esbcCi0qFMYs+ zQnf%kfMZWWFdrVWz>-j!q`gxGWRw&XlWLv`v^Y+xQay3TFr}@=V9`%1UZF{%k?qC` zL`vw;nK`B-BBQOP!9arzRrfY|r1@Xj;1~j;-qZEY%VsSBorAW!EP|)y`&=o5Y~EDD zZu5f)jkCvz_#^SVCFk)zyU3f;o2!}FW+K$rPa3)46qSSG1HwqA1Agi6b|$Vgqk!u6 zVuxUb;^(K*ljFFP{3m}>o5L>t=$&bv`P(P%4onNJV%Wo&Cf&=|uG_nN^&G6jn|;?P zgLX|@t(&L$M(;1~RJ5&qyDDPEQckm-Mz6ZOGin~7$&;6BYgFxZ!8lRWpzOed+DTboHYt;my`m@f83cxrXCIKLX$F1=KZVyYT@s^2P&b zX-4NJZsH_UPfD)&Al}8@m#WM0>+&a;8nWGPXD{Otnx%XEl4vK(2*M@}o*^9@PYx3r z#_eT$0ZD2!*IM#^^=DGuHG_RF~2Wxt8B;V z5!$8m>Q=P*n0{EWp0TX0Hv)`mE)=S<7%}|wI^ykj;&*r>+4;d{)WMnInkBjop|&(L zK(xI$FfG_@W)j%UgN%lz7cRX16U4erFI%k0=qmY}va*Lt5+x%^zXkCE7&`F`lUzg& znJNx=lw56;m5&Q0-EB{)z>pSEIB6S2_{5GP^llp5X`AKZl#cTj$j!cA{ftK6=iMow z6e%`IF&1h?uDvCl24@*DUtlTDd~Om7e#6nz$rFFUMPd;T{W8`6bm$F*YH2^P zTQ4zs-%q^c;c}DU6(|)E*!#uPr}j4x^w*nW=svgnz&~*-`y27Y8o+Q9zI?6q2&7eN zFzB-!u>>E5olX_)fWK-QU-W`|DZ&{`h})ExIFjoovGx01XY)k_YMvXIK$X zVqhaYe^|FdNZfig`zW@KdJxCiFV=}U9;weyHU6d%D5NBP3>h}JjQ$3w{y$=>5&0pz z`YrN5m-)}V`Omuff4munq+nJZ-&p>#HGkU!I+Xv-A3w|E{(>U=={B;V)dkIUn@X~< zbvomW=Q6TPnp007)5pjDd;ol|V)ZqJ0gEQYu{Xxjr`9z1@#-IX`V<6MG$XiM&fnJP zZ|wJha9s%bm-C(=mAK7^Gp+j=@PsfMh?{Yyc_2t6RuYef~!JEZY>s9kS|$tR$!S zfNIkL@(PY<=?SwK?;W4f$Iz*V-T--a@ctkn4#e8XuS(y3>ks9YYSFn7zl(4fa9NA@GmxRA%=hew9D<(Tqm_lyKX*iL3GQH6Vg48IfX`}V#CTR!I?sg1f^KkZ|_JG!N>^U{irh-i&ChA&CCz+FS|Jbkr zIiPqnFf39^=~@DwAHo&OvFijGyS9Ce3YILwcsqx6cZ~3F3M9?k+Hbt1IA8HW)C0PN zZ7STCA02@qSCHb8hU^SLar>3TB%-O_XD=VRBp7)hVC+jC?F{})rRev#Dk53Gp>^*4 zPSyf*UhM3j^NE!7Dc^&A{dYaLy~R0=Ts<^QH?WOK%6ed;)=UiFaQ*&b;nMz53GlHi zs)BB`bqIW>26utE+JR#uzdIYqj8I|YMh?xY-^HDXz#R%Nq|~1Zyu| zzDJF?zdLWm>6ooa$=+~ZtXq)qyI717SwwOCZexw%jp&}+&9v#|lTVAdg}LuAaA5yo zb=@9CZvX~(&0sgL;R{^PAG@c8gN{@jUM@`og;a(#k$>$N#l?)?B% zADM~_jAal%ESu~2($J3$4}3fKj9YKlNwQ6$b)A*I^IIv4>X#Ajem75;et5$K#BQ<+ zmt0Jwcnm>N%Dp~@xKYk^uzf%e?}^J{cQHpJuzcoYxYnl~$NCDLk6EV&N^3?*9MiX~ zF%fm{y9i3h6&%sC>fhq0*{(1BX(;9v&Nc@O{p4Y^%^Mz&ftr*iKz?`9_kN+y@vZ+^ zy5IWDJh!iwyPacs10adX)ii%RSJv7HjB}e_fbT0?x4&x#)tQ}#44B2pX9ri}eOri?sbP1ngg^M!y&mS9e-#EJ}{+PHzS z!F%w2%A;+Ish%fRgx5_%e4rp^L%x;q)Yuy)`R3W-%|aRJQ)M0x6*3&b^|u^q`W_9T z)<&M|5sn9jbqyTi_nk5Pvt*B_96aWx*vdB(KiReU^#%pa+i|omKg^{4hs1p8?@>0n z=)a+Izd(z~ra3YZ)Q$s$`5gBzNj!XLJw8nBF%{3(4r4~u3mEUbEIV6M!4=;q$TD5` z$XaZ?W=kw|+%$M-&|cGJq9LCgSP7*Z9%}j7?pWxH<61Q=q;{KSwNnBGn@O!PVwGbGfWGZ#dizTzE>eDmw+x) z_4V!=?n}g~$;~cd99p}}{^dRBeJQo)?$$n_>;DA4Q@^IRnQT~Oe6cm>1j9sJZAkbz z15?C4H*JRcEQB+}ylp^Ko8z2)-6LJON3`x}g(XD3uS>Xt4Ef39V#uMSlLG@VL9F4` z?0cBzW7XT?STM8hma-r$NSbVE{ChB_akpaAODs;3 zRiex%sP7%AdyL@KtiNWQgj;LIAgw1LOCP~booSjp(^BD;lKk8lyn1mF1X-F7 zzjxekKD34WWYr{>>q|9AlFoWiTdWZD$?@X&yDFd&ga0I z0@FzVRO)lRncK&h(+|Md9N9Xa!#jcJsMaW%NG0@9A`X=WL6U2A4|0)n){M>)7p#_< zQ$}7Z+&$#>i)4avtvb(eFwPbw>WrdnJ~- zd-I~V#cwx*oG0O;o(LW%$i0kv2Y{VU>G{zEuscKW`%>1U)AE*D$oagt^xy2MvGc5P zf^nb6>GO?x1ng26??_Ww%;C?*i1=l%LxUFWdA>vn2W&9Xv60*woO!?GzY(*1QgOFF zr~PB0`rc(9+dp<$`6h^IU*hNr!gl1|8+7g!HsU_7z!0}eaz2s{eaSHe%>HS)YOiuM zMr?BRh#qTaV_!y6n`Egp3i~^^=}|gub(2GG93@+8XlDRU)h@2SdGV7JOk503U$=l( z9CAtkjFtN0{OEFYu8185R+lFz2SDhkA#Oz-pIDKdu}0JYF`S+kr2qB;AYCtD+|j0k zjCVrc2zW3c4OMSNLfO2`!iWL+935b08y$)ZU3@M1yxT;6tn}hhamk9}WYmq6cV~Z+ z0{&6D00vtP6cEZ>qXw|53^X-3?}$B5J7DFS4RdksR$9yij2zPNz|NZy)=d*r3xcPY z#q8pROoofDLB>~{pWL{P+@raATr^dAvYNJk^t>EV1oTct?dQwtDD8H^3k_5A`&Q}h z!~Lf+YfG*Jr_!>0>j5<%mC_qDQn(+9eJuheIgJp;i7wX~I!&Y(-2qFvrs(I{!GJ96 zjC0w0f%74cv#x4WoK!a1IwMB!+!u514!%Qf;E=fIwb0~ZYPEe|sg)#-yOGZ!e%rYq z4kyleynUZH3Sci-VLPSOI=_5J>*$r%lFCIERn~-fLAxhxnGz<&dUrFfmi;J|_ol_@ z=rmre*1ZlGcr2*B=l}bWT6oO-A2GbNluX}92JS=>i!p0HPeXXC(1eBHFZ$n-i31(svM3n}7wzig`OyMvQk9JpUQ zQ3_j8j)hYx0HizRV zIa7%(M+rm+Sbnmq=k82My8I3MP{18Y9CdgO&?Le{owhM2{Cz|p`rH^MY+*)j>x!`$ zg0>+mD@t=42ZTXP3tK%j+v8F(`)8u?`sE5L0i5a$Y{B6vWgoh)VrkB~icrBG3! z*RZ>tI^%^9>&)dvuVcQ}rC4&?s$K8_S5Y^1wr`hLVN;5)u-Kak748|LALp3G^`(#* zCb9}UJ}hs_bA91c0wr=IK(Bu5nPLQFyKAcPu{`xAw%+;0W~{MvBlDZAl^Hdeo+6_Z z&#Q{Cb|;zJL>Kw+Xw`d@`1KC20B`%O-*II0vfM)p;6ek;1Vd%@CY9>^S`uHSf* znjB%ZJ_m~;2CX&x&S5z&Mt`_G^_^-wjO+IH(J%bZaQy}H*4LbT|4dc8HjnNk#!D@8 z#O;>O7l`wDS0)g|GP7g-KzJc<=iJ>FW;>!yIc3FA+2!S9!eOZ<9NayJBFBTenI5Sh zL-6o9M2~~70bY0kD&Ewd9Y&L4E92mXV}`|rW$dwEv_H9ncJPoHETAG?87s&4=*vhuXXSZt&XmJ0LtyfO z(LC5^=QL%3P1|@s`{+ncuhsv4M?jz<&QZ^Ff-h)A*am~?dj`?RxOqs*^-+{sM-|h7 zH;abEW~g;{qdXXTf)*Wdo5Vctf)`(p2>4CU?Z5O(*;Q zqA*^ow71^O+(*q^51vt8B|Tf`-NGbF*FTn?Z>uK9>2fD{SndmqJ{HOGY;j=;?jopC zTR)8nSwNjyAij&mzq(>!cZtJ>P5JIlWzqV|%%O-I+HvH(bOk(1M51@X&i!qf|K(=n z@w|hPcpvDM+~-3m%KGjx>vHY97Xoiar?|^C`k2a^5vF4CzKHm$u?0tLv5=Q~U|=`P#rF!FqnnO%qLUStc)%d<%Arz(%}KNfsIRky+X z#{8K%+>CS=^(~P){!;})R2d?T^nh@Tvqj>j>5A+7<5-9b+LDKj$(4$rv0Fc6!u zT+f>Eu_|+#)WeOT+3Kc8Kt+?j&zy@R-fX}6nCRSQXZ!J1i=p~OcAEYyGpNU1Z~}I% zpKeaEo5Hza$DWP&*1&Es;tb4ZuRajXHx%2urkh*snws19nKmr58-W!}p=x)srzil^ z#zT8qJkzrL|6o6H66=EvpzDpyTbNc2u)wk>5O0|7IJHF^mp@H+%SDvU#0H-XA4AVY zzB%f3-`ICfo$prpK$C*@y2sMjrJBeAWeDwb zid^5XxqW{_)Kkt+>1OK0(}o!jo$0QsDgsZNyF;n00rC1^0b zZ7S6+G?sfvzb>Yt&8~juP+pJg{AFF}SL3^zV_X3VnNQ4zpRU16qvPvB8l)QNYIl57 zdMqpXEf>Q5n0n&#K7zOEZ?np=@i4Q_&kr)nT(;uhN1aDnx%Ms88Cg$BjWjwXmev)m zykX8Vn=)B6A6^P_MUZ>vd95^{u5aY{IopKWPKbA&?E0r=oco%_I3o_|*BvOi$?>x; z)54{~>I4~oG-|j@_gZ0}ZuQXCyNhK^6+7v7WG>}t8VqZ%PhIaex?VA??KK{oYdgKE zqB3YTg}Yu&cbE^!wlt-+LE?Cn?3W~j91@RB7X5k*RnDtMAUmTbX(FzTX~L$}Q8x!M zhAe>Y5ud<$l@$6RmyU^u*};UPx%h3}iX3H=`@5;^TQiMM+(jaG+A#O!{<^1AAk??x z>Tl7zS=V+e+~tS|Sl%EQ>%ZQ!AW7+p;7 z3)GGt9mM}3hcEf|@$Nb|nn&#v4$y5cd|i@#gp}r22v1D!oH7`zo_hK0@Y54qbL2m> zQ6sLjN4sOkAynlnorfcX>84CW#2yB}eebnDAagshY z5j5dC%E$`0ZavYkrw=iWpzffO57#8If*KqpG#oqn=$cw-8HDFkL1NgMkWkg=MlF^h z*o$r(L6+j6T#C2sz-q!{Nj0#T^$;K0AEwDX(AtVD!{`Yfv^8;khu!$c5mH)y(cdF^ zyHMwh2=glQ({o0AfU5r1xGhCE}UBc%b(7pnbeqPYt1=he2a$zD?tLUgVrmF^Lz9CSq4ATw7~}T;lXY` zINPjY6M1`^ys`n2pFfi3jh&I&dUrFhj_%)icb|i{SU1yb7d~54I(=2eM8Ou4$r|aF246$Ph)(!Daf9lgS55vbtMuTOolY-#G2 zxJB#Ge-|OJCXJ!B^Q=aKJtu-wj>^|g7o!#ks{7~$@mfULEl!EK?QV6H*Wso0Le)qr zsG4Dg&MPk7-0tv32n|tD{slj8lHSMu&*B#-#8mxuUK|*0dw;c~#$&cyJuT8d6;@m6 z6$qXyS9G<5-_6M4&df!kK_LlfAKdU7pgk_7fXyFdVTBQ>gw#AoJewu1kzywI=YYWu zM&ld_v5ks@uTC!Y2qVfoTj@h3M6W}>QhIZ8^N>s#I*HYN0U5Z^!zRBQE=6!LL3V6p zuXg6yp1~M*w&rhddxJb`kVfJh4LeDx)?1#qSLW>KIpw!;C z#O&}bc*DRTU~dy^FZ%V$WwXQvUuW<@))OIXooe^&6!sly3~$8*{A;gCtg?kC94UnM zziK9$MG+W_&`k6?(3oi+t7sc7ztXOXwK6j2Aw;B`NI|=hQ?8D1DN%5~xqcJjU${wc zZq*H|I73V}T}G) zKPJYn);X>Sj8sWDrQiei#MU*ml{I{eQ%dsqxFDl=jmKic5=+mNJIW`Pnp4{tmxRflVy4;pu4`58L*u6VT zJ=x)yY{1@|O~!T=or6lxTgnbChc-at6!{l?k5}QnVh0`fW#tRzX_tCyI5bm{IMo_B_AK@WiQpKk}SKar=QQMlSDBQQIeU>cG zVC#tmEn4&uWo-r2uzMVj!}2fstOc5!i`2DhF$#yo(!_*ECM;v~)?MjJWvvV3&u*f| z5Ej}>6ue<5= z4^OqP!PEJtE9-1cfXSX8E{A7q{z9=hK|TQ7u7TwKegl8?0Rc^rjxO+oB&=sgIT zj(RzQ#fLX!3^De!sr%*WY;P<;u*#rhP9?t{9+5+lI>%jW?tZ;pWuejc95IG2IB?;PP_iQs+5Y0&v$Hp1Qclm>ziQUa zmHb?7A;E9cn^+o%?9344s5LRt4UWVqr4Yv7ZQDB5DsqpmZE}D$6V`IePXfrGEKIrj)BzY{qdSC zN>U%akqP=q^I6L9Te1#?i5~B;iw_!@_&39c<7=-@xxz;MX`7Fe)9&UeEBruC&(Mm> zd+>8vDT(pKTcn)sN|>^^kPdP1IdnL^OB?&YSAY`$Tg^v)@A(H+sbH_4X;4Bs>!J0yfP<~LoqKo*u6sF3WA*J+~TW%aSt;lQg>G-G%B&OA1%$%vqxGun#bIO zNJWHFR6uPpbn~Q$FD4jsG9#v(VH-c`Azi^%{ORsCuJr?TVdb7f4tbdeYNiHRCSq&7 zu5p2jPSGo3k#vXpYOK`rzP@(k%EVO zDI<{I2BcQRB*vGwM=&u1_6hgzr8_c~(dRyUJ^{z`ic?Qnm|-y6JaM=3hL=Gv1PLO5 z;q6=3vIuC__QW;{si-AE{(Nu9#q5Gy{^rb<4ik!6_I2*u=Vym}mML&R02~Y7A^g%b z2e|~Y_87hC{3#_r9`A?el!p_PaCr$~0hg-OR_#V}Zk;+4*qV&V2fJA1*+HKV8{yc{ z<|v$ML`qAEIx0){s0mSArklcZgJ3=;Ay+1`md+=+MV;ES3yZ_j7$X?D&B$Y?(9@kK zxmhQ)p5_Jv&CXqryEBghVZiG1 zNXdI;IMtop`}5^)+4Xh@yWRI5^A5O>MJ10ayd{K;J(! zU`Nz#Xir*U%S{1JwSQ>+;r7oLMT9gk^m6jZ63)hU!Ww-zt+^rXEW7xWB{IV9=Sy-C zqWftXI!lAl6+;1e36XP~jHJ^|ZhE_02;2Fv7WDZ`tR-29N~+^h2lkrI6?Tn`@3yBN z;^VD)nmNHnAq^At9*XHv;nm&qqv435?Izfh^%M4BuC0)98Jm8B7zvjU+|7lFd}}&y z0>H#P7``pnw8M^Xgs&4N3zl9q1Orb#g;)10t8v;IO97;(X(vR^+u)Z zMkj`bTJ;qF)CN%aiUN3%JO7B+u|vl zQAs#&l=8E#NFHbK4QlktcCuHmm2%}!r}w>XTxFq!dgf_jXjYvxT=HYVwfB(^2w2il zu;bc%EI}}{0}Eud?jg-#(Yol1NG6>+!gLX7N?KzW9u8(4sucEyV=EsL{1;C773?&%YFAl8lXUrs zSU#k4+;v2q4<}9j44_+m+O+C3gM0m%j?ZB%XnnP+sfp^nie4@Icx*Ad;GDOLYV6aU zbo*dD*mlSHBKX5gq}X#T802W&v7_(#WO$$kHUopH(z(Ppqxr^8NCMq1TsV6iiB|TdPTnB1TT(d;6RO zi4t^F)VQ1(9vXLtQ=!wV8AMsynTY-G4xXXAWTkREv_#GxsgWrI@g)sc!&FPq;V1sp zFL#@r=!Sxht!MeZ!3N z?&wnMH+^GhK0X{%D*P(KNEo9gx;a_?Ww0Z}Jq9;$#A&NG&77;VBkcUC9V&mQy3R4( z>rm<2!trh>_1eb89BnGq+@mCA=$|aN*BSm|Y`PprhM5r;{nugcFB{%=n9AQ?2dkHQ zs(8nlO}>(!SeoOmO^_-{@y%6l(^zFeXxBX%`(f9jAWs?o^H*1G*pQXIu+qEiZ1Jv4 z$gH9zY$Pcw$YFVd*555d^Fzo%gLRI}hlZPOQ?O}@P^dMwMtlv4-49**W+q=-C?%6TD6B&y$T2*ta7ggrV z(KP9{c~_S5g<~-aG99FumBnO07A6yJ+M?_&G?wJ4rr%AvR$^D?SKx!VQxtfauL)VQ z;{RI7k4XX=rE88M6BsH@m{y1+wLO^Yp^bYqEMQ!_T3m)cLNzM=p(_fuRWi!rs_abz z^$F_UL@@_)-%rrnRXvPH0HoW)tc;u3~icPtR*gfBQsmi-vU*ieGGVE=K=WzWVhNoQohc z(wQ|m6v7XpJ?`Nb{}pni@{W^t#DDF$H+Db^x-d-9z{QL9)U+f0_w4AO#KTLnTg2jL z@_C0nF|NFq z3r+*UykEuib>g0YzN_Cbwle>0vqow=&7O0r_R8&U@WV}6_du^BJvhjG@2bOgJd|Rr z)3J*`{4JX4*-BamWnv|~0BdE$r^kt1MXHIo%avY}1T9&!N3B7S|pEA3^5!{Q5+w=a#NHbhU7E{R3D>mE5 zNj7Z*gTwTSl^GakcNv?$(>@6JG;QXaOzRaPt~o9UN6`d~6y;mZPvsY{KL)SiO7!y9 z`YNWjzdDEJJHV@bphuo5QHgl|7-0J=(__FJAWi#k?7eqTlYP@adW#5=9*Uqym!cq@ z&_NJErHl05QL03wh2D{lNC!a#q!;NBI!JFyC-g)*A(YV1h2GEmKKC=f-< zZaZ{;zYQ;SdGk*4T}0~-(w>fqH;|pxNbUWaIuBGgZo=D6?QwqvozmxW6T4QzBFA-X z>Xbxwwv&@*xu1C_05nlqE|fV8g5gz#p{Ll#$1YQIfXRjD({#EMVOxX?D)9&N ztj043Hf6YDUl=1|k+`&6vXY`ZH$6Rb4qEo_-xd*0uRq$H?D`V=OcSyzBs-o-Oc?fU+$@>rDtxH>rt#2Vgylfg_rTu;|VK6*}n5mWg zEcgnl{k;aejVoOPu+DS*-ws zxP}}n;=W?nA9w6t!-P>q2P_pHRWlO}b>#IT714E$tniQPW9qY8Q6AL-VB>&>eMu^5 zR9xa$%xzSyG>g+x553@z;QJNr^kBgP3r_fUo$A8?J{BFA-u1Rv@JB8k&_-9u<|mo4 z5N0;lF%n3gb%iVm-Oh2sCe4oSOy%Y3pl5;UfW3hOyX>Zs$Mza`BBhb)%3V758Dj26 z0ry*8XxTBS)?Fe`CbuC8B9$U2?;%SsdZ%@`cd>r)+NLB^W4ZX8l*&)Xwa`*t5zm&E z+CD7luTQbkiC!_`i*K}mp?FBi&i6b9ycY2c{qKoZfNq{Lj_hrL!YsH$oedvsz?64$v0EsrG_5+ z<)WeZokBvX?^fequ?^u84`{)j5TV>9^SvJnoV6+#>S#3v-mFr|_@;Q8hJ>ic2q(rk zN)9#myFWW|-toaqvg9sx$`6XRd1~Mt?yNP=p&IwzY`R5YfU2~=+>LMJQzCd2GKX9D z>76Wdu1&u$4a+D`j+Qh#Nk=f5iad)rn=o=VO6!KSaT;;2cyx+HhEOsMY99|h7$>x^ z?Nk3X)~KeMO?QqtlD3qH#tt-_2Q1lgXI^C?&I}ZQ1!Ec9^8Hz@YH`4F8yC9=X>jL+ z&uLwvJGNC_h0-abb=+xZe&Q@c#ap@UOt@^hj2Q~|Q1xtk&M@XAok)FpVoG|C(l{9f zwD=dI&gEBc<&7%$+UCscXw%8#EXegVXrtB6lHTKflz#%uo-|vE&zukNx2x?5U7fLS zhHj?jRzTraoNmKWa`q}eznIG-w(TTacejEhaD(RrzY`D$*VWylxgG3^o9Z_ur(oj_ zF9zS;?X@$3TH|Ywk&!3Gv?|nWnbaSM&D(wJ$-UV=avCYSNJ*MOIzOAE^>oE$tuw{s zK@=mffK){iC-=E}F_+3ADFYL%?UN2TWPXOodyhJ(5rq6ja^xG<&+#uVfI#&%NMM;a z$t|7_6(8qqo3E3TkiKwdq&zW?clSMsZxV@r3RC`YDycgsPtr6p`) zmZ|c0;m)`qj2$<&?p^yBeu<$SqiW|E@YMu6OwSbpL2wv-z=+Qe1Db7{J?J$ zrEqqef)Y}MmwI3Ks@};qK*EZP_w%y z%JOR3_8BaFODep5snA&BB*t5`tE4aNQ37Mkn#H4EA{oVnj&!$c@(+3Wsf^FvH%B{% z%W?;xPj#xCq6B>ma+f%l911-=MX%j3f1w6D9}6BXtU0~49bkqi?CS&3k{cIxknv2( z%Sp~qF~RqUeD*KItxmFq4F+rXKEQzILrG^*-we{vko_)!dl%OlS7(pYom3$6l!y=Y z+}8K;J!ezJ>%k#Pfo$*cg(XecsiR2SwZ*=ubxs%@%9I4Qb+fDTQXfim-j?Filc#EK z4IzGFM?`dsfk2!Keqz20fBel%SUe*h1e4iWF((yoSzRFtcfBy+@g{=|hj=Kl_FGPK zZ^!fGsvnU!K3%}tb!rc?He(vb7bH=u(R0W+t?tZ~Y-_CAfL%6ry>IApP6Q0n2+0ER zylvFV>-Q}0PeB3%hYI@aLiSDR?f0l2%y>&UuZnkz9HM9uov4WZ8B(A~SrNw@bpWR>gm1Qp1b-Ixi zhQ;{zXpKGt{DNH4$0;1zHvA>tQs1?9cPEHzL^Y+E_ufigd>7eSkll8LY?QrX*OY-- z5`Sk4z2$NgVW(0tTU10k*@F2odSoX0EPxu58sbl%a&ZLEPTOO=L(p*?18N_zy}7c? zQn;P;K3g9TJHR9*%3E{D7H-7mGt24RyzJ%Y< zzXw|u#syy4+?3@@oHK1WfLj9>z z8-Hz11lR>jH+J&9nYgD%iYPHf3%L!~0LZY|6@Fd9eq>)kUUKS^xbI>+CzsK(Xq!VI zu2ewXv0yjtG#n8iXE)NKd7cr_5n(B&Sa)7t7j@>EnAk zw3|)s#}yg6_So7Jh4^-&Z`n^H>L`sh4s;)wa>N2z@l!@^StcXAl^=5!l>@}?U$YxC&cy^{7l%*_O4Uf> z!=HtHA8QSoWTNv1A;!CX5aHFfn&xnf8)Jqt+?J~#bQtAC9{pND6PWAnuW71)3vaMQmx(b+sxbOL%v zm?M^`&9du9h|;Bic!OH_qAL_TDF+1TvmOIq2A3m4aRZ5oJ>YHeyD5QoY&m9bbiI|) zi9FgG42w$DX=b}lvYf)pvUxV0)6l1^UDO+Fx#{7-6vL3;@_mX5~M2`GNqiuA}JM&ViamSJetB6*l;i z@?3?HGHt@5qkuy&ogySsu54GF*6wLYTeEO!Q;4ohmF58U_WLDgN+=0!fg8Q4YoK9` zqt)l62rn}kJG$@keH5ubg^j(-XTzf0Y+l6ujQpVz7>w{n84jFlSaoN-K{VUn8LvcEoB$Jx_Wq{Y`*7CI*eswYR~w{^&81Oq}`W^$|SXSC}NZj87z z(y`l7Cp&0Z$GEy+9@a#Kl}f{EfW`=nZcqv8roU1g<^T%pjT+H}m$ZT2Bnl^{(EA5@ zWV(LJ!KQw1(f*>`DS_f$1ZUm`7>QU7S{_u&DMQkS_{(dHm-?v=3$as*=F7eVM0T%R zSTIrzBiscqkxBP%;Eiaf&_Vt66|l;xb_*8tS%a-Hvp_fzA66hS7FuHa60Kp$QIxf*cbeAq{iZGRx{^ zRjXj*^KM22W3=R{UfYp81OPetLF7QA`HzIFl$6F(O+(z?vXAD37SXlQI+uQ5%i22y zGBn37!J&G;Gy}2WS+&a}k+YTJ&ZF_=!_{!DP=B|sajBKE?|(}pKHTQ*nr zYecK^aTbpx40q5F1+12Ny%rg<>z%da`+UC*&un@Wb*q+`#t3G~1uH+(K+)aiI-?8V z#T)tx>3&@RJV63kXW#=84~+9ptG1!VcID!S`!Yk?jyeDu`xx4tbe|8L;#hX5Fb)k~ zF_~Qa8)ekhfgL~>RT{QlkyIbB_tTy4*AF}t9rD1IShWYv3z>SzzJa)HSuAnd&S+`% z2VP@s^tMvje6zOyz0$YEhoc<=AH~8!^hl^V*VTM6h;3yHi`%I!8k6dof>8t7GJGnx0bBy|rH^~z1 zK^|Q67WfX$egeD`wm*JsmSG&mZQ$WlWAj38^JIro>8-xNZUebcZ{IY0v^+|lU`&N z$s4r{wqf%3Erx$RG+ zJ~$b3^w}%JW=HWI25kDYTPZbh_pzhX(ag?IaD z5PBlhN9WD5QN#B6=&XpDk^upu>#+c{|oiu?W1#<8TB9-rIenl%$z zaX7pZEdTw9$Mh8SBp^&O{JX50bB*8oyY8O&x`)_M z7hMmVT$D6E_6$floT{=_%upEBE^~)2ox6%Q;L!qY{QdV_c9%F&%-`reUr@6T^Wo(~ zd3#CE=exoq3QH;qMC)dBx!2n{6{%B`-+cxxu|jK(V3yE3P-$Dv47BE^=!%>57&$Fs zYBToP+<9X=RJkB{pr&XWQqy!eO2~b=X_h+ngPOrFV25{nax%AT>M8lpnmI%x23h!K zsW6jetyx@NH4G*+99${(4Wh48ei2AA*FS%GX3sY3nrC>u#A&{|XLRZYkHM7rDC#IT zl9*JKhOB)raR$1P_1XwY z_*TAG#$J9Ae44@{C@A7j);7S+?oxdcT1v)DIuV(8RMXI{ zEkK<5&8eZD@1p5={r#X@Kx?q!w0dVqLYtEYwE22E-<|8Mwt?ZdjK9(@ff$ZxAiVvX z(3V!D^q~(;uRFC_3lo>VSkQ1eh?{bH{;0y%Jnwj}@vIe9MrtIWfu_1v<`b==A~I#t z9$9>?I6>fti6CnDLo34Io|T6{DCHQx`KFT4Y!=V`^v1>3 zl>3gcIk;i#ys=Bf&K9a3M@p`mJWwPO;3n^W$MP#%(Wscc9j9+U~amib5+HPt&J27 zoXh9i2>Uu%)+>g0y^Pw}%f`Z)Z^z+=PKTBpUex7ucz2?KYieIpO2G=i$UIwUYP$m( z37qw(S(sAuMS*s{eZoF(YCPiH_#&^YUpri>s+3`KSmF}X2d1U`S14&^il&OS4W?@N zt{ajlIJ7H0KKY!1(QQe6MYu3$DHBN6+|9g9^iloh9xEmxvPW`MWX^%S(06DmK7GfU z=TVRrauu4-MfiOrej3&|w0p1K2mA0{Wse00sFM+07BsR9x%)0>?R67b06pioDQUtU z?6F$i8hzrP3`aJ@&G%wHheW6+&$r(oge7dR67s}Ch`F2Ei)~1{wzTD~Kwg;(L)U5Ur|{60%7M!1j%i)Dn!vH9;cmA(Bi(jHX7N2I@08WY0LSM{ zEIZ_=Y;R}9;#WYn0-L?nW1Jzj9a^LW?hUzb_Qk`{d@7h&`nY%&$`DNKHyK=FZMPCw zW^Gs+Z9w?7m3OJvlnROj;O+Gf5}@oCU{MdMmfv&PzBG`V~`HvNET zi9Ra;Y`6iDm`(e+Gg|9ZT%@2decEi*M1Q-_=lmegGx|r3y0waAp`%5KwsGc5GxBby zOizdADTljB=E6;0wvyh$C$2=g-9(M-0gGeC)f*hRmzM7{g*de@HtleVdOQxBxdyMN zqQVBE$|^~dW_7r8X@OEk0gs9 z9{;KpBAZW;BT50H{2DE@D5>Sy_p_d{P8Y3wVM#;sr%{JqZxCN5y7Czw*36%u8FWQnVy6VsbZc(0^gOl`h%2b z!xO;jiysygH^9AAq6Ov1+G4Yxy@%+Al^O`om3Nk}@jkU>Tw?^6{U#`}3jU3S=Pz-J zr%!^PJf({eQ-X8t_1Gid&(~!V+{~JrJop%i?R7ckpmXU~c36y3?|oRi+2(7gr$Fc9 zJWbee*ERKupeh3pR9hT+yNg@x09MQ_bLI`PCAJ}+td8llN!CFY$Yji3cxm8sJ9Sr5 z&*s;Vm5~jHUpI8IBHVE?&(7W z^)bhBT0UmPXilK!_xc%D`VA-6vET6VKn03AWv4hc+TFF#zS?Do65}8yK83zjzqp4f zHg(RWB58he9B@{te|w2@s|DzaW0`f}yxJ6cB*9G0@QsG^g*z3lCk2bubEy1{EwY{C*FnW0^7G~ZZn@AgJMXQ(md(#QxXVvK_^4l><~yQ{-r9DuJS_1hE$w6n zP*EF(O?h?2{YI$x;;L2yX5n8?oRFP zsus8>+FQ zer^N$P>+3%dncPMcKXSatM4A9K9XwI0Dz#{8df-xCLhpDRyoGZc$u4~g9TsLQ;W*h zaxTe-u_J1K-{Sh!84u{SPOnb*2)BY&ZGFkMdiHGwDAX6U;{W&AvhlA^#9&j*Kh>v! zK9KprRPa~>MGJEa94p##*$3&XTax-<@|L$ZLijS5R9?SfBz5A31Z(P|v48UhAJP50teg-^O)lNB%j+kDTdK)r+vV-%{@XmLVX<-=8OdSmy5o z$4jUI;`(1o>Rwpso$G&o%_{rI_6zx%jgZFgJNqpwKwROWe+$U%6Y!*bYQRf;4t&KP zK;D+7sqs@-*L3U|xv>!5AK^X^^c{cBIy&+?_exm*^`Qj&!MyW;iFwVmTaT^)}5KvWO4UIZ+{>|7Nr!xdk9ih;;;pLe`Kh}jQdUN zL2Q(kz>|N;SA@9%tpKg&#r;<_^A&$9m~hTL+sBK$Z% z%)G4pLQ}7`WXrc7x6XkR^z4y*VcRD4>o4&8v+(=DDX-2bAM<>T`lF|-OrPpXB=+By z{I?BV8PlUDWHHz~%9=~IxGZMcedgRAjB6k;hXv5vi^^dq-;tokF+9uaj&$d+V!`L% zhdy!#*_3dSN~UxEV`*Sk1LNL<%WEWHAE1RiKKues;{5w0_yzxSlK&6Lq>e@!7>)h8 z?qt1ipd*Z8U@?Tjh-g5It`OumBz-w+;C;Mmb#Zo#e9r^tEi9@CKL&=dRyxdCrP^B^ z&inXafbrl~Uc~9Zo~!1D>&!PnWmNc*F~v-7 zC3v^7z@XahO<+tl^4I5Rh;_QlM_tdYs%7s|!-i^iVAyjUVmW|RikH@|%`xb;NP9c_%qoQL9dBVteZksXgX`#>)1%XT_0Ro$ z-FG#RB4MRBW~Ve8U(Yx;H%)nL6c4lo5f-{BV#Xwj6ZYB-U8(+kalZzBs`6;bcK;V( z^R2(Of})ezlvCv5icbfxLrca7_wB$IKPFc%sqA3HlVm%=Y&;UCH z_ygkGtFf*K8ostzneTEh4^}@%OF84o<0|qDpUh9DmBEgk+2L0M)pIoqwYJ$ClXo3X z08N^89LrS}uq`T=-%Qaqa{qF?*T*yGgPBp4K3`d{L8jOWgViVh#RaHuo&(0I<6TQ! z>eRkC2PWKyQKZEItWmBp&e{k=Wa%oa^5vj67EdG@j#iKDgM`bi(*AqrM~> z@w+12t!L@nbFBtaEUV|<=|jIWg>;-(qcS|}BgudrZS2`pcs?NIR6W$r1G@{;?L{wO zgfnTLCOEA0mFcLAJ0l~y$}G#a_+>fF1sGBI@AC8O2b{A=D>k%{e{9Eu&$VZ;NbjV# za*O0oExvgy_~G|YS!ZrFhLG858G9Y*TnAl-=Gj(S4Uj_!9*ZIjdn8)teJ;J&%eDc< zSHYrJ8+PCV3iR$d6mE`92U=rUk3#|{q!)az6J?aRwK-Juuvtc-mo|WxifVjiJ(NKw z&K8TUx0~XJc7PQ`RHu64*~QC$CIbDS$8TS>8&xFyOy%d$y0uo|o|%spupYcy6Lu{c zPrd>->(zggJ&kPU7lkHmSgCs8z9$?6XoAijz=G{W7;01h z`@;kO_p7*Cd8)4TuL8FpH!@MuF_Ghw-2ElN+IHhQW}!78 z-DH2l*eBWW-Bf3q$NJY?jXaf``biFR!0NR8A&{S#ho+GR5qK6~fZNF~+sgd3)$ROX zB3hjI+D)F^rX$46?t(^REV=0xVDv)EU6D{kAjmAy@LjSfb|Ii|_D*RB)8pVI1qQDl zCNx*$Wd?w?z>j&3pWM>F&s62c`ZbzRwXyuV1nNM8rDoY);V& zP;~gj>+HA^tjF}LJkdpw$zUMoc*J~fV>s)vpzFeci>`{LLdrHp8FhmMsROV^Arr7yL2NCEe`Ew!?K z&+hoceT}rdZ9r$X#>vkHr~NJwukzAoq5YU4y|%$H>9WB4H5@f1C`jh9Bb168^SB~cH`65kQh7NZWuD~So^XKJ4raWMg)qy%l*yI zRX`m-)cMg%oN)qPo9S<_#0=M>Oe0%8(@$V~J?z{mI83Ww*>_xwmBHt1WD|b6ZPns??dD8e4~-%ovj}8zST1wa z+G!=ZcD;@J>6+ejb%}&KhEU1l{Undr&sm5&bvUjGS@aY6XjUH#s=-ZpVuKIIThTIj}nc) z;*#wm>_|)o_JBe3$(N_im(Y3PefIg=l*~b8U%AwueYjxU3VDX#1rANw2vqz>L<4w% z4^n+QCTy0-jag{Eu)s5XE{;{)=6o*Cl_T#6rL_`&rtoLHVSYoFqe~>hktUG$LIT}E z!M(|j;{Ysb=0a$o9Lk<1hIe%FnL?C8MD_|k;T?Z40UyyDE)+gh5f%U27N- zH4j|=r&n9x&Fz;Rmbi*-%Qre{+e*yJdUIn6<9s+==e=sogD3A&YVj={mlYblY~w}L zK~c5(2$u=NT%XH}5(X1xba(rk)!qc|2(C-jDCr1$&@7F%7ZW~Puw_J&+fC+bGu#mI zCE;L6qNE3%3XHKM;pIcM?T>tZs4H0h$lnVs1xA3UdVc&4RpMB@Di%xNn<_CBx5ipD z;&(R*dypHsH*8fycbEccI(+|O4{S@EP~22XvYF~xXT_Rp1g-Huu6uLu2PV}dE8Mh( zO`3+(!TJGi+=zwo+n9~1+Uc|E9x z1H;nxna2qB?CQ}USj1k=t{El8|S|J4Ew$bLEIojoBd-V){-tIrLdl}$b#+zBW z7QVX;dUSa_<~&5a<#)~AdJ|8PZHU^db2`a7gzs5qrF&HM9#?C#QIkh`U*#)57seq97|nbR zfg3W}y8i~X0U6>Ue4X8ZR@Nqneq$ISvWofr36-cq85Vcr`)DJnLmlaM` z@t?m7V_4zLgst?|d?x|SAiPMl;6LLDDq#B*s;6Lu%kw^0fOXo0VW|a?v7Qd|UPmt4 zM?eikZ3{~mHIWN~BNVhIsnm!KN0Tha3V-lWRgZWKQb5!OtK(j0kbfS*LJ4T*7n)DI zN;1Agx31KFE>i56o`z*OE1b&a0KPOOI^3IwIc3|NHkeM9KIe=%D z;oj&=1_2fjdelLDehw#mub(Ajp|f2GjkxIv-!tHSHznc=t{M&}3J)4*>C^ z<~ym4?b1ll$x%gxP=tN}Ha3S|O)ri|Jd`5cnkTmrbAC_Y6~6-%j3&yUMZ z?6m%O{b^{UcilGT8fwH5g#2gXwjkUQlO71nsQ^Rv#mmR(?ouTWDAp~P`LJxHpL&Q= zChn+++7j#m4hDCwJSca9EXgukGcfPs}v*IdVzMg6+32RjjY z!rj0Gr6jVnKqUn}xbPP>k$31%15%k=+l#W9F~Ssl;kO-m2laP(f5^Q)m{jy{btaM1~v}lH-HK{{{LGTQ3Cl=5jJOYW=WQ-vEJGYF_ zt*%S>?0WA79o;aM_cF!ENXgpveM;KN`&_%j@Vnpxyw}F$luSCgOGz{pN%M^0+Rn9G zY@we5ecR9OYLXFv6~I<)snTRP#{(gwq(+21$`Wb;?s-kv@LiRF97Zu{AHrY>o;O{W0fIqogmye@^d{A^ponDRcw1?r0)QIzH5y9MQne0k`qvRTt&KK;oEFwn z!TfDg$8j7-UoHnwr}r4{v=vk5XVc#nk#ldBGkn~vE#>!8zHq1c5)srVN3)vD(jlD9 z5$uNBbG35{!MHQDCJdm;$!BjCmw{>?u*PIgt2|EQd-({OMOobwwPcW-n>ZLJo@^CP zR!z?1XxuiWYIzVx6rS*;{xyRAckwk|5T7dHVsKhdzd%1NCFD8STR>Ch7t_AaOBoJ} zNMBzXuD~8;YXLd6;k_l>?BO$mRASaZkkxsM_#j~{C?aexWr<4h&JEd$rPz%d$#P5F z6Qw})hZrLc7>CMHJKQ+_o88oCae)7g+_QkEl-!_CpsW??CvJ!IS?cGMo0oFkOHI!# zNILrE%<7ItEdz-BS-xZg#918ARtX(#W)EJr^V)u^2-fA5ZamsVvfhBsp~SH9uiokM zgIdgS7N@XsCQ?|^Qx=t?v2i|$(9o{5U zkzB7cX*^2_;xnI94^N0GshPJz+LCJryF`4GA6+yh(N+%uBEa;z;l6n;%bqeL#b+wS z>xgTO)>zj)kq||ovk#`TM(KeT7d?7ps7j?+Gs)umssnJ!2)RCdpm+zhwcijd;3S-6eWoV zVbm783oP04wJ!lCu-D|<&S3s=>7w_$@|~agW1WaJ5Uzoiq|}XA5N4w(O_vt@F2hCp z!H`y?mqyT+Binmzgcy32ck`b_bb;)yc~{63@pynBbp+>0X50XcA|&H{5)|Hdp_833 z>;Mr2Ux?|1YWZ`}&VrKGFhRS|5$~Lv$oGhkCpc&tTBl1zZF1G0MXva;J@<-h*%v@_s;2sCK7hm#q|HvCFf@cJdu>- zjtP)FP@zJD;eE&Y0D+-uXEl2*#@dTH0FW%FEXDh)2QX#LBNIcyC$KLnQn*a==4~5H z+$@aXJN!DMXc%j=ak`j`Sw^WF85@Ckno9FpQX6Z`*O6Y7cGr8#$6mD-58x*lv>@{b zc3O5a8L~&@S8ue#5Ful?*U~~yX?%%YCbj11B#Y$TfGVDzK8HP=eN&k7gd)_Vno|~L zSxJ?Fw8Kulo3a1QnY;uFY?Cv)jgi1%9^z&S8@pe32zw$Y%Lr^hY~+*q#!tda`+pWi zk{NVgRZ<*5r8<2nI&aZ%2pi!4Bwa5*CkBO0X%wYv{U{C~KNK zMD5S1PKO*wrJf00*m$-=4Y|!0ey^R+?HUE0Ke2jX8A|Gm^eo=?Tx}gu{;zo+^Ub4l8j3E}~oC#`+(VO0GoI z{4CT~b$553V^UeXSqY$$4H8a)`ox|bxxL5hlA-7`Mqyg|Ubkw-ICEb@!k2Xpr6t;R zxcowT;I0(j_|V&V86K~bd#!+bVUp#q@bk4UQ_YmRDFeJMXRw5OKD5SRF0Dp&*d#)m z3sK&>i@fL}H|BKZT6d^pB|EUw2B?rn^z?~gdta@m>tC7?Gj)Qc(BZ4;WSY*&7%KH} zPx6AFs1jw??`pz3mYE#;b_Ml$PoR$1i=3!&Php=P>#xP+NL1mFOd5w=wWBAnN_1O`b;XtUd{G1J zr1K8#jvaFwZr>G_0D+Y0@+D)}qc68==4Y+&8;t;y zhI0c!`*kbW25MtciQPv5HTa8!_oZrH4KIWOJ7ObEBF!!h`~7+E)8bd7Ibj&RR;TbX z8ZLnOE(PNkKSDEqcR`!_qGg$0I=^h4p=zZX(w>%ZS^vSG=Z2;-f6&Fq&^AT;1H>d@ zKmBde-L2W_QZ5)m^v&S2*Xg0U=Ge}Sr&p0DT;77+QFjh=1K>6Thi4fHbgr{}?lWz8O&S~P;%Z2k(?aZMbT5IPBz4m_1A0&WpOVtre^Z}5a4Sik zCp+gbg00TsP$Tdu+xA$_$}SksucfAQGz#oGSkhY>?eE->SexqbLm4|^v_4?ho227| z5XyKgbC`T_cgf_w9{s*kfmjruk}hrRMhY=|4*Ms@;d1yS0CA>w7pXF}G^*I>jT_mS z`GxS5Eu{CPwKNE6`cb7KCd(chWBq9qRSZwUPftSefmdqx;`1dQfR#B<1S=MxDM`fp z>yo3&Ew+Y!D@xF$s?D;~9pZV0A5-s3iz21Z_s1snrfNc(D?eJ;F|}!6X=&+`g+&4K z%!I3twS9}5NpZ}-hyk>KTv8e}CV%{D)_)YBz*+V4GI$G>W@qavR( z;cGMVyo~s6o%SYJ9B-K^Yqbs7n4QkfkLHy(#*2v&?KCg=@|hd)n!P zA`?KlxI??|(yzEV#X82wsyxHJL12!_%k*{@1vRx^2Gcr(u4cC{^oit_7?~_e>`BFU zN0E~xgyszfQ5*d#3E>FLqo&`Au4 zD820+q*b71pj_a0xf{7|i+np6Ba{x(6*4=lkYoF42)5W1t#&%D?^uWVew~z5PHbF1 z)DHCJ?4f4sMuH3ZeKxY}NvG~A@ZjYcMzzKpgj^bshjlhkII4{M_N(tx+e$V2@c?AN zF}{yQ=XI0>0$y;MjUun)ZW>I4&|WGARu&eU)@1}c$SR-WUF;mca)#}*O(_0a5;u+T zLZ`-u7PZxy5clMAL5z7b054d;URx^JiUoeSH3KvZ_ocA4MF^aER(#`9-;qD3S7m%% zLc#M-Tppl}*j;I5nBpI;Owc!3U|zXPZ{=oub}(V=5F&{uCMV-4v8n^Ki6~lFlaYEL zf#GE|>sO}yNWJ_Z>jjvg4w6vmY9w|JV*mWUkOC|JoO(KL-f6Y2D_ zAlK{i))KaN1Z@@?jRIt~xu=V4Uy}IA$#W7NhvvIU@FC$BC$^}B!C;(|dLTh$>3s8K z`lB}NiNzDS7MnqVK5Q|Noub|8A#j-m#?Q_U0ko&gkSKHV)vu4sOe=9Nqw%@hh<2!C z-vqZqwN!>^P+1{~@NkgDW1u$^;MF}hq~!AUe%G4+SVPb(=mbc3;{au{cAZQ<3y|8U4fXUYi;VwPgaYJm`RhTwWa{mEX=9AB)S@V(F_6oo)Xld zmHQX~1Vv@A%aT-PMOzE&knM(vwWkf`;sgDA%`5-1{jLO5E?spK0IpC*$7|x7I%eAG zs+#Tc6td%<{<$jdZ+^soF)JP|f`N3|9Q$u<7f>>^m1`iias18bDZ{GQ>8nlDyb|V5 z6<;=D#a{p5K>f^qQIDIAn=;B8fBCV{ac>^K)8Kk!=H-Er8mx1TBPB-2pvM%@!|rwXL54Y+7FNB@-lhiY>`uJqbd=_)@D?9~Ar3<}uYf|U;CTZcMJ zwo)FsygT~beN}nJRs;d=!W^T@!3P1)& z3CP~}1%8bBkGx<$(baYCE5Z`6cO;(^tbk;(V6i0F>y>HdO8u9I7 zf=?82FJ-(mYP~dwgvPMP#ypyu=!9Rl0$;O7@MihkEX{ZjQuKfSg3fPamm}}KURedQ zhp9yJT2~j8xnXd1!5Zif;h-7ePBnqZ%mQF*QAm{71#)K%SIO$D)UO7Ka#(5S?n=QG z*CCl~*dX!dL%RQ@C*1x`Pxy${^IfuiTSJ8ae^!Wwn0G~l6-@%zxb-2B4g0?Q7T|Bi z`9^5f>r@L$}q z5g^R`UvtOK6Meg)gZ=-|W;EFW0#3nE*);n<&`co1@;9Z<=zl16zvWGT%h}*xTmXPg z@;~&t|KVZ&zs19RL7sFBvCPBsN@nBdb=enusAv&r#hw%>wo$g_iL16Hch} zM5R(QXGt{O=@lj7dd)l|>(u*_y3VxWa{XC8hQ`y6bsHZy1~7!Eq@8AcQ^>3vMWti$ zH1dqofj8B$%4KtsJy%JP5Y;bO+Ps&2ypS$D0I&({bEPm9Qw|~NaR`Aq+ZyrgDEEy6 zZE_=2tf=AuiTI9|GmHF^?V%wzTFCR=swhVK(pu4-TTK`B99GOVS|1I1Dfye(m-&B< z*{5cQD|N9T36&iS7$_(#vC_KOgXUm8o-_*#M0|~(EeA?vrk1z$FWI~BbR!yEYjx2B z6UdR{(j@!Wn`Ec27Sr;SG99-1@7R`Dl8A1CG`t$N22`alOpH9yH=wPbQk4hJNkZvn zz9>^*a;c15=b(<5^Ki$3ZU%Vb3)sNQbZF_kHzwiL#PL4Ap_^=;gH?MZ} zi>%)p9;^GW(7026;oG~1#Mx_lwx#wN?k8a%mNw$7Gx_FbAi^jE!S6)$Z`E(3g>+}6 zPxsBo3J^xfmU`m;^ks(kbk%L)>GpT%OgbmG24XzfQh*|7epacH%HxmvsXw*!2N-dr z`48baKsD=xJnTz^kF&Z;!){`x5C6|`L!P{e{-2%M>i{3gbCle;(po!10PsMD$GRQd zgA)g-{Mr~lp=xoboDqL0(%u930p!g6F z^pE!Rys38ebze14$;K)#Ee{*FX0rm_>Ag_)RzM{sax7Z9?7*=5hgfaK+Q<=kW$FP5 z$;bS9CX0FLrCY9axZ6O5kI>JWV$;E$k^9`J4S=}xY8&Wqt>k0TC_|KoNOjbl1Nzlx z)tM|P-+v={xT^zI-*K?oom=jYdx9B=A7c|GMQ*vy{(LKa3WIj2cXy^syCUBTxU|_| zx;Xp#g@N`K<3)D5qz58c>Tl?&_{^)4m@#fx-ygMoYbY; z`+Svpn?_a}+VuM8I5cCtc*fOvi}>Sh4F-O51!7^x8MPmmle^;vh0tiI(%47)rKr<$ z01l+_-i&<{95V~F4{s1{8u|;f*_fAr5sD9CSLg=dn!mnA&)tm_G?d5Pd-`bT4gKN< zoqp|Z;iG|lJ8eIemlS_6-52D;26@+*|K+2o5#Tmj+sq8k4d|Y83pI%`05)~isoJXT z+%>renwUH!eSXg#eE`Ue@8SBUsaDBZXTQOoLt5E1I$z%En4KY+(~;iPp?(kNvG@4p z8A@6!RC0K~4*P%;GOjvJrT#~;y>_;Qi4ImXHMI?Y-c>`XX#>P01Wz%OU(0ddkhFMt ziT;jv3EfH=Pz|L~|s+849 zhIa)8{!H5BlwHGl6cQLJCrD@|{c2F;h*_glox+u??N@xX3ct%8LC94drD=Tkym;Fg zP=|;t35!kzT2ymf^|7Xf9;rdwSI-ABPB$I(^!R9G&{-oa10EZWCJ2wMRT1>TiQ=iQ zx^of**Ae21H=}2kb*PO8Yq# zGlfqlN>GrSYQM}~?NDP%k#w$n&1d_c-H(bV27s0s>)Ts|=IOgrRivW*cSo_H8m5J{ z?mQZaW}a_ZWiu|v#y0Cx)vDIg=zP(}W4>I6_V$6jXlcP^I|jSTITB>Lu(1%~{K;sM zhW-D+*;_`%xdvOpCkeqdSc1C~g1aOHx8UyX?(XgcrwPH`U4y&325SgTqm9nXIp@yY zZ`OD3nwdZ8{9vu_SE_2)uKnyNOs!$N^z_Cys>g9(XAY25Sp+ZP z%#&Zo^^0lPA7QdO?4z96-UvJ-6*oWoBg6x%Ins1mT~oHl8rWUfcuyKaTQ{U81P_Cz z*yq(|qB1Y@mMk|k;O$W0oGT0@r{4y{LNRtnhXwOFrL+Fg-3 zgey80qrA*QuszZ2k8N~3(P|^GeFylfCp~g(uw8Y7HkDP>``lHox&D~#F?y^K-f&N) zL$2{FQvtIJS5S^2ktdalO-;+=WzS8)AOF@=Jl-RtlWZPpsu*&e1{*09DZo zsEP}>(?ew6&|j+JS5Atup#Q_FDAa}Yvr+KV(k!s7pwxlo9J~57iR0NrW1SajP7S!M zcD$IWUd&z&eN3_7!a8}yKF{f^$t2%~-5%HPty9-+it8!I$9MXkpOSPM@Dt-(xaRco zxWjT>H>c*7$FgTDA+dXU*p4;19wF`{l)PQ7mE-=U<+EMYR5e0yxfWrY`D5wv=xO|o z`~0@zSo4}3+aB_Jqn6Ai>OYK?M0PP3UqyRYonvp&d-j|Khukg5jBNCUUn?fep-iar znVky=v{;8W-LE1ZKE-DV{IL()XjT@(LS)}jC$wZx}CGf|Hbpv!8^u|d&S&C zlxcCm!PPdJ|31`rDQ*D1eofZQq}|as+I^X&F31tza)tEJUL+z-jMui!0%!58 z+Hxhcq!{t_-W|S`T0!}GEBjpYK$cfY^9~^L6LeX8Pyc24Q^U|OV1dg!F|xO=ssEy! zZt5B2huP%M&$lL1ts6<)nX3(Ts#})39e{14=-K>$%`U3=n#>ES1KN)G zk)B~$V(ZXTtyKB*T(N||(&dFe68ZyBRjNGR<;nl1M5 zGk@Ul0p1FLKg7aUP@o%Ug5kxIYlke22aq3kn_a461Pb>OIdrYLVCz7nd_s>y@V>x?5lNbk=>c+BT}4qZ z8KuT}VOzFFZO^n1kNc}Eq#$CSGabiWsA*q#Z?9B9EiVg24S1)y4m=v^ZCX=H}A1O!X5pE zuaL1NV!d3WAwGMJOA2=?if?=H1(nIuM*qvXGvKIe0<0sV=um-30YVEHJMSYENZ9Q0 z%W=|u?ux&PxLfk>sKiL5E?MvdDVNVJ8eUkOBy=E@C})=}O8y?qr_ONifMjw+N;(F9!G~pkZ}y}b=4ay)P>kFMX3$e!-__qv<)Wyn-_$* z&lNvOJX-^F1TW5OTgpG*0j{5W%fIgHZJIivm)mHg-JG3KBXj9RvHP=Q$ zYDsT5s;<-bKhxWpd7(Pi!k4mDIUbmR6I%|b=f#PwvbY;{gU`;d*|(R`xOc%WzfDhC zZFl~dm$JMTS6A-TQ9mIrWTcqJu#nR7lqVW}Wh7MS? zyX!+HnSorq58XfsT(au}(kfSYMvRIH8-&qAP*b$$^z&BxRH?4Fo)F02JpRVw#6ud-hk_GfI~SA zB_9M0<lbS%Qsj@mxo!7g%L4CO4->#YB^=2&v&=@&=Q#@r zrTuZdHT}MDGzMWbD9UE0D)|EW3eZ2_Ow8kZ@Z)Q6t0nXCPJs2TZN}SoEN1>()+RbY zwhn=fOXgI-3WKuFO~43?uWW^*+HM!nBL*RN?d%T;v^$MQ(ii8BQ@j+!T)z;n2|pMXsrpXx0ZID>o&KK}Z0 zYj_|>3d2e?XwB=)`KE&C9j_H&lLZD^+-LrN8zQyB<(88TcMwST)57mvEyw3K(%J1poS=23w`HF;_@zHKJpEt-^6`p*-Ui-`Zp^dl0q6W#uh4kAodmWsQ-6ge zK2$t7rM|uJ4vy7raLeJ|T`V}6z(s9YsylS>SS9TQ_Bs^<*6zkpDWNCL^+*$(?k6=t z%HliC_6v*)X)Cr(EYnEjYunT2tuZf$(}58r*Xy*=48s#53!A34&gScD&#ZkwE+;p@ z$-P*|U0vIJ6vDgZ0~TGg@ei;C@>1+J>_29(>3XhX_IL@(lx#h4gD{vC> zdvHaYW@dLD=`vQ($2G?Tt*RS&;MDco+TwW`0{Uda_m`Dvp-cC-kb@dZn~%PL(}0!0 z$VB;Bga1;=9mjS0)hGgCYx=S4GP}zWZnn3@2_eT~utZ;?(-r%?-Bppfa65Dd?WD^~lUPN9|ifU3lTq^gtHR{#;qL!^!rPrQe-oI=F|*c%#1QC-oR_;k4}3^yHF-5V*~?tbOlt$Nd42CT;VzzdZBy0d z_zrs4{x|QWJ&BfR6vumDw{@#lUSXEEfAK<5IPb{=Q5G_&^uaZ-CnLkQ6FLlVNkPzy zn~gDTOyby)&GZr*9U~K&B}fw{%3(vOXsnaLnPz1>fu`g64J>=9rhdowrFL+K%KP}U^kyE}jhsfZc>b>2r=HW1&5&&Z|>^)&gs;+dAUA%CH&wz2Y| zvUjp3Q%8+cbsD#K4KScCZU+=U1~8u8hTH8e5&WUfmr=!N&Fr+6iVnj%Rs$NCha!=a z$eQc8+o$;Y7K`k6YWOxQdu5bHtT`D7Osmr&4C`Ja2PYqP+dn@o^>Iv8FugeMx4HuT zz7|LwGCK|Tzl@jY4)z_h$VuBY?N%Hhv!0=NFSXNPv0=%Ydr%sRNYY z>z}!AV&Q)9Ux9!m5TWty1V)qt9dxVe+l6$OKbL`??M31dAynX~O?G}YeBqfMN=#tf zu;N2K(*&LRh_(!@AGcX|GfrcDFPnCwkoRgLvU0Y5qED53icJkjm+zey{pC)|mqQ6m zP)ioN${Lmr!1XC98p7*XwK42;Xdk%O!@IDewBILseWoGuUa+1IG`5}T#fuMkdzxqz zGVtTJWymC>b<*0hQjyW6rB{{T^&ggP2;PXr7Lap2T_09*1exQ~FUmL)IRd^K#tju6 zS2tyFt<7ZBOcRu}Zgv^H&=t+(zU*^CEPrDf?hHhckSMlaKK&wG@;sm15vjLc6W zpOX%}qz(7j7n+u)OjLQrwm(D9;ZD=vs!R&Iwcj$&+%N#5rXP`^a4}$v|Y`oKIs9Fl6 z&wQB{w03DWBDUUw*Z{D%8I8DHY0;eSfIDfM3$_Va({N#Zh5u%7Vg%f!7hbYL(dy!@ z*IVwA_$#zxBj$8+4l#^9v+I!acXhMg3oWhHmhE-h0o_f>Hn=)iF0EX$PgzsEuE*82 z_0_Af-f%P-zJ<%w8+Dulj7S^fNz@j9eCkCkwe7e&eZSl~{}~J{)f;78&@xc(Ai~rg zI^|in@$kCpqcrJT&}|+=fG-@VyZ61Wcr6`ac&I~$PGaWw`{d-Lrm1yh>*U1d*V>x% zln9QiD}v4EK;bV5&z*%o_QpoXrR65ZS&$qLy`e^a zr}Num4FvLw$EJOH+S?Yp`##VF?_0XfyRz))vN2v>mx{u4=5yOMK?J{%N2^~XFZ)5q z4B<4HHW#k=wxW_tmhqF>dUc9NObRQ4r2zG3@V0uzN*$x~0q^J;tEE%6!7VH*B0Tg! zmJQIy>#iXeDN)0_VY~)uA-BPsq_of(Zojiq*k^ss;%h_)|2n{rgaqE}5dOXPVcC%C z*iU{G&!3~3wZeo?snl+8zx7Z+95L?(RmGbqWzt05{jB)h*1sT5;6M=rYE6dthF4tn zd%0n1X8E#souf47Kg#8F+{?OCb(PNZm>)AKSBK;TY61J+2VhW(HC#@FWma4{m6>)9 z8=rQ>pJ85o7DBf}d6j!uP`YE;DLaNe^@gb%B;5ZtVhw=|8~N`|n#`q) z-!z_hXGQor=UH(m@fKu-y=UPcFxzC6hEpQS6kCD1Jsne0efwGUR$rYfiyl?LNCd6Ubyi zbc~Ek4qX(C2k>e#<3PIZqa!M29a4uVYhseBBXeuED^+Td2E3Y^!uN%ZP%TzfsX%&7 zitx&oS~p{Xp|m?V)prK}199#>7P<4`E;d9NjWlo}*oz%L)d&~3BFvYU#OOT%If)@T zD+LYw-~$4phl0w#S8FNs&($I#a16HOOQysZFs+%95}W2d<3T;PuIVil%9D7Uu2y`< zu`1_qpv!e0<~$yHoaG2y`l{C`a2;=_`?lZfV3-XI;rplWphB?i*YH$n$IqKJ6jPqa zXZeG?*jerpRR8|A)6ZS|Eq@T9Jcg@wlt{~t`&^XAB`E?=kDR1b&1)gdWob#% zcO3}ts(1EnSyPrrQBRf1E4G#rW-=Bk&3@f!(}Bx!XG_bO{Oj=8Y^c7MD~zNSh!cG# z`AYWtc*pV^T;IL`Wzh{Q6Vpy9@6=99ke4&-4}~m}-y-mC4$Nm{soDo6QPEOW5;+sV zjfgKMf&V*L&dPyDQeL~eJKgDsgx$!Xb(^hhHCOkcw&uGh&Tu+C@xT`1AyyE0hki?Y zj#g8suk3)|D}c^6(u*;ZIX>YNpVx7CkLT>iEIElcaGCAa5a~I@ra&+2=?eV%3RMY= zUx(M31&j9ENYI1XA}J~$LeOE_Fkg`$7SQjMg8NzqZPD__l9X|cia3?%l7b-)Z4(km z3kTo@mfsnFIrLs9xF3miDoYi~-BS%Cae5~cJm|VeW_?VIm?L7>K#ruF1o0Rn7O@N1 zLzL(WOzFlmxW#NFQz@EfjuWE-{0cG9A?#+r6=uJyYgJFify`x@IcR_BX!-E1SQh<^ z42G}yy&J~pM`z{!e#*tXNmCf3r$ME^qd^G$b{^wX%ls0V?(_YrAa=%>?e04s$1Qko zw`bX3I8S@1Uc?KI&ue^rn zM54!AzUH0qj#4HEiEWnZAephqpfEO-MVGTP`PBwY_2Ta$?TODEBUm5h$G<>scjDjF zS*<(qxagrhLLZ9h-mo+{<>Sp-Prkoe`$5^8r_JX0?s%pC`^nnV22VDvr_F(&2O_0- z=I9?6Nm1az-0}d|D<=(OF!mo>0DDuWU@vFtbgh)QN*h?E5=@vj_*ZftVRc|?rL@0S z+aqw$UVeC$Cdh*QckV-n>2QXLs&Mu=U;F*aar=Ohr^yscYdT(rcH=OOkt2CFZ>994yxQIFnZTmrYTGRfW|tM5hR4VN3Re-HLh!Yz-vV4 z8~#M^IS8r=hZEl;%vgwjuC~M?SF!qT3ckf)bDtGT|7453%cL>=F=iZmKfhOZDfbd} z{*?%?+MQ?AA@vGlu(PhSMsrt8f1v4PG9DT=?w$IB#H+2s18B#&eXP{e-y`=@{mLbT zsYYwmCq@LbXZ~!5nmkdKLv%BTFo&Vui?!#t0FOfFr#w9A$FKQzG4LuHuiFKJ*_V+s< z(6!*_H&s6l4@I;E5lH-`lauo_$_>m~Jlm%nRjHg8`DmM;9}Tg2FNuE1F5}ldOkt9a zb6xemMiq%7p#0`3X(adAmb-s#{o;m3sp#85OKo0BEwm%Voz@!br51MKV^r1K5G~gx zYRrBVb55_z8Gj6c3ot&OhO+&Tk~*07g_*myF0at-bcip(8j~GTEL7Afj=P?d(oSDc z4v{r9VaOrNA3iizU5NI=s8?`0`c0Iv`KoM0hL|_1Am+nSQb~wnR-E+9D`oH! zdiJJbK-T`h4!3!@^01zeChI>e#z-)84Jf5Wnj?2ftiJl`4ZRbjNezRlSgD`Nn8woh zMBQ6;xE!auFRps-ZND7+h0(#e_j7; zZRha9RIbPh+DfI=Ajt@nrGnz}@r089 zPY}jM`As=vS`s2VRwXm8)nps=$5~CMC1>;X*NYxo$@s z-JFFuKkT!pU+1i;(z3mF-p6yORLe?fn!EykJ#_* z{}~{VKgVu5q}tmQoacjqN~tFUV@OybPu9r7g~;WY@g)>N*t5a5psX39-s{)*=qC~w zA+isFva;E($quKon~D=1pl@(^bz)0q`yH@biV@x;J4<;A`aFm$ilO~1KUAI04ZWcS z%A@&4A6&@6ll3<_jZfZu$E|kRWM3o7hL`WgzU8~Hz`?lW4jOEDHO z<8QOYWv^6EPRJ9NS#revv3R~Uqf0C>+2>ej#zYKyIx{p&Z#JlvahNC2^xQcF-tHpg z#55*dKGmf$*#w7TP7A+gVG{6`S~kp1SoqME=ZKCfjOPjCJA zRhf=0qI6Z%Cu`t8znkQD4OTl8lPFDft~O$$FwO#^3@%fv&EhwRkbbI>D`TLc5Xf@lfB?#w}vGLg&4l5Pl) z?t3)sj}dc++Px1gVNRW_O9=?{>dSFfPnPnz-mxcvU?00ZKM^vfa)8jr7N*uT7yjtZ ztv4k#ezUh&Er{gFhgB;WuVQ_P9N`DUXLq0S~v+iUV6NDo`SY zoJc;MN?`;$epSvKS_tf)KigtQOBpFTogNbGcx<_5o&9-?mZ~&bzaJwz+%*wVA@$ll zC5STsTx&t59=!8~c776kG%6A1)71{SDE(;Wl)W`Rj`)Auh7M}nl*d;6gGXN4Lv$Ee zS6iu4oUw+p)yLbL4W(8m0jf0HndF`ntA2w8VN?s^4OR@RB;@(h1;T}^WgLPX%VQSk zpX0*?hCuIOLhVm;q&uaWn-3Or7drwXgQ}cznEz!{EIz23_6DdfFTheKz|V``eMm@( z^X)&s1LhTz$Y1NtZ)XMy&8B?w>s#eCepI>q@xq(Tr$l+r8a=F9d5BKBBka@pxMwy! z_{z!Y*~a61k5!}bhK*7@NvYU(^gU2MenYLd*0Y-$_)`5L1KQSpDR z!o%v^6$LffcsgX0?UZH|2L5*l*L9RwhRzPs?Tn)!34&dswq&8)4vfxCZmBPmTQoc7 zawm`5$thg1^nT9h)Xj-_%c}UDC%X9N41x5gd)z)AXC5aZZ(S$|VsZy)=yNd}b%Vni zy#I&_30w|7{lx>hXGLl*Vp@y&-9el;uTVH4jr8aOcT>Kq@jM({YbGA+!sZ2O(~Gp9 ztCcZaZgcry&FeBp=8|6?BP3d>pKmZ*Wh`c~;25Xq{g}S4dd-8Sux@ywc9@CBHbn7H zllM=r2Zuh{(M&g6B=#sKS8w(!+A^hBA%{V!TnF=j+bOeW-Ws}Kt~ zvZk8^3LkQ1$knR!B8@4&2LXjy=p2)3bMiH(Szka}#HDtdJ27TiywYP>mS*P{{kw}L zMLN~;n27fKr(;bXM@SeMg{I3%E`*fBcAKt`o8uw2#Cyu1H?N(y(XMShiveHTUvxsv z%Azv2VQmu~t_bd?a_R@SNEDWOaFp9Um|eTsZa%;(SlW4jik;RI6rdE8U)&RdQn-ak zyz~B7);($=fB}|!_n|WbFbr_cmQudnZgPPG6G~8n0huo52+YX`^KG-4gM2!IH%~rS zkhj(vwNXBuUjDGr84|{6-{uo)cjygnc_O*<%Ch>p-Y6>Bn8Ur2J5dB+*hL){WNyq) z(izy-OZt1LuPai|rMQ%Zc(R6xt$On5HD>L!Pw6kX*K|0Q z$=Rx*ELynKl?X|RD=!WQjAE$+IQaSW!}1W>HgrCpcv?)>1fIwEzM<>@dOhLo1FR}* z^0%4oW66h8ucM>lDzB+a<0|ja9K_vYhZ+-})Fe><$-XbKd+UMWCr@)t{Xc6e471xs zL4?TLi0?no9aVXb@Awy4C7T3oJocxR*E`aQ2=DWBe3MEX3Ta zR57AT$ZeQo(oOxS%Oz~-lJ$e@9JAyhtaQ1}3vMNk>!#;diy4~O{M!?M)+ZRCNX3J|2m6V+acMDJ)ju_8QE0>5eEM? zXtfZin_zwH|Na?#%;J@i+LNyC0uOBqCe4{2HJz*D(Pa14=UG)p$c!%RKs^rcl#RVJ zk{Mm~wGaR7gn0#vFjFMzJ>oOz8_01gTfcXtcob2GH^e+XmKa=!<9`DZ@aavyU7*#U z-rF_VZ2z+!cM0SCWGm3$-G25c)3$dQ*`$@hyU@S!t z?6AZ5btc8yX;C8jw^sc&jecik{z9dWf`6TTvGi{GtfiWsaseKtgQ8I~38>YCxBcOw zIYDS#vFg2yM$`Ps<8KKKE<-Xptr~(y4##2<0r6WE#^gY{`7*gjQE#5yss)a6kb|=2 zOjk84!@-mtGqrB$&7ZHOYLuq4bs|Sgn#*5dczN%x2UQOtBs!ldT2*vU~1bs_V^`gc1b7fBDGl zSc>Dji2?sw(o@7rg2g5B>E?grV48;3fxXYK;h}eZq-ge>SP=GJl<+m!KjqH4a&me3 z|5l^FKVU?=)>#@yI+DoUzoEwk8U=WzBDW6*v#x3+P{LIrXQp)K`0;bm6X1j;7ORUE-M(T- zIs{B_utOsoi($|-o(0K>=nPo=7#xw3-@p&6u-mPk6#=y%ip0RL7|Mm6PQw)2XCE-* zcD_LgM#PqU^jx2Cz>an4Fa>R12YQxZ11FcYI zC$st~x^HE52?pJwnD}2ci|>ChE9L4tI#4XQKTPVRv${uhqoxi-XNO6u6cuoM4|B8x zu`Ml1Ak9zS>U+O{b&rcXN=gY(ga(9aLFK-DB^U+VS6ld+eWB4>)nctaJNV{eg|kv= zqf)o|t2wJljyio~i9*2?~y8&=_Yf?4M-8p%usN?SAAk|zS*DlzRq zQ#<{rGE_JZoVa2NMYC}a z?=9c$6MCTK+>JelZvRZfG{5va8OHRz*pw4qT&6jmd9D2PX~e#mdr$|a z1U%8i%b~Kwg3@sUBbof1&L-ydK@tgY-P=IF$+y6DTHSNmgC_W`->MnjI)hz5Ns3ul zUtiF?_b;CUS-N6YH`3o#gUdZ@I-{!M>6_4v4tqxaHs8?h;QO?dq{Q^PV-t+rVL~ng zpE9*ife$Jl3+`QQ>7flponLd=KGePgJ!2IM4)rWaU9THf@CkZcEKNDL*MFpOmQw#; zMFbs&FWRrwu!E@cO2PZaLV_GHu_=(u>gQ<1ry7+@0!Xs@g}24asr+^;Rn!&jc^Dz0 zL#l@j`lqNvYuEg6aERfKdthe(n6Ia z`#O}?2`>_c6%LgC&wauEeMbRTtIf=Fr2Mx zQ+fMR0nk@kd6A5j?F;O&fR9~e0&QQx`Bcw6UBU51p#CU9@VZ%-R3U$1F3!$-u7&G!kNcDJ}igk-h1Oi7E8Aq-Qd zWGA{*vMhUpJv)c=R0#_+&&+?=Li2N2jD|MP(iS^MvTZh+By;av4PUcSE0^w-La1fZ zRN|TNSYi&k_-05;AVfhfF%ejqPLb!WeX;wCPj-LJA6asa}9S}i+oXO_1G2+ z%rkPpau)sXFOV-#@QuZd{6D!S2~`D{S#MZ|g;TtJ1?OI-%JifZ@|R|;4zn_|^Qd0B zfXn+ao%-kj*2+6=@>t2FN=Y&I?HDyteILZ0onw>a2A_9k7qNG|?8U`kz{r45O4pDT zRpAamx=Llzl*t_$W-Wz81^c!uu*yohpAVk#=s6~q;OZg=zWe5>N69Zss}pWj8LuG% z6*7Qq-V+6X0v0p7Y{d`|zkv}Frl4<{b;&0p%V7iB7~?*IJl6Hteo#ge3~YdtxAC}R zZdEigX~lqHhA63ya*5N2@0rWdrP-h{fI-S*e?`s6u+DV$SsDPVX-*H~lYK;jrnAve8Y5X2&FewJl`qI&LpMyzp6_6Oq$9!G{F%`U!;n+2Glp4>!1Dw&O7h^8_@QH(-(lZg25=LoNrS9TF-V?9jV?g4V8XCb#>ZU`^A3h$ zX3SF6>>^(32n9!5JD~3nV8^ZI-x_f6G%AwA5e}iQkTE`aJtgJ)D$=42QD)qvwOtAw zIa}+bxLE1nhF!jK4;kayy13j!X*BtD)M+10rBg2(P1f5Q3-Qma%k%V8YK{__lu zO0tj{YDq-QMPpr|RQu|RhFBJqcLs0TR{w*2q!kaZVr-1Ey7yW``pJEqV(3xFXRHm@$hT*Uu3_BOa^|R82&Fmb2ME zSp+cWx_38jBCVH-M&k%Su9n!mt)~Y40_80^Xbxi=KIeL?&YN~$bpP455UlJ=?Ykl> z$M;!YNn0C{U-?1j{5?8;;Ygqu*(tsXP86Q?cZ*%jblKRw;;KWt1zE_&PZZ<(|@_e{7Qi`kjD$c zN0nGgXS=}SK3x*MjriPyh%9LhM;v#M45 zt(HD2%nS`Rrk<54t?s$?o=s$^Mzx9e0jEWT2=-_5iwZ?f?5PsUXp-QRAA+9vU&hs? z?p>^NIj-oIDB7*B5Hi4;3zf8Auy;V5k9WjM%~m?{rAs#shzbQ*cIcvQ`leQDM|* z`SRl~#L2cW?3R$)mPSrrZ0G}!V4;SW?)I}EqgS=6Nd0z$YUxk4qCSAw2j7|+rTx~& z+$*BPQ=t=0xxMR%M3Z*x#(*NmI*H3>6k4W?D=G`TXHLoN7$=&GV;UeGy)?#Wgmn5` zDf9`2;5Gvx04o0k@4s4={E3MUXLK~wV8Q%miMN3JWptiEi_s3SH4Oyq1Yr!PD><~* zdhWk56v3ED@NoeFD&oU)Z&s?dV(u^U=6P3BKID=wL=wZwnjs5PWM-b49kX-0svu4Zu*bnQ9WI_t#_bSPgfiH-tv0= zpScac9OoJ>5c27ib3Z#?@gGkNf4Wr(HHhMdB8D5#uzjAQ!6{wt`fWFpZtMh-_#I#N zb*CrdYfJ+FlKyNYPsxI9K~mj`!>a00@OHmk(KSoO-)(4sf8k!q>a#B(Q}~;TGyVDB zQE}~1LqJ#|c((A@yaB4q31{X3n|ZEJMXwR8kpW?THu=}Q#}p!-(OmWGajk0ZICe$` zFGoAR5w5l=`}qrOP|o~2;dIxc^`NP{LidxuSrRUvJRKIv4%DM%jlRh0y)|_@Q@R)l z&rnod&k~B%c%hDdxun^ExVdHJHR{5M?V?b22!K_uiTS?+T$<^WwQGZVElo&SHO`7+ z>j#cG;GQa+JCMFDPw7WJk+>*~E`&1JqbwP#Ic)6Rw#EH<9rFjuiSE73swV@n1j10$ zH{3?T&Vt!k{}+m!U9TZyCTWgNA{W>mkf)jk4;RK)Hhh|EW4of8JvpVxX5A^ReaYEd zMcP#*1F2@~`@lEkRtI|0+8V@FhF+iK9b0`n1EJYN3d4JtH80fA^I`^7-G~C4IONVk z#s{uaV_zAi={uv*8j9m-Qpa@wE1|x0ka1j8NjScln`<;=U9K{~7&HEGCN0fCZm(;2 z@*i3N^~B0pb?_>+r9w<&dN7XdDt-}ak{CaVri&7@QG7rIxzRGoLWq0Rn@?`_!U2n` zXTPSn4uG8f(txa{9eP-1u{r~Cm4ETe%f}a<*NaH9@8?r`rjK?`cNz&;sPoB)q?F4} zlxBcex9G1D(TjqIexpMw54UI3Q|_MPGkk6cRvw)*z9JbaVJ>A{-M7M}3DKGfwR$>( zJQ{*)@;APvE;ZjIzuab;6M%Pn=FC%rEkOzc`?S{praezD`SHW`+EPa=PyyCpB(BXbxMsAnu!Bd2PTIeuLbKsVjTHwqn@~rAY<3!s z1E0iUQ1tfLM&)!)X{>WLg6ZG_(BV6b>Jmq`tS*^YU2m30Mqe&2jWC|%_Pu4fA#$Fj z9aC&@ZhW#()FQFgq8o$ZE9?HONBH};hIi@0Wh?B7&2x)}vN(FNR%3c_fxJ_2Tdh)1 zeO2;;BgmDqje+#7mVbWOxouqd^*cTf3}IXea$Rfv5(x%Zd@c?3m=&oo1U%4gEFq=o zpsML!?9^VLe5+o54D(QOpZz=odjQQqj%=jHZjtkX&(2CiX|bdj zAf(_f#`f#uZQ~=&DQM5A<2iCibh6RVbbpUes(bvw#P1tu1Wp|c`Z2##Dr!VlP|C>7 z);k)~o59^ophRH*7f0lt;8w@hCo}{{L?UEsPgJ%c{1tpD0C%VDiQVk(e1&bV9}lOa zi2Tn$<6i>HMx-n89t-GAd6K^V_eERIF0OL-!ZdG%vx2r*laVNr{>_%ngsNsx6c`FS zWF$^~m~V}*LTk$&>)zoYtrC|v52_qbFpas}Lxx&J`Yl`mn_W#@6Dq@05ggDc1mV?s zzu@s3GGj9;`#-2K<_H{T(EPL|8ITQ7El;T1O>NL%`O@*_b(vZ+g=D~ch2C+BXs`O< zVvZZQn5-NlhronYi4Ht(oy+Cu#rjU&=At||{W|#qNygo|vLXCHEboW?O=^eC-Jh%` zW9B8(ggb|-d(gYIqz0dqw{OKRm;+32fn@VN9)V)c${k%Zwm7cRqgD%0<^5-3?}a06 zB>Uk?g`3#~u(&3G;P1aB7JrlIdNy9Z}F>_o&6j7d0^>1%;5YUX!sFHaiei;Rc_;}JB z@7;>WP<4_+uocw9vN9Xv&|VnXT~qkC^|y5mJG%C3gAdGiZTsw67qG{RrbQ9jcPpd$ zN=-)G%E*rVA_te3K|>3rd?8_Lma;LefnLs77uZDk?-g^IbxDI`T*FJ5b&*V^zn(7I zb*YRl_++Ke-0dRCFc-0V1qS4nGv6Kdz|r+X2K3cUP<1+E68?|wZYLmo0FX7N0zkAzSJ~ke^38({!aWerb4LP(PcmPH4Ov``%>30a3QB(|EN*2{`uu-m6%^ zr~D?q8mb}o(Wp+-_6Y_K>hVIvYF7LpV==9~<{T=s2=n_9U}_c97EBOunYrDcG@O>+ zZOA&~*rlL#*V zvUL8O&4R@v{oeWcUx|`*HQgAmKi8)z-)M=PFTo0eDI+PL`^FHs99PIdC>2X&jOe^r z-mvzyP9v|VG!lzECQ-$w?(WM~s)?p>89$It5WG4m*(>E;IkJIZSG$6xiG<_s`ySLg zj^hvl9NcGSEU6Zsj7IC5a>2JR$uTuQEQ=eMyhUJ9mb_h!i|l9hUlbAtDx~OzuY$1%z808vSl#3GQrF zmh{+)&ZDE01)gdnsN1i%#K*FEVoK&Lzwn2~uTqFM880X>tE00YyRfRda@z8aRq)!a$Pnrf~yu^&O4A{ zD-MlD#5*b(S>pB1!jV0>nPNcfEt|*zW6Rl|=iV8RNvqyb64Bng;){t{BRJFJ>6$ir z$33`nSW6vS^dgnoUN+uUPA}ABRu&Ww&xL!vcyWTp%%MnFb!)zKC7Foz#!Ph)8V^2gE+qZXrM`7&UD&I^6j@csgi6$5F&T{rddl zy}h&ox-+18Qp_8a-=AF1idy?HoW##Lrq&SdgG{&#lO+!sB?4C@R2EHWTf1;VJ{$;oMX`=S)gEV)oKtR>$ z@Udh989O$EN`#Mrt!Qt2a6CCcYrXU-mr;R6<0(|Aj~D0U{&gYo!O0ws$kQNeykf!4 zoYf*_>d2B+kq$s?8Q@i#`~z5p5-OQg`lWiSzYrjkZl6zksR|z~qv@BltBs5x|&A@yt8_}VfmfsXGS){t3mDbu^ zf`Qp(8J}@S@(v;tv6x4EVABqc#U$Yi&g%x8Hw3pa2MZ$Xm;8xTK$Xpbk0(L7#WZ<4@Uo5TT%PV4%{p6{3A-ZpUVU;v3i>_4^R$- zFjRHeSpu)N?`FmDalorr?NCgu9r~8n-R9?`5)ol;=X;J3?`aV+@y}^o{eJ?wH|t2( zpxg@bak@5K`$1w-0}48=pP6KabhzUsgK94fa-Vhl2NF+w6}o~pkxh0BdjB&u7Sj@s ziM3|;EI=mnBv(E{J7e^XrKm402?{Lzk`~c*@CD04qa_dpW8~b$+;jS*;=&NI=w*Nb z7^{0nhY6XX(sMVURB4eq60F73OADMpphjQ+H628^+O5`f3R z^&@6$HNj5M?hhPXZesFFb|jy-jFhU}%;_|nlQqQRF9%h3-0en zul+PQxwZeZjL~6Ldx!p@)9Hjw=KZ>Ae8m|L+|pXk4SG5-Ai$}onHYZQx*e8Y&ilgX&^!1rjt9hQ&gQvRnG1--N^X$0*6tdP@dm|(N++-h&V4kv z9LWd;>tZ)NJ7-34eHE~%!V?Q6OSd%k=)Qfm^;lvjv)_eIwWkFldsu3JJ4mki&q?&( znQv3TzmrlH1Jq$J0QhI<+pA?24lOV(sk*L@0Hf-xo0OK-Sig0HAAJLXd!q+g16CY= z?Hsnc+RoM=TxG}?cJxR?M~753-)#i8Hs&iwn!rUEf}cP>-s}g_V+#x9=I1G2g}@Uj zUo}rC|B-I|Mu0IYzkoS#ko(WA)89m}9WaLcXL4&3f(5B}YuJ&bvz6&WXyyMZ&6Zsz zQS5cq0uaJ=az@c|vMmnhzU%Zc1|8Q{GivHTZ5LgWg4Xook+Tq5%6_Zde^#r8v26EqGWijQ=|@fsCnlkr}#t z#G{WPGbR-&%6C`npKi)&SI*2g$fu`)3J;Evo~~(f4q}iNaZ1ObR-7b85LO4+UWo@1UA{Jx0d)g?e?74X0G|piHT6>DiaSQRAtm<~2tkD6 zniRkZj(uRJhFVS3cm4`@B0cX@P}vSJFg8ED`yL@g#OukuORvza)F|5Va|0-i2Hlku zd-p#m$Vm^ItX4i}P_(VPc^o>o_jSQTdEBRXN>nT2(k)XR#evpKp!TxA|0+E{kCTMt zVgO%M{f_4E$Z;hnvCMLf?Yz|aAB>%4SeD(^u3rHuNona6q)|Y+yGy#erMp$SL%KVq zr5;qeyQRC4^da|Ty}s-C_BxKe_VfA$7Pe#UNfiAs-Km@!Yj}5mgS`g z$Ug5&lI;rBuHAZegO^prL*d%p(~+PZ9s#1IG4HCXrVytiRy5sa(Vc! zs1YBZT>nbfJG&djayJQf?*a5}K1Vx|k^Ag2Qucy^ThW{mHRpdH=EvfK)7-}-e!Fe| z(H&zv$NXn@8xjLr(Y*%wl-tS5Z8EFbnmh^$pEB&C$=%l~jY4^9RcSM?%0l|uwvUyH z6>}J3ScD-_8TAp!iL}n;io^eh5figs*?x`)C!scmV5l>?!Lf=_E zX#L=fthP(i!d9l@T=gYR`^5WZ0FE@?%vYotbtREjYkshj$-v2*S;%-{JhyU=>oH6u zAbd!umY0-qqd0N7I8`q|@-y%GoC=1~`^ulbRHGWRX}RBpAj7FFaWs)cf*CicMci`p z8b6y|7Q^ZUzAXp7od&f8tQ!H#c)|m~&Qt-X)30AgC6|2F*P#)|591jgpmgf;$0ne) z)0j4H+7BS`dyLtk98M0QVy!UGC%~%MY7kQp{;xB}oeQRB+5-v~bM&80G&#mUDB1@^ zx{ZASqO&*lr5dj|c6SKEcLCNoSzHqxN4Dg1#B>L4(@YSs)uAz|d7SgX=vfACTL47*&encIG+h42aG^#rbc1*L9q{^F z=MgAqr)pr}Z;meyP@1R+Suv5T?;Bjg+EIQulGe zo#VoS@_Fs#$2zZoj2JqJfK0FY_NVV=Ol{Rj!c)qV&wy2A{b>+`?%%eI0nC{C;enU_ z&ienksssL6)h{1c^%!G}KR|l&Ds z;JxU2P#U&SO5KTfQK1DKnPdjBWC1928;{Z$iYn=|Nd#BhA*v;v&wVp9%=(IuHx9J6 z-Q{%_>m=dLqz}JhylZG&$`0P|+uijw%Hg%VE^7YQVM`?BI+y9ja59d#k4lfHvWzBI zqV^dwnXk)PWNrewrBsH;X$@?bmF`<(m`H-z4&h8qO8sEjWHB&|sk89v{<@mtJ=C+Y z|E}m}SJLDCe`Kc&nvH~69X>{Bw_Vf~Lm@?%X7;r}<|L`@Z2g%y02Hy1Bni+zl`*Sl2E zHK0})^kjifuA2c^KpgkRnseg1hf=EElVJn9ddp9<=c!` z8zaSQ}_<)--US~J+2s&?1HqvR}Aapiutu7GxI-3A&-VHjIe(y zBIOTMr!L1)jFSyQI4`&zqE5?o+V&c3@_9L3jRYimU1@Bl(pbB zk~QFmROHH#vS)q&(YtF%8@P-b?QiA#-d3IfsIcp7Y8Ml#FWy$|kRwgECI%dvzjD7P zRsPFGyHbGItF>JK%e;vDA0JYf@O=CdDvQQ+>n%IW0p&d|Pk-R{cMKs=p- zz!W-%8dcfTkGY%o!L}4DK#i1y#T$rA06rGcfd-O%{oIGrqcRVlwdH@QNU~^6aWp9V zGEOwnIl#lAQj5l?ibFa_*&y`s2Pnn+n57|rst+y284!xiyBB^oji8wR8%a)Am_ZL? zEM`3t=Vc~1yjcT$LjmX`8?({zenKg6grLM{zmpAEAU*eimX&pg1^q24#Mc%N46lV=4eb1Fd9 z6vR{wqleNJEzN#aEWrLn0T(e$DslXdIZ&?o)!zm62@W|hyxxR0)5zic$Lj-*LF|9F z+kQd(G&g*TDeby zs%^fnRWgD5D~;EbJygrw2$Boq9`u9$joHCh&_EFY&pffMUQ4=BcNn3`h&sj)yYzC$7&*8!Viw3edTVxSeexm zbZn->vB7{85+buwAemFBO8$De;Kwv7D|REZAt?onCRs0*-I1_Vz>!R=eoukTDMaxv zk&nJcT_$asA&>F~_ECiolN1xy`>7%WD5E?81t1txqMH1Pafi>%2n%t$!Mfe@Z%k4$Lqx1z+#N^ev2ayq5=yHt{>4eX-!f=I$ z`y2Gbmfzpb3AZG7bxNkr0!&6mg^VEcr2Gm@lR~Mg%5O!<44m%~p)V*t$-N-|`?DJS z^(4_6&}NA{iR&xD-`HRVp2JktAcx#?+zc7uQSu%7(K=1D06cQR@6fnnUNG-Y&}?%@V@%}=Jz8v%sy6D$6R+TzC^#!3mR{xN5hY{lVQccT zIF=Wa+#1wR2JDWQ+T63LRPRT7XQd8`tcBGT4C}7nZE9CVBoLYrqo+VKGTs#5vvaZk z89TDgXLU~dt0)(Zr?p~omTyfl=~DMuVN?MKv86-WJXxhw46EBM5f9qJ{ZQnQvmSgo z-iZmj15l~3L2;nNVB*m0D%}y1^Z*Fa0nxA|j9nwuvw;phKU;k;O9+@u3>9{*&)1`K zQa+3bLu^|VFL|Fb?g2?=Zf;kHUpxH!L{D_F#a5>)z2x`I!!42nAlzb$- z`o%J`+4phY-MTHK^5 zWhN~+wWnw}C*YcM1?78w(_*^M;rLu(Vz0^h1iZ#+=f60z0_$sV9JETcj2!2c(x?9I z5lI~D=_22}!sPxpWlG{w6Qr%DA)DJT<4HezeUnCk5NWq^(M#dCwU6DlXsnSS6LD7R zu4-H`)3NmsOGI3kUU(-4ex&;QQ3GPafh*_Z&+6Eue;ITC$CEDo`zV|KZ}AOZ@J5EC__RRUb&A;+?@bZj#ZqiDJ1jQN z$|;3_eO2)YyIuLPyP|l&KUN45+(M7Id+6lmR{&WN$$nSTdd@8Hu(j&+)1|ThRvrn9 z28D61b(v9Z-t*FL{k{6{;YGJqe!pl5*8!iOno^SXSv)t_^-~|eD4RMKf#|kwm8R1A z_J%DPFHlT;lY(oo*8&#asT`!UcS^qe;jtUMFh}=W-5I)bMQ@jldbmJKgcp~!{|JQt zKfNK=_ZGnvC(gE01yxA&hb#C3$zBjWUazpdggyo8e)YTnis8jp^f79_t3h8DoLbQnmMW35zo%K88^yEgs*3=s&lrUawyyUY<_l! z&s`_>gj^qXJ=okCa7G{akg)?!(z}iFM%I9EKP1k$vyF%eixU_yTKch8L(Bqm@%FCN zb7_rQRyNU^Gq)6=cBg*s51D2WgyOTG`Bty}B5{KI-<8Nc4h6ye=d7c@Z8X~hIS%+l z3_Cu~klOTN=3*wZJ7CMb7}Kv;8X<$dnGgHOMqfa0EkZxoSdPSa^?jQ7xGTTy;%5u= z1}Bxvmx`F@Ynt9IRHtp?waKuME9?P}U%hyAdEHX&^jgZN15s~=m2X9* z4%YT<7N>k1AHxQ0rfXfVq@)wxh=q`JtXQOjdN}XZM%n)C(*l#?hjr`(WQeLJ_*hZS zz^3d%FO|Xp>q7cQoHNS=lN_bm^>^KDRf_^R5S0 z*&!MV}@21MQen!j$Opm;BNG`w*XM5ditWpVuIT)wg~Ftsz;9QNIrGOg0p;!I*#aR!S|rkKR_|u#P+@0A!zoNx}uO*$4lwRj<&4J{)qVt7+8KMpp#W!6bUc z#y6)pc!H_&>Fx6r(O{&q91Ey&C8)rOx7TeC>|^*vu+LZ7>b1^1r3u_eTsR0vnzu-D zIzNZU?wXocpH&`z8#p^RqG(!FgY(>!B`vaKGLXB4or-S79}R}smg6dQJMH5F5BDP5 zToD7I9XE8E{dNCp_Kgq=DKomx4PkB zRn~(#kimIxmfWR%ZL@SVgzk*d^Rg$%tM$GhfA~&<7Dj#?xQ8;*W=gE856Hk>=OkX{cV(^98}wUa8+YM`nNQWDe{~&(9^ZlSJO6bXU&1`|ghx@7OJSJFOgEu$XCOuf2e#0>{Z^1D}-ByBn1) zA`e+8QEU}imW|Y@ole__Yd1S7%-Ll7so}iyXIQHG#@EjQiS-q(s5-D^S!(!_?(OOK z(Z7~Fr}sF&J23BaOPue@%52LQ#H!%MpnH9{l3IwqAm9ppHDk5&g)S9Y=??d>M5Kzm zuvNsfE-&$bi^sMk3z^p_a=W(WC z<$Q#L&x|QQ$`L5X%7*T=`#t3O*EYgVMZc|9n^l=6fqMR)SE&zpqac24|3-kL0%RAM zaDCZz4rO$V9nIpk`TnerO5>OG^S^sD#3n@iyLWp(>o|xFpgj1fKA=A-j7scJ)arTw zI#by$r`&jz;xD#0u9uDz$LpWY3q2TOTh1m?#jxk%h?nZj=q{}GWq@XD$XsN+(dbwm z_aOr3B|j9*hG@4)dY>1I*l4vxkfA37Z@<-a&P(F?miR|}qO#MQZ-_*3{nO>@#UXZ? zt|-}go=`ZN4Q-+x=NA2sr`RJ&I`uegNdSQcWKWe6X4DA3%^K0iry&Ogw}e;4A2>g}XgH8O^hB$9 zw*Ltyyx6|S>^9ww)yL~c8lPY4+sBL;EdX4cvWWh8;__&5e2%5@LcGBwcwkGeD06-I zPJFa;^={{&FK=)0uW@7&oiQVgxbMjd#_*TfB3`Z-txMz=KTD3^@6G3C0w+`??@fb= z4?7#IF{kCmU);^4&$CA?7MCUx$dr@=e^HxQ3XSsmoQCAG%q3SzN&)j3vsff<2aNSb z2D?Lk|INmj2n{vm>&#sZqp!6;2uW<*t2AB~BPNu+3qI&W(+A%3WqXmGja66%*88Nj zP8mtlSrx4D$*rYJE&T?PHsSatX0Y1w+>V#N;JML|WUVGJsWgxYIDUgC@du$}d9Pna zjg}IFyEPc9w#;o5XQn5;e#4b27gg`!ZM`2GluGa%rPAX$iSp_>!xztKh~S^+D9-J+ zx(VIKJ;Tmt+hT7uM2JkY444DfWb7x~CB6tAe-*&gyAl7)W2`t4D9_}Q*w|n8yXzvU z;hGKSCY-XE4Yc|^y!{3mX8)X49X0KZdWDzP_+&W<$|&R2`Bixb+#?yM0Ke0ZoUVwi za+;e(G{;rS6@TWOnaych$DR$Ebt~c@*U0OR59>3kO875^Qdt7!iTOnBmmypBU2v*} zj&fGBr7GF>2niWrh??=%(SNDcSE~JlhXYtFfw@)SA~JO2pz+!1UOlQPM?~K*RJcIG z<2XQ87GL!8ZqZLmeQ=#^b>#{`>G9%(_ggR>Q4v&MRJq9{=CLLqHjnmsh%8G)aQe~N zS0dF?iQ7$SaRx*#Bx`mEU-i}pR6@Oqct+>~$|u7uq*-}Vo&;8bl5S?lgyKCP6wMou z*iuri@i_)ze5#n4*KcGkqEI-}v(T;RT~Y0Ff%vp;8#K>tafe^5il6c^{4FpE%LANh zFjRdMpnv25(+DFnkv04$1cWK%LPFn&GylQyN;(v5J64M2gnoJRo!-<6rg1#h*2}HT zC(T?#X{C%rpSzds*{N4yyAl_ozv>FJ(Hg%p{UbRtV6t3Sz(IX5-cKWA+;Ow32P@w& z@v;#A6smaANyf|PmepcSVeR0KG&ecL6J65%mO|e_FoE00jGxR(G)E*@DW4Iou-sIa z=kF*b)qE2VoHJ~ah=Jtt2tol&-4^d97N}%$Xf6CV0%Q#Vt3_*XNa}LZ>@?ts1Ib3fdN)OQ zs{o@&Syo+nolw##s*cj^bpsH#x~S7lpEEN(9&uB%_hztcI+%p;;wdRhr{Xo}oXb z*_u4t3KQOXNV_})dAZiuTwP)FiOZ%fwMD@%_0}GTcz<+uT7AA*?5{v~9lmYuN%Lyi zp?xorxltUZtV z;E6@2ENJJ8HB!PJHAK(8YAPYuJY67`I`^HCARL0+Bcr?zrv1O+kE_A#O{pYos)Q{b z$8hdFEK)UXKu|4GG7r5W>iE@S8fTb+*ulUr5ik0hC&z-FPV9cZ2LA6qo0=L%1T{!R z5Pd{--FwfweUIx!jo&8xifB)08p)bC1jQO%J!YG13{$>Bykl$FRqga62HcyGQ7|Mb zb|YJI0EAunl=~89&&_IEi+p?)3jqO6B8-e?luGTW`2C~I=FYS+Z_^-e4^iV@` zBVfs5Bi)y`tPrQa0OODNK0Fk;gS-w8uu?QK!``LD!_vn=E8YVmG!ebt3$99cf+!0h z01AnrLI)o@C$jecU#f ze$!3n)Gjmi?ibsJo6F@Y-L;gsX~YD?F0NY)(-GgXf+ z@~LL{E)GTlC=+|l5&Xp%XR`v*-re8;$Wr|Og`pmWK<8UBs^)d-j=7Rw)E#7pSKyva zy(|9$kerExv7iG_$UKEUP;xAMy@;)J8z~OABcHb-*1=*f4$Hv*QKy)r{A2B@F$_Mw zrpiB^+J}!*okIIu$TTO|bc(06@uGEiWjEA1{PwcgJt4&M9g8k64Q8tH<6zehv>agi z*IAt;=zDDc8;bx`!FJ@Xs;-$uc4NR-avwC0Xk4MPJEh};KUYa$l-Cf~jY%A|50%`d)`3+_6J$QaU-b9a?VsIuxWBfeh4_?c8!(}qUi za|zyiy^ppEUeu;#CJQ=-wlf&xFF^`<~Oe=$0Agb$(Ca!jU~FjiH{k8rohlDdO^c6 z?+s!D#J(PFY+s;~&i0rqti<7zz8c2~{ymK(ilpkCE+^h9Q4g_n=@`vc$ujJp6_)`l zNg?p}D<|mZ){e1{FANKRZmz}6-iO$d? zko#7vo9Imr7jRVZViXMA=YH>hr4R+rshCj;fZ}L!~q@h zZl=e`dV7||C0VFwyGW`%p5d809x$~NN|Y>bRm>T3`Zkh4t?KOD{d#0#lAjkz7R)<; zihTCo^+{g!978yIsCU~3_NOCZP2`yU{(}sO^f=!UZJDn6Zf{IM_+2@}_(Gjs_A<3o ze(rJdt!~4G0a*~kA+goQ!c$w2&EXx4y>8zJ7aBJjh}t5z@wRsWx!f84DUK5WFJ*o+ zJ=%Imhyp6DmwQ5#+JGoCafQ3I?$W$jw(Xv?6>5B-{j1^7%K$fB5G6`KRV(t z{1o)+hN-&Fen$uQhp#E{|8986V-xPp?&+5Y#SF$}v)USupCwuqL`G1#@2RY&Qm{!; z`A3YzenPn{eG*kXSUortDp8@WrE@OMKsnO@?r^n0Hv;L-k*{vPJW-aQZY#mmj?t%A zU5i((1lo?nYoW{3qIqwP3%}NB3o6uB?A4SS=%`rOkfQ@gPJW4&2=M*4-a#{uBCh4i zECKGt_kG;wwE*mD2oXHgHtK-XO2^Ie9{y~N@@%k`1(__W%P=PmfWHh2ZEfI;jEJjx zVfGer5|Oy#NM6p0j#-BYKGz4NpUDI|0g#8Gi*pzQvgPybYH0d`9vkynm0--&(^>68 zIxoc^6Y4Tw;qCR!pUq|%x26@N|nhjplZDl0Y1&O(feUDdR7Gj6f3*TB=`Teew zZ-pHJ$%Zi{zA8q1O8A{cO|1~QKKZ_P_-1yro}ZmVukGS}0K*h=$u8hw=mJUeiF!+& zZHS)ya7KSXW0r|ga33Btelky~zlzkf*=riE{e8Uzl@KeL)Ic5N=C&;@(@U8<5oq9> z3a;TJ^a8FnZv;YlI+cnzIS=PCsMm3CKAGe4{I~zxV7%9@GS3R5tMb3GpUy9}vI;}2 zj&pxd4`Ak(4o%d{Y$p;=Q@1d6CHyeO1b*e^RL1 zc$4ta#{EpobQi#2vUPC9SFngtymZE|M*N(PZ7LX9zCDq0TWcuj;oFd?r4B1s#Rb0M zHQ5Pj1w?bkT`8}l3j!I^f2&d)|wDSeMP9OqHW7(9D+VL!}P&7@B{Q~6FrR2 z>s>i};I9$$XUY-?`gE};n7W$lqMID7p)WLPRPkv%bjvav`q5)LIZ-0@%jqW!Y~k3$HH`@Kab&dCGdKYvsH+)gf`*9P1ONrz>9d3~GhVnsE!nng3@p1oIX7^Um zP9?_(eUk13cu zp*z|{-4qLPK_@Xhkq&fTII4A%B}o9qpf-ZC#+6`afQ^`yW>IU{NB7@XMFI2qUB_>P zAwS%UK^E;RR~p`;!Fb$fqaeT6_so@H`ARBNLnf8`L!M?%SBQvSuQx>{Ppk6*HNb~} zTc3@@Fc(cU%|m#QBj$bT%RF(&nm>9?#+DQ%PdaLqLDd~uFopu^#i}CrS8Be+O{izw zW~QOeUs9S*3i3qmYbzupVNlLYb|^AMk9w}ZN7SiE-lGVky(`VJ{wIz zQ=*fK7md85bQ{{7*dg-m=2=Z+2-0T(TcXR~L?sYQAzERdryp2*JGCOU>7of3!4Gwa z&$7NXHJud&-=Nn&flo^H(s5XUOrRXgh)RTG3=wK|KIbM1A!)wW@SwLwf)91+$9;dE z{OWk*f{5_lz%$r*kh=bs0hnH=c$bGvbL;z8n^J;)iSmld5rZ`vwdjqx)6`8ASVz+f zFVXkY3x`S(81yrcGuFP08+I`AcnmQa?z!(L==G92GViRvp9XJih#E8(bh( z4ng@%sYfF>;KyAl_g))OSbF^Wnd()m{h1vsvzp6!4)as|0*&fXd$(+j*$~)g=|g=p zK`m~IjqfgyNNN?oe&bH6dXMy~mmGF5a@6)>3j1t|q0>J};q_OxXkFA@SMOk$1hLdZgkI3VhvLOcv zYnGQdL?*Jbg^9nW&`A7}PN?m(iwBTHBZ~i<7uo9!!J(e!i_JXA!;p9KZB$&o>`>Co z;_h604{-RD32xAr@(fkW;f|z>0>6bS?gA#ISdNCny3!Gh7}vd&ZPG7kqaVcY)@x~^ z@*~TVM&o<@6w7CS6;t%21O~(LN>}hj0m>eT5^$!MUD0w`i|>7S@_xF%;yeiFSBYt-7IKL?BQu)O5cN&DdA!moPO$k247eqL0*x-; zq5{C~KCH1-9Ia6oX1r)Ho)%)tUg5d!tX zw2u%WHne|yX?p2)`Y`BMN0gj(;!l<}f;IbIsgC03A1F<{52-gk29(w!u{LEyLalUP zwnjet@_~<6UqW6tn@L0u-*K=(_1PPC_)hJEQR|TYW9dQlSYlUKXOVaO<zR|>qa7iTyw27Co#Z5{2#oAYDqES?Utgje8M;@2m=K9B;S?MtzOh&mx~nde53Bh$D`IKl|x#r8DR`ph!? zFMOKHgnhkkC1Wm}#r80iHwoEG1I?|EZZ_5b62~SenI`adv>U$qxVP`%ZeFna&v69^ z6gC*~8E*HPfc%sWt&S<_XBO9z=Yh=qY+bYL$^5Awpy9&!;{IfFEYp(s zUJ*#$hSV}1z^Si;%Tc_s|2cuU;o<-or+>)?M0~m`VpPy|iU5!F(R((}JyPU%@PmNY z_7lI&-}k=;248nUDgVs^%%x#S&6yNMd=kc%4yW)^tx-3jqSPj^jg6(vqoPA5z?^$6 zEiR@h(Ee}-@mt(}@XpF~%5=y)jynC6YOVv%qHy=7?H zVM}}k`C$dlaGP<2%KqlIOMUFl;4~Cl~*?3x?)5;cO%2imjvy9|q?Wieg9nv$smCN*X-eGg8@sK~e z+e>8=IXk??pfcRB<{!C)M9b1F``&$&y|b{B?_oab6Mx@)-X)|ddh0WgPt4)fA>S=T zk=8lavSFoNS=V7lav5KB&%yO0xvu|hra6`bM49xb%$8FdWdQN+L3;J|qJR##nymB( zWySmyevmzg{VJmFsQNM3sB z%Zc8RLGM0TgFk=8>E$BQ!UP~B>T|`Ip9CbsT<&M?hu%CztVk*Khz&Olshma%osterurl?v+mCu5!{yt!LEj0&C zDi-jsZ5P^=bIMWK_I~!JCMFug@TIW5PWZezxt_M*+fAY|98vt07C%f*ft z;Agdq1td!4lnHC_x`e*bL*uIQJ*ch+NuP+OI)?0J>>cCMpRyk{k_LMhEHhElt-Bel z)8Ksh_Vuoe{3Y>{DB4)&jRe27%RK#(eWC?D%F$#mHIKh~;QY*(NgMRrBndT}eGBpa z{D21uowF4Ap=}za**JxMo9#`%Xk8RL6+`S^zJAexWJgDI!n3L-Iww0i(IIuKNPF#; zVYb<;BbTHYl1T-NbjP2V4dnwN4k`z35rP^q@aZoq^JrRQJu)e|Ls){y9M=#tvuSzp zrgo|@i7tnBkGsVDJzRzx^%*BZf(N}nyhd{(+prc{SN--=j*h(D+guuTaeAAL2k*=D z(HE6F=l6wj9Bu1bvi-&BxM zX8HM~%;(#oMb^5l>dwtKjOOpWT!7ikhL6rnCQp_k@EQExve2t20XaQdmVI-cN>g8> zuJfjCCWjL7r39p=%FFvlT1p(Ds752JOUq{lJsaZe*=Nl)f8DdkHC@Fy$yX0sMR|Aa zzNE!^GBz>18*{fTO+VACr57lRRo>!oS>Q0E3n}6(l&zVQb>+B;{QCSV{rMHNsZ#xv zUGB|LZGyKPfuc`3g`)W5TBD^*#&S{Xp~1BAoKS`fD-f70x;d?#tJ%-I(Gp=t==xh{fEzKF->1+J`MJZU*h&t_HO{_LiBAkC zvy}o&@}x<;@2s{qtNv-?cGf~ueTA2c#+k9r5WSTdpONyqeM8suX`jS#4~@PQU&IsZ zLzn49EN+)taZU10x3+>DwiiB z7m=Rph~gEWeqExuoqXSRGKj@ybxTNan)JdpCb%fWEtXZs&W4r$_ZKsc3}%;=`IPGn zzA6aa+0SQk*la!&HOJZAa*UWrW`1d>2(ToCpG$(0aG}!YuT8M-KGtcPPN_{Ou6B+j zU+#%=o#p45QW|5}`6%N-N7^5%$G+jeA!UhO#K>v-uoI_%qRIEGt*WMwXhS>Y_bt9i z-B3dbVQp+xD}piaCnu^9Z&TTYOvg`m8NE2&rp>Gtz8BD!(?Z$DJ{`gf2iLEEHL-H| z(5zMTNxzaEe@7v7A@jIbxaSomOs%xco81s^5Q`+pvF2|g(=*sl1awRB5M3=q%Ruf~ zwzXyE16<%|}BF)GAK(XuLz%uu3=Q}-Cw`Oi*b;-hv{kM~dRNOf43*To6 zDgh|&-$kp?-#C4w{D|cj!DUA^9}dL}^tbmvKD+3{vh`fImGiV2*Y)b7oX(2ZCN?AH z)@nR93W-YPh-B+5YuDp~<3?kDyC;}5wP=d2{R&<=sUFT~kXcuY6Ne%$EBW_3y~U5C zKAc@N1k7WZ7pZ~vuQ1^2#-DH@;Qj_ zb}(8Xx2INA(?zz#k#U(!>%7hLR7F-7S2n}gNXKS?gOn&ZZ#p%tmLJ@21^=r zr8)=J^kQlzA!kIk<}3hf9AjPaq#bG;K|L+M=h?8BMV1k9=HKY*HEAw_tmie>#F3i8 zM<{HDd@1lY@x<0chSGFjh9fY3Kc#5+*s;}exu1r+F3-lecUbHhEV83wPb<_7u9$st0r8ffC`vbB_~f4`~YO zT~+tnrqqj{W>`MIV=GEw>*f*uRUKdlJQpH6(vpBvntzeg8{&e6r=O9Po=?7vCWK12m?zI`G^R~B|)5DQAdLHR!lgIa&y#4mhG1kTWqPc$(@gODM`(wh}lCB2| znxI~F=I`G%Dy)glt&UYkUg)I}J9$41{>0+#uw$*^CnJ*@lR8QmEBa=%m2^u|v2D}4 zPksL4&TYAu(JV8W+p=|PQ}@b>Y_aO=_ZO4alf{^6-jQdAJ=FS^%q4E5942F(+ga?k z5A(h+!M)-s2gd0NQLjxfWAEjEzYAHRwn8)iR4@C+F~hSZdERHybi@a}?(>*=uD;YJ z&vMLY#1kjFSZATbbrQrbugF^Xi)$9hy6X6kn2*FQ8R|Lmwx+RFm4L*$<0O{WX=ImG zfM!4me_Yo_n`^GUH^u6zSF1QY;6>uXd{ZRI^wV@27Gl5kqr!TM;{vrsi*1@&)O68& z>o1C7{MQr%lAPZCntY#k@k|URd|VSq^3&Ld8KAxFG*8x8Mn-(=K&kSKqdZ(ZAMPSXkERVPdyjB%er2 z-WisQBw^M%2`5-UKh#>>PxVb1=Z;aH)HC3hv0>`+_6T}7qJ3sBloS5iZJdGpmiDv4GE;+x2e|;dJW4gXKT2h7^rG+gX8Lx;(B#t z8ln;8%={aR+G)b^9%%LnPaUp&&|+#mbyNDVFSUl5?-mU1a?C&zFQbkSnuK zA!dAz8}v@+8Jv2i+gL>Qz7IZ1g}O!QiycRtPN9_geb=W;5teM88yGw@C%P*eA1DLV zXP^Bv3z>>uddNbXTJT}!I*zj*@;!G9+AK$bO8cl^etesBd%ModqrW=pHu;O#Mqrk( zW{Lm3guV*8Yw6(Y{RsiH;qKM)>J%ppf!o+xrKzv)cx0n#a;c?MVQYHSqRVF<16xb1GWhvC;Aj%4vm z30@n@2g#9eL5T2*c&J8CHSEjuo^AzwnoW+*&ExjFB=TD(t{d!q+0E$m&5RiPk@>`Bb`8`d!*DkY74k(a1%Hkjc~UaOeBE3P2En3Ec&Q_3QC&4{ZqMC`iN?VCyV z-`#wy+k~={!voMn?BJYjR7FmO1TS8ao}cUedAw48!MJ*~4s+{x;Us5|HqT`poaPc+ z)qmQn=aejdOFCSR?ahu-SqnA^j{Iv@H1JbnXr6Y@PcLfcYCotR*Y6)NbjaOle0T&2Kb6e`VyyDpX>bHgSixlpAGpU6j_OLvtEaV2 z*R-?d-W({wow^XlyIv$cp@d<6$vwrlAJS&E(7YQY(QaZrycH)ivTLO56dkkFi2B7c zTMav)G)%B5z6u(p%{oFS4cX%HgxnSa5qipun}@u5|3{D7lzJ@ z&^z36WSQG>`K3NUUDIU>xsjS&Ul)6Slr`RDd+`#57*QFmk=q2F>)Yf_t5a`$&P>ZQ zGX1uT;yX^y{n#i1b{!^xsxp_<;`Sc?t*kcFBF~mSy&GaG_L=1`KPu|B^C{6AEW@m) zDvkGJXIN*BAS1pt7Y+54jS9@FaAA@DaaGVr!M~rK({o`!)mE7A+`OMb)g$YVg zvh24t%IWs}RN70ktyasucA#E#u+^C(I|0s(`P4#V5<;e92)o?9M$PP{54h7DdmPh^ ze?1Td7u=ljdV!Qtg$6>a0O1vBn6nF3)RFqN01*@6`qKakSGnN{LhNdwNsdRiQ=4 zkcFk2J~hr|M(wG2en*}Q;l~Xq6%qR^Jr}$KZm=u;FJHuHePwg>mrI2|zwiW1GRO0? zXn};1`0jyhFZ<6~+avMxH%p8?>9b=TJg=bhd!6!*V#_TDe2E0mW|75fHt&P7YVypp z!v+>ye;$*|@{~{hd`gx30xTz7&wQbpOCDz~1fY)gu>3iZf>aqK< z^!P!j|9rv1@)y#OLX={Dc}!YLxUeSt1kAzE=2GK$M2G(6pOXe_RZ?0HX z`Q{VQy$(0K%rj$ruh2ht4@Duj{JfW+Uj@Zh<^)h4JdF3K2+sCo=ZyBXxR!mED`p1x z?kb0*#U-r1>g7KkNH76LnJVBRG{ye+PqJAv%?PgQowE;)%wME4mmVpzmv+%8(RP!( ztH|%L-0pw+`3j`d^l&W6SWOlyktZ52m?8A~-9shBAt#=%jwNr%3zi^tH-r}+jpq z)arO2GS-au)2q9a9^yM6+dA%EG%mFJ-S{*o!5Ti%OWwz|Lq^}pZGrKY(ffK||26zH?bgZC?ZnytS%OPZG8 zYrKMB^V>oPAGf%HZ=V8iJ1VduzTwB9bJwM$d)hH;dM&0t{ixM9-v)UOpsbXh?Obf``Zl6p*!sHlqes^eDB{&3zymt>(SJXU2<*&Y(*-jt$NEjh^jJh zdRez|&OFs0ZCX0yYp#3|1`4%p==|<~1rMe|8(NJddvMgXG~s!<8(_Ge#k%y(z0_u_ z*&VYT$DHg75{G=7nz^J; zgLk_8?rxt#i;0ZGj^`&9vr-6v&1jKqpU@(f-$P`D^llva_=}2<$Vvs1z%ctAJKRyaJ*0-p zuKh`e@qW+MvJd5*{a0l@FOOCR*0+=Mz7o5$hrE`Ks{6Ab?t>Hi|FHMn;c$NK)=7v6 zqW9iQbfOb&^b$3q6GZe*Fo;g{9=#1A!RXzH-h$|zMDM)~hVSt!=Y7BPo^ze|e1Cuc zjEif`JiFZYUiVsS?;Y@g{OFgM^9q3&WUtS%x6GT5%lzs&N#)Du?-XIOGGDKZQ&uzL z37-QAF?e-!gBhO2lXE2~lM{)}y#7{AmWyfoYqSdO^l4!MYSC@c^FNp~AhUQNFPFI~ zC9@?{eyJtEsIIY|ij$7y_Mni?@I6`EIL-*4hTHvqwB_4%mO37aDUMMM8mJaOv%UTz zgHF_c%ECDiX{8Bq&&;&=tPoK@s>n)!Zy!0wO1zAH%%#E-bi~7ae0I=!TRC_x$5}9w zQf~RUW?(@RZ4ceXkJ}g8zq*qtV91WOF!8FEAJk{fNNT@6=fDit!(bToc+HATO`0tb zw2~OkIUS}g-vtChjBWi;pz20oW5Tepk z<_z$9kGQW6{O(veqrCO9d~J)G?}pMVe-XK4!;@)zvG|IEoglyOT6hjZ4=gT`JZ9W( zn;c+?t#`KrWlPniZr2a+@H>y|ji+XE1K;qnQ!d^$)0`-S=xdCo z3>|!mn+7P?8w@S(q%jBPhYXNLu7E{t?b@NW0%16aJv^7@BUfb>onofCi|@^zE$&bj zzAN^9SDdi-X{KtznZWFuZZUXKSMa9s)oXi=FvLJZroO&1Q!_&5tsu=#->JG;k~YsU z^SGWKDh95Hfr9~xfBi<{3mUEenLYi#mlS{_1(cXUP%~x?&a(xTJs}5*hWn*@Y_2wO zrs`ckMIdDg5}82X_Y`)%-$I$u2|C>|3HnsTR^jtsnS;6J*Jy=0xULX&G6N86j3%b- z19&y-w#e5xw_q+}X;(aM6#r#&fT33Iod2ZZf_SDGh+X;ijBhGYTgOl(%ja#7ftM-~ zgR!Qx4%Piyk`(?OSZnD3+DDDXIi;c6@C4~u-%V1aH$qz1CezJKpja*1u!0QNQTojs zT7{pL$ExJ=BROn|T+-G)ER-6I$B3?!OHP`~__m zEUEmMds6ivBIBhFHgOyd+r#j+h*y>a6;bDjcFN;JD|m~&HyKf2pVeINH30Swz*7t@ zHSgBqS%<_YWT#YbO;j04moo*7Q&i;yYEi~gmac`z3U!RVB!{*gBj;U*O@x}eD)@u* zTGv@?dZVYm)w&98NF_X~sKtO?xWkvLVl3D-Q>r|_;jQjQl<6{7t^-wT!rYYs2UXf> z?6*=_Aw#4{rZ22owA?o@4O*SM#78uD8Ut}z1Xv8^)x&4bzqcr8C%|@25@Nazfl;d^ z=+H!OhX&Pmsi);OB@M~VhwDs)iW*9oyMFo2VhBDs;zFi!YhAxDt|yB*Q-wZW{LyD` zs^*h-o&zAdyW02uC6@Bd3l~BXL)&g^ceItt{k_yA*RyO1k6c*E)bX_v+?QR|2N@)w>XshO)~G_NdZtHNHBnYyO*+AaFqwwf7Q`OOLqfN}~fxKKn+4`!Gs z2hX+J!ONZ#D(6*8Z4Y(G&C-?IFBFgx$sBr)cgMU#vA8!eP}^bt4|KD%7!FW^$g0%+tpsn5V!9^TrV zX1|&=%NpU7LRUeULcb|_t?LbZjdU6g;w;UxkuWQ!FzXSrW{J&{xGC7}WsAF|Q%1Cb z2Rod2i}3cz{6QI*J#2!%vQc3v%*hc+nnxAv8z~A9Bhtb~42DDdsaAMa*sp~4QO&3g z=OApm?-cYj@SaJx`-bubQpafs!v-S-mlM$Bkwxti3E0}F=m>{-NW0$tX~G{gm?x;h>OXcgtbrT zjj5+USqIhx&WGofR3tQzP-+J`c5!mhL>)*T;S{5$iy+dQdBiV_%y96-d!o9J5!=&Wl`yn5U-go0_oQ}o3{G$l} zAkvote$1MsvZrlLfo?qz_O}-x&o3&iHgQE&xIpS^I6E@;Zln##d|oqVrf7O1QdNKl z+4kf$Q~%MWWE+TpbS%d_;7v1jC9}rf@D{q`54YbrSMJqy!@K< zT%e&7CnMdqHaOK-zrQV1AOWD@G`Gp#I#=kn3X{8q6&E*kPIfIi@7UQXpsZWSFPWM# z&B4iX-R*nw$@)0O2UC*mR4aED;+OnVJS>%~qD3uF>*tL^kqE9a`+0uC&IVz>lAf+a zc1$8SMSD{?I@j#&aEeDX`_ZVJJvJO{tnicGp%2Exatr=K%2YwwEo6IN;|GQ9GMUUY zO$il-9N{*cpFcboKBq28fS099+OY6}^WSEjQX`Z)C*tQvx|XvM3(Pg7h^; z6S*JN|1AfK1+q(C+4eMjfQp0Or;pn)UjzQ&%sxrD$x#pK$-P)d53wZ|ey8zzh3tbQ z7?VdG66;-tNwI2Ey0_E2hb3lBgV?JJ=CL~lZ8^}A>lihJi1MEK;L%k2s@5xf^SU&e z8(;~+7MsJ)m!IMvREBJ%9e0oGV5ocLx(cs=uP^Y@_c`xw5HX_szG;X%gVmkco$(jy zYzNx%0@DR&8CB|*^x=fw!n<0eP2^9SwOuoMpR$|)L^e+N={vCnCJrdC47(FenVS(al8*q z=w8c3roW|g-kVPM3XQ~qoVbixT6R+$C&B978)WAJQlD%>sT!BwUMoDpf&7?{x#q;U z`+;9x_$@=!+E~KU;0xZTaI`o@d>jZXPG98GLjW1qU}Ev8Yb_pOw66fcS#7?DJ}hdl z;M$$jr*3Bp@&G7z+2HbaH$@mDFi;u*@r5g4GW!j0tr{;nacrpdjp~7~ZZKg5BZZnJ zo)DmkzKc12Gc+k2P-U1>1=-LRcwK+qoG?esRG_ox6{?wnS1A~qyHb6^S#cmYWb0R5 zTI_ug#?7zquhVQ}FXys!-F;-!>2eQ9z0P?msZWLT^A}<~l|%vWNCpC(1mB2zq#kbx z@jPw})7!r*ME?8>#gNAWhTfm~7FSKzu3ZCpIYV<4MFrsZaI}y4EF{jHK1qiWiKTsWXc$GhWLwxFT}0H*i0i8$EPn!1OBV7Il7xys;!Z{PWGevh4Q@)P*x z!*ubaKj9jBy&GojlxfKD=&K<)7Ea&qqA}*ike)XFpkk~~H_b{%*(_(@oEK^^<(Cun z14M0n2fub85!lN|o5F4Lb`O0$diPu1`Dd;QogbmoQIs_7itORhC&`=rtzlPNJ(y`k}!fKq*)E4Vf8(!;JUAh#(0GIL0t;)=BwKM1( zTVe0x374cc?0!zr}V34X7pZFQw{2ar9-X;fKt>o-Z-Z)7p}MX9)>m**IsvK zdOa;h=*>>AKexPZ%>0sV$`#=qEW=^!c6T}e-*ernDbXzmNmy~BhNff9ty5Q`f zl;NT{`vyRrJbC{c%xMS%RLRVyUzIPDwP?4G;zAFqYGe*5`GxUDa6&HbG4U%g9(l%= zyHbUoi)0li3=#~u*M`c}XRq?5zHcEVB1@3Dmc2GPyDLL_BKUR`#O=TX`FMIt`~}NN zZS|R)NVODQMo0P&WE4~ z`@!Iiiak~St-u3p5B5&Jtzef$Gwa()F7?lg?hQK#U^iM=UHVo1;){{}Spj3-xPHdQ zf7Pmx3TR5E8IU#S%RMAqQ;uy^~+&_dIiuV?@UygwjxtQW<_J>u7i;OmKv0YO4h z-Zg~sswVj{0fLh6^r0?bH(G1|c0JqZJKmxqEpO}zFfaUk$HF^hOq|0v!jooz3F~o+ zQOP~s7bF2lxlkX`SFa82&XlqG_#HJO0$t8OeAAm7c&|w|X{cm3RmIsTydwtbvs=nD z?N;yirqLBR=k`Y3uEhG_T7u|{gWlG5Ewsk=U!2ymtDMKg?CDi_2O`+j!5<>b=!70x z&>5c?3LtE($bYQ7Rg?Ovf36blqN+f~29zwaTK_iLk8N41)U?iu^Vv>$wflKt)^yR4 z!*pY=>2e;6Le0g#{+Rh`zVvFBfOyhOA^+x4s6aW34NWH{1-O+a1=dH5<`M@r9BNP- zYCP;mg90}VLkBBs`ack{>;&XVx85%ET6vlvK&v~8*A(Q$aOcacxjC&0_1>v7W%h>9 zY;xmVMD!)3%zDA_0yQW}uO1@nI6W_LwotE;1Z8 z2*p9bNAi`?mgQE>@K_$}I`-nQDo|(-Ng@xs*p-xvmvY37Yy^u-&i4b$Fw=TtSE<&_Z4ux663*ah zGy9EaK=7c%<{Se__9g2a;ItZ2AC!fyH`!lJjS7BdYRds>)$OL^+@p&(|9f=NIqwv@ zpQ#YKv;EPFyZyT-AbwkrCCr{4J#{emrQPF_mGkRd-HemAePNWCSSO0nUOU2}kA4Q+ zdKt7ysPAB-IfbjDr1y)TqH+54D|SOimM6o>Ex2#WfMv&sL|k=f%hka4n9p9%z|Wi| zlTPnNSa}AQd$p^0jhzXk%=inhRVZhRs&J;Sk!q6VjgFD$ntU9TQ}iUrV7k=DWcIfl zWuuO~-R3Ki3`RI9(Nf3xysA4F-`R~7wc((y9PC!`;)#%LjGdj{j!}z4*~cls>zTxz zAHu|FywLG;15g#F)1aI$uOTW0Dw(F%NCLbdTaJ|rXzL9<=gSj_0^d!MguzMXzK?`7 z4WAic+F=FRbfqpO0^Z4Fqi;aG76d(DnN~P!Y6~G7z{6Ym`twD`yRvPkMuUX|qx5*O zl=@8#@wUEZZ-4?w43_4!2)gjSYo(;Z+pvHO6fl1n`o z6uJmd0_j7;eG38?gn5z0q3JJ&%2B$Ls}aN7Y{OB$CGAblHS9yTkDaGuI$08yN28T+j<6l*s{ zheK>kHOqWYCWt1nQGZW%#8!d|y%>Nd@exaNf6};)BzdeM(=F%3|S0|qLuZ?7w8 zkk%R_hHLyTrZ`ms!3@XwN;{0TtxT@lj5cX{-yo8=b|X@$h&JzvKAo_Zm6&NA-;9eu zpGE$;1L}j%@?9+5)J^SZ0|O{zcJZIVkC8B%01e9MwVss)P{~JE4!YM2Ub4jwi^J87 zyzC?SX7_6^d)89qd z8w|a_cqN0y?~RISJj^^Et2x(eS4zVvdW^V@7CUzZ7`|5(xrNWD*zs`E`S0@#;J}Or zSmXo><<_*+O-k30-rXpHIjKd@l^Qe@Z~4uOD9M}4=r1#2`wq$rQ*uwRVh2@Rw@WS- zRg(Ze9~N;}ICDstjQG`}2>l7~5>*f{%cpd*QGy>QNkU6ti>2USA0+ViY1Y%1iWlwA z9ptJ-ITDP7mOQo)e`sX*xDWF<=lI1Xe2uGnOB%lPU6I+@YBY;mk3BF3gdFE)xc=u= zT0uWRFK~Uiu9kX@@R-~xShV+^?UCHQXDzC&=Vm4y-28UlhfM^x-r7!eQ|#u+^{FNKJ$7FFI23f=U+7BcNmb{AZ7;|&9Hr6)N?`>BKq&WLp|Xn*@hz-P5YThBlOTDgCn7K(PsX1?~3~q}mBFAX3A)%dldbUOh@Qn#y&-w1fCf%j+_ywPA zq}3859X%vAskGLYDH)t&jIR=d(S0lID4VJ|?6KkfV%SZA`z?dApmZ`etGV2{2Iu!x z`K%aDv#T}+;0DUKp3~o(%ohQ2u8O^uKj=#6H5I3DmFDAPx2*7>7NBxs$85c;{fV2I z8AS=cK1)e{pu&61H|OzD)tj!%DKf{mGD1VuY(H<2DsEw>lj?U?x5ydEpVJEMp|usS zTr0m-0GsY{Q)W}2-tR%$*qwz2RMq|b7wsJ$OK*E8EeEe|Ycc_^RVOHNe@QLc#_y9; zz%0$l2YJ0pgV>Q@ObJ3kXAPK~>mY z1S+*KQ(h|r;M5Q}oI8K__P5T;VO6rzi@8N}{r5{}8OjSM2E>12)vAeCF@c_17-$84 zSM!&s^g>@c{ge(V05tQz(|=q`zwOhNAkd|LO?P{%K z&Qa*onfPOgL3@T&)Y|&^$QXySzy}jW*&1FqbY_}>5edc4z*Fenb12i0i@6&}dR|!`itD`%O15Ng3qPb{(K)Y=2sk{%oKk+21;=z<(=( zC&WX)BK_57i-3>&Nuv~-;>m1rG2$5{S3hUnw)>ucDbCDs_%`0T0Rhy}~~U zu)qHPkyb4jQ2vBufYOEF{^h?mE3v@83jb|!VMN_DDpp3(YbySy5&Oe_K!g0rQ}rHU z$=tvE*NliF>?`J91D*KvQKQcD1pi6<=l?PCe?FErz}@}nAW;AJJFnk7rzZXH8T5Cv z2>`p!y{|&^_5I8L@235v|KCmj)h3dE3B~`Rn-bpwdC5Zth`N7dt^WwAzar&7|IkuE zwmz6@X5M=ZM8|Z50*umYGneH;_Kyz%gwD&O6`9tRevs%@ldEl>S%D5m1i)BZ2mc3S zEiu)&{jz;l{Gap8efHa?`F#hRbG23U zu>C4Epyl_qo!N?v;Ljulm?@_k!9T~m|N8we)6@NDVgpH;H2P~il!iF2Nsn&5oj?H! zC*NFpX{>#$Pj9% z$cf-*a@u`548|QbAfS0)1s_>gKVNt_Ucs_ZQH;i}i6}zD6|JKEK6U$*HZg-;{$FGH5_MBA z1==KtuW`&k?;Apty%RuULsx($?0 z(PVNy(dEc~8>Ilw@ED~?|N67?#p%X+x4jTZ=7C z803O#dNtNbp2N{1`^+}bmRjX>{!)_zOs~TpDnGVoq%UGsGll!u>*h6LJpfnNM&(A9 zcfSHtCOzW%g3~J-eYEe<%Ts9UZ9L59x_>1`oMXx7BJ&1uvBcxP`$oiG0rK20Ciqsh za?2VwBNTWntFdGCXk(Lxon>#+XI#rSHzIk7Jp&Ywi#s|m-zd5-Q!I9||1ua_R`)Z6>Lr7*G}qOK?JDQZ+|UrZVjZA=B>HH99*CzP9rLQn~zzRdHvs)S_yTe_*!Kz^Al- zDL)>%_1tWlTvNMu3JSYYr1V_H@jPg`SY7Z~N)mEezsK0CfKm{H@p(UqR=R-bUjveX z=6=Th1g3@w1PCPDEg+DJ{~T1BzXp{DdTTNc1Fni-ec`=7-*7yt#MwuvZ%gxp+)*NU z$$i#}N$BJUFHn@_!~@ocozgnC;@Sa@Ci5~(I@!^m*$hg_Zz=8bZdE1IGLOn|~3b-E0l0N-oT`%=%@{>NM z?A}-62zu|`3OVzIh?CpuGN02CxdQz<`^t7S)?W8HJEm-(^HhWbgYlTuo9z65nY>+p zOx|%I@wCYUvzTENh$#&@iKnnpioaW0$%XKQucg>s%_p<#_jh24lTR0XCAUgBzsJyl zi+Hk;kcC<)F-{dJ)@=UQ+Q)8n3+`3LAxnVOsCNNcwrXSk0N3bqVHvQMT6MZ4VBxjl z`e$IS>Nz5R)vqfnshzbBxV?jKWCIOUb~$bPzk>_-O^xJ_Y+HGsZCQW}0=my2+o(Y} zFA5fKx949{q;HEh1|R5uzU>6Gz?;UiNqxI#ZG4k&q|^9p7+btAlKG|#TvCS)?kk`AIz}wOo3d3@`c5);fq7eL z()I|xnIDtv>8pk|ba%5Mp6u!igrQxw@6D0i9JJ+PQJ4hjHtfo7_;vOQQ zpn`DEWCdo-@q~a-Tm!z|Xgw1<2+l@lPyXdS3b-31i=wu_y zR}P28@1`yJY^Q&|Z9M8GoAdBHY}mZW1lN39R^Gpo?mX8v09$xxM9L^JRv z=T00sl)W_puWMj*l&UEk&D;uXdpNVtIv32l<6Zr(0vC z``9eRFBliEjeX9vBM2HVfCTlXa&y`&CaG#jSn0K{1%db0T8c4;$Aa4w&z`JYPZSx^ zbT+fknXq&xoKhTds(HPaN3~DHR__uEj!cPc)m`3(3cSV#A zsYShW{_wXKU@Em$t`xov6FAJ3Wn*!LpZ~ltYVn$x>dC}v$g<2eiy&l=f5t!keMoMT zIIg5&ow%E{FWLR`2KSd}O0n5mm$L~SqSJWa6I&=fNw%5=S_6>g!8qgB{|X22@7HfI zK9CLnIf$qub6C#KbvDkornNRk*uhKfr%ae|v^I%_=?+YhAGq(a)nX0CeVFVZI@Foe z+47dkrAlG4VV%GY5^pL|DqVmMeD?TP+Y_opN-?iqU>qFiTy+fHv`IwVOBm>Bo>42F z0OxlFbDv6!+O{Ij9|@st^@&h_p!TKl%mgtQ)8Nx^x2;u=sp92P;h@TFw;H*Q$nZaY z!RJ<7x|3uzR;U|4`^p`~pFkECFtgg1z<$)%lW zG#bmcbXbCHgVRbkp@_U%ui7aJEqeRmC)TpwYVV5$kGiWMiD^tq(P`L4FI^12{I4(B zzO%S}06Prx$p+3O$Q#(?HZz7AiRs%MSM*RO0JG}lH~VOEq0z#Xkn*=|BP3z&wE8q8 zZIP$#r-287jjh#>d2l6D5N7*Egay!OjEEC4)l?oCRna3*1e2N4p25}C58mPFR>^j_ zf`Isg(?EHpb$my;qLHoclgpKux>gZWjCVgin~t%I&;pV%=@)S)H@-O&4ukri+oVA} zW>T#`wP8qC!IaV;Ypuq*U()V+(*-_odG7W7ru*c*%{=_)#M5_KHjB*-ZfPdXKPT{1 z1>3f@rpcwf1A$w+{O%TnDSsii*Qw}l45nLd2%jc8^zQ7&f|e?MMRELq8APk*mHIEM zY~qVYz%m3_r2psa2h4^MYKEODr~{cQgcgKJKQ>;h$!--21iVOcECVj~sJ~v3Ds^F_ z>(<*o*1SRIr*W1|`;Ffi-dHgq2YGWsh~hqt3k`;XgKd9_agdJHgrf4xlVo8XZPGsn zc;yC-09+ijVkPkpg*x9uiquiJTh3=$nPg4PvDLxY3fR7=ojf4k&!Z?rzx$e~H!wvH zC6vM%XCVgnnP`Bs^#&JEOi54EY|1LUBtb|0tk4WAadVCj(w%hUF7u8Z&+P2vp!H$u z@@n1R70R@lH-(T%8lUFM-(k;2aY#MjiROr(iGvz+TvJ=-qpmW$`OsL>1ku>x&`QwE zPd+p5R~E3!N@@v^k{A^XOjc2NohI1s`hH7HBRc;%m`&3 zNsz-y-N1QZInybiY6ywFE&G>MSo`5Uu9Z{ho5GF+vWN~XPy;Z#A-AKZX*l?~*=I+q zxKS>&U$86t0gnP{3VL&bHj)l<^3sgPRTpyIR3mfRV&ZE^(fUGs8Seap0Za7oql!8P z%Ojy>0^09kJOe4bi@#0W)PzbjO0-L&Ngh&Xig?sQa8|tnQJy_Fcx7#T8y@fec~hO%W(j;Uh=#27 zMx1(w+L8n*%t4;I6}CoeY)DV9UJR=83Pv^0lA*dj+p+Wmb`z>s-FTGA>l{=W(DPfR z1ITO}?LhJs^}qt}LQF8NalUViiSyOgmpRzU;AOB@Rctq8Vex%_R3#=s<5LCn5j4pd ziD^yCBnL`mYRra^@orG15%{Rb-^lla0n(Qdr+U0FC@6nD-Z2vrBwChT?h(*T@Eg%qXCUpr+@L-8%XJmP=HO_4Yxzk9?V z)itjG(Jp{`L>vlc807KVPO3kl8EFUM_yI6-Qh^}_z-e$8fZA>7a&^gWmdOc<+>{LM zI^&?~Jxt2Oc{c!N#7{gW{ZEy=#C=HIR~$6=6$h||&>;N2*c!({rF*Z~f|zLV5$2!p zbVXmP<3a!~nX9qyqh2Xl31M$&^RZC{M_X{+QdOug!#)l&a99)y6jWSz(PAi=rend; zjueB)+(g1r8E~Uf1;Yve5H_t<18N;%qE|USixIyq2cfQR0%h5|>{mVpsG8d=vc=(K zvM>!M=Opdd#Sf~KY7}P;bRq~fFaoegdpJQL4I-XM@S0H=9j%>9REzh^8rHUys{{y) zVLSds7djzZd2w<0D7{-n0&ubWUZ`N``!jUGgvJ{8z1d_}8FDz9l|1QW(XwkiDKs|9 zH}?R9@KQLClDQv|ws_a2@r=iTwo-#SM87llNKqR^_Uk()G1tl7&0jLgXFsEi^%BI6 zIN=9?8F==_ zUWwPZA=OR4jp)o&#sy4Sz)nQZR^zarfiF%9b=+7!7<;Y}%y@2Ovd&BSD9IN}0S9gD z0+m%wjj1OVy;VoZGducRfn{Ezk5+L|g}kQ-5NQoFTtBf_r`MwQVtp`uAeKh@xhqbK ze^9hEZ2c2BxHI6vvrphN*-IJq@$z0BRC&~2{eW?f&yM-R%TUf5u;ZYTLzqDNAN38i;s)T4xq=uuKAcp0 z>3D?G0}AL)IYq<&pwR#3@!!$`h2-UZ{Spud5HpGppgJyymLQhY+}?u@?Tu9w59RA$ z!h*M6(e&!*RAIjj>1~I^Ljd!O2jpppqrP3u1P_!^g`34)A!>)gn2gWbJRhG%rCCbR zAWw=0A0Xz+DDBl6RWCT?vTALnRDs=Jruis=qimKPT~e7eH;*95@-$9w}gE*jzj71-_R*sDs!`5dR2D>~KJOC5s`zk?^g z|Kcm@_ES8hP`~o`C!ibstGZFynP7ltNWC?Tr%ALqL{85ZY#MITIk_H-+(ZE>vo^aO9gIz2tg@zUMx)Fg0G*QEa5op(5@+$y%GVo(}hic!+t zD@9l|DW9UM*qN4+ERIUDO0?*)JQ+jteZ77U^7~eRLXji>`4<`3s2{CQG2n&L(X(Q> zzIA9tOh)5bK{`e;UBvhd6eOQLRqZ!%=$`?gr6Ev_Ql&p9Z6|vagR0};N)HhV67**F z0~{EoCmX|c#c>EXD`W7HOs+5Ml@vl%uK5o5F6U{}&>63CY?vsLf+FnLcd_k_N`~MF zZd@<0g~vvdLHul7v-WjiikhkvY<=6tfCpoqCfJF-9>9}(g5Hm5qg|^YOHs7oISBqc zjG~Jlbs-X>$EEolzgD~E#035bop(D#$lT_BF_dCK4INK$nju}X%j4i@w8lW@0pEee z_3ax9N>g{%#lBcZu5^<**v=91_W%w9d;1R4hlb9NNCxVpwyoClCv=P#Zt-=N>O&o< zo)Ge&f5`D#IoYYSHwAQr-va%jmzTA1;Xr;1cB5 zCmbnX@Bchw-t_kJ@OLdC6%OOL?KkCPf_S73I+T}@Q^fiLbGE$tcTkmdeiCW^?qiQd zPbLkf8Puu<+Q$$buAip8-M=u^E4IIWdwrRy7a)0tx*!7#NHE_v9Y|0NaXMBxB)rr6 zgbJ+oTenPBFLn%RL=W<=lj{zcr*q8Tx}(VETtAJcu!RK0S$4fofK_c%V;71CQyWW$ zdIomjyP$v`scI9iaBrXZNJdI2hJ;7>2LBW~pEk3zUIZ$znT{7Aio5ekaPYX;W=>#9 zhax~Ru{*VS3}zsyD=?xmLaRGAdHbJo6?%(OwQup z{i1UCg*xQ^?Mr3#%V;z;c3KJ%;5DDr3GQ+7jm(L5^)&3 zpF?dxZyUbgu^Oen_Q~5(4?36>;=yJJFB5dy%5Dqsg8Igb{yil>(N(D4V_ec4E{~S*-3oQRZ}{o}Ky2LrB(pv6<~qmJJ$#)>JQ0jN zL~v#E#5DR8(JeH*9D<3Y94!t8i{kaD4iOtU@l_#enrV8A$M&L*FBUh>fAd-nRd1kl zZfOvwVxKiKpJ;10g!m}m-6Hmc1Gv+CIsh8le#55RJX-cUUMRBz;}btB_^lw8=Y_06 zpzq4hwuWPXoT^n-&8R8jjlz2qib;`Nd_4%WJN(t=&faD9$svgki-~?)I{yGD=$h>U z#t^ojy`xqP@M?w)BJgm|pPbX1r<#;4la&F~9^&Qq{I6&vLXOy=X3LqJ+h1xSs$7=W zN9M3;pxSe3|32GpG|VcRz(--JkU=+t6=*Y@C1wYIz_dI3xlZt)>cTwwU9^kpdr8i9 z;l;k$uC(vq{zfE)cJi$t9}!hU$#(LjZ$>`a6c^?0tbKSPYUfL1w8kSX0CTiBl(L&T zV#w1}4&{_%2~u5#^fWskbrH?+#dx*}^~<)s>6>C^-r_6xcrTaa18X#@tW^TkmVBmn z0$}ETk(El}2B`Py@aNhp3Vrv&8vkvrJI@N7B0*dws2m^96;1?p@-`rK=m`ssE2-?& z!K@|jp`n~cb(&t02&qW!#lV~aR@v;wPiXO08u2i@V@Zi0+>kM2rjqY-)l0iN{*nuG z4m^ERZsL!4wvsUQQRM|ew_MN?m&V9T4L4e!>+`|QWZtVok4OX1hG3s8EN_aqnVZlB zwewBTNEW{n08|T!ai?R^W@!X_%+Q|bZD>C>R zAHx(Ul`uZV;(#1XF*dF*$cVX75{`-UQh;@wIe`o#?3*>_Y={G}U>xw=f+NFNm|BGjkE*S#>JK|Ga~0WU1dx^tAQwZs8F0mPIy zW`NbO?UvV8j?oa2OWf>vpenLwKBbJx3SE_k9g|xlSJmctN0QlvHiteq7r_J$xo|n- zdrBgk?)18TC0e8;4rvtAFmzmcshC%}WtI8Sh_Z3`ZE#Kiz2oxsz;Tpx4x9l%L1ks@ z+eEUq>rXO-sY$76r_sxqXaa{Tx(R#o5yq^eZDXl5;VmrQTIc?; z&wh~X`gBaPb3W{XB^@0r@*fjaCLP3{;I5KFo{*q&T~%%bpLWgo9}v*+1bNOr1#w?V zoeH^p(V1qow5dx0%k+F6GZ^O`2I!#vlFv(gzFg4vti&^ebEw*r%)4KuI*yP>^BH-i6f$|mmCCR?E3oVLK0sJY`PmNs0COw@1cKKsKk8D&?{J@ z4@=MkZDD)US1|(le#2#tIhPlFEzB>@UaA)OOX)s6kk!` zObs4s$Fg?a38At^4@q<9edpE8F$H-+@?OWzWAwc)nl_6a;BcWtt!$^O-JyKn%8{DT zsvKZKIM6@8+s}`AIV(%QQXQP&z;R5_>{MccNu#95FM8AR(PZs9WOXfahoF*N;&_E^ zjX^pv_LtBm4e}FuE==l~UY#(lj9MB4C(@_rJRiX_jTaUh)=urkvqHQQq;Eg>J9j%#BVekfZCKI9oW&j|wmtC*%}f z;FI37EjWM|0GSTR74OniFwwR7+aida^8uw|5}oSk3)zvJ=!fiJB1*~x+e>*#IW*Gd z*h7ispT8JX=Y2rZzgnV+m=Z)GxQpIaEpv&8^SPrJ5I3Y4XmjTu4$SD9ZhXb%myzjia(0F ztdfQxw=&iEfoUvS_&N^JPGrfQeafyHPoF@Ux<9=~>wPI-yT8yh7Zt5DFB!?*Ws{O- zNPbSNKU(4RxliZtMbAohAd@WVVFy{F(_!HcXA~?yE$oAy^u(7dG}}R8NO=MBmZ7v^o2Dm%*BUly)onI0<76aplfVUwv>|K`RKT2F z42GdhbfE9pr?^SYx|X2NN29CSC^lXB!2gvAvX1anSim`!s*VZjVU?je^ivz>7`W-#u1{s11x zi#;+qRG|(4Cp~NgWhiIdhI%9%xXHPfZ)jv%u3S5|hFUp8b@{`mOxjO?9ILL(# zmBDT}6TrO3a%Mx#km^89UO@-&g?+J7ot)7Fkxz@00rA${ou=UO<3l9dNi-(&qV}yA zp{LGSPHjWK-3a?aOo`$B*B>}fcpw}QOm_-DQuhn69PDjGdtuMb$}lZ1f7Ucg*dmk5 zvjfKKXFg8qBZ4U=Or5_w<9N+wOpVS`>8%ygWf7*{o_spD~`|rX4mWN98lVjd74? zjC~~dIiWbNZ zb27)S+WxBV6E>Esh)p)LD8tVexD8S4QJ$D&4)}7h)Gc{b(=KgP`Z+q^vDvnG%@UQ` zyT$A>>4k~XnO8zMj-LIVANcI_j)NhSehlqgtu;38F_|#pyKgAPWYVJ!%%O{@g>i-X z>JD|chz&UeN5?uN!Vu@deL$T75*Bvk%_?gIegb3YciVIUx&DgcTo~#joUS|t6)gTd z0Q4}v<43w7!;KiUQH!FGTUU!r-#Dh^XE%rlHRX`vQkHpxF?}e(d}15Mi+Ok&XnB3z z)*YxoW!lyyA_T|;Z--5@Hbq5=x1LabVJk0Vd_uMp!O_?o9}9^wO&CClPyt~J$!`m3 z2rl<^6~?=o*)|`Q6k>5?(g{j$)82IZKScUiXU}xuE#`iN`$XS-wKvY4E)m@(k^MdrA^|#NgUvJV7lzuwO?8|kjj<)EAQt4| z9t+IZnQC?3$oG1#=tl`qaKevgVj$I%l#h2-Tazja4h)`Q-PW%YNz22(BGE)=gES{&Z-02#{4=^V3k=V3VA!EE{NXL z3#C3L$QQ7DLvF)O`Q)kWyGY=^CTGe+CDK*a4Z&Vw>q<>CBK6~%L*d;#385-Sjvov_ zkPT_XUd5tj*yYRPXP%1p#6hQ@288j#9_n~8t7b2xW^d|WEi?7!oqicFbBfE&aB~7v zx43WsWem;}w|-9(-Y`YjTRcEiCqaNBveaR6Z_hPcVc<5;Y zY(;X9_RS1NrTV%=$Gez8taO52b%%boje@TAgs#b_WN5eROZ3MgWI}aqr_pBd)g7iR zS(!QO{i0vlj6U?4>(H;-7irnvM<0W=GKpujG9SPb_5r zm5Bxaq^fZOIJ>v=B4)scQI`dkeL6bmxXz!YF;GbSgiCBM85xX)$ercemH4I#`Jcl- zad1;PN-U?6*8(w+?gArQr(K6=BmgK%!=|vocpsNh+Uwu^4Kb7Ww`d{5|}{MC~=SJx0&h;k$n(s>@L3 zv~Op#p2Q=s$l^{DA-HnErT4SP@w7TD7kmo4H9g(hv*6jz%V`|A?1oUO|A9*jBgA zgKBO=c22#Um?$3r_TX2M=gs@=8FSI}I}?=ob|Zo{8XWwi-5urVlgZTbbwGXR>*ZPO z#DK{OX4x;Od6M7Bd>(N6flBJ4)Mr(zc`hHz%vbT_0EfS_*}OpJYFvAzyR)m)EW!ex z{ql-xrv%r0I3y$VbNA~F)DDfG3a7EX?x?1_M)u&g2zX${(c~$|<4o61E5NTojiAFu z(HF!-jT8egOH{tTQ|mzeQl61<)*qbP{ypcm%49qvU-?m5F@Bba7I!p6C1LKgg3FNt zyb(RNIvZvRyV?5C-n}s=yON59eyFeL z!Kmbdmqar(B(>GI@~ms`KH) zjR&m&JnO$c3Nc6Omwf6Ew$Pk!aIu^lvIUyC_8q8YBow9H@hk=(V)M|Zs{;2D&PaQ?2Rabo75KlJI8brprRWg`H!+unvknx)tuXB-F3 z{OvNv)8BDQzX(btHgpR$Skf7%CO9g}$|DUl2x7wfZ=_{~$)Dd?$t`>^6_7kI0CBrN z%}^~@;bMARfQAXY+1TmbhUkI3@YRE2ML+p1MZ!h{scO2Bq1*q%-djgS`F4M!l+rMC z3Ji@jNH+{EQqnPm2!cq7bazRElA<)ifOHMgNQ2VdNO#A3qkf;~@psnqp7ow}{y2Y} zwPxK56fS1=wXeNDd+*P+_n}*rM=AE;n>5V^q*nkVsPd=#1Jfn%HG@{-;3f2Cx^YA+ z-j(QebAAVYdD?DpKm!stgjm>){zX+)?HWp=7_Hg3Y>rv7v^|RdOIq_M(^z)>{AAB; zoYWg8x-&n7jDIfGGR15Kx5!R@gszj>&`)5N3q6#5QNPKnUfwHZEJY23sf3@^Bgx)g zKp;H0uJ;Uxo*8Hyj8r1efDgu4-yX?AtzK(3nHNEl7{y&%H^2~nAK`ht z*lpM*!GVmn+`u{n`bY6wN=Pq`HvI4MPjm%$`6us##IV>7cQ_QO~7 z6x+n6isW~MV#d}rfE$=0@)xbav34D-ly3#;&vQtZS#7^=#`Q7H@yvxgrTDVB6g<1K zjZ%VN?9%C{_9eJH4(Q+lwm2)imH4NM{e{Vug2Og#|4c)3mTy!6L0EbdgTMXHQZF90vs9QkfE)n=sjZ znbphO4wt*-hBS_ZPVfZ;oRe-p8ln*J7#&~!0;ISSxemc!82l+Fb8m*?xNf8H5`wl! z8eYzPCiPW%nMVDa4v&n`6vRRv&IlHDJf4u~2T6cf(D$|wmoeMx3Ceouwv=VqA5L%Jb?c*sOVKKzJUbWmLuwQUep7Z64j0IW zQ{lBA*oexj0zlscn%sgHkAD;iFtD8pdL+2Ov+c&WLwo6%1K0utsX#x0VQc{Z<)+JI zcEvh_;C=6O6bL<{I8b4j+WRaEi4f%4D~tAq4bA4IEa-S0GdAgS&@7vs`s5B0hhtP? zkx`^A>J*{-GD-42=QqQ)^5IC&Vz%_Q{ytHuAC(enAr8ybBnw6l?-aIuwM*ceaIj{O zjy;akN*e$&9A2~TxV%?}9t+5Ry`#p~Z~)a1|JY2=6-Nb0$C`5HrDAl&?zS)iq7Wl~ zl3BFJyR3kzHr{=ZvA{24Jd6dS$6BG{k^g2f_ZBy z<9T=|jcK$F!>4%N-F+xKxtysu%NiuQ?`3*1AII>^}L0_0rD4WLY zmd%dvN0#O`<$SmOh2AI?CgzRdSiV^e<5`c^E}f5QLSv}140R@mXyaVcBRO)xzQ=h8 zC((RWh{Fg@tEb^~O3mU17iOc#q0%n4RcLCbhVuPD;jX`6N)>*;65BB>(o2e^8R%L+ z8+{$S>4tl2;>4iw{zHJFtMCC|{$~Py!IutCbX-x*71WT;qs);Dyum$X<#9Q^5~sYQ zpIgVh!2LBOoeXw+%DPhz{6eug4Qwq}`H|QG)Yb1N{?jnwdEZ#~u;QM{4H}U3_j>ux z`=D}M_GebP1@_ef~_h z$X}k_z35BD$Z;@2flcuSw~55xQ-%CoAMJn$89k&XKNfaZ-eZ`bs95CwNtArD3=kXp zuijsHQwowY?KWeo8i(o(-zM;!E6Go1v8~rHp4KcEA)|5vL#!uE_Elne1QizJU@`TS zHTI%k-z@{wW)n+Id2EtUJs)?#=zr?Pa91gy1&z@d6+hA{){kEtT5aKhA@R$1FV=h)@`oX_v0~FFG2!BOALRi zgefp8>+9ws$dj4}v=Aq%P@Z?|x4!dek5zg5dUbBx#v!;9zD_jCHQHuq172pss~v|L z2EJ6Qzp~9U%C^M`M>(0x1lFvg>YC4X!aF^{`f|UN$}LE^OOPQ(rH^8Q;zv{Wk8c@b z9+Pe&%rWg2ANTSea8x~cP6thICC8yGdBt;vAJun-7)zdfhj?FjCYw5?-WYrTLcFVg z^e@gV@F^o(1|hAbdGBgeP+BXK&l=b2MzSk3mj8T%ANM8&=4vi=uxxNhG+@GC1$@ia z&DQ@y4jv2?%HxTj`p{aD8ujTkOzgc4@(ic#M*u-QpV!af=}u^ft^vdjYjn}828uFi zU?iz+dsbp=rT_!T%L!E)7|;h?U8s7+!S`4&U(hctM1jE;mh3rW7Qy~m{s7>lKm6K$ z>tt^s3M3YfBC$CuJ@)AzYc7%(??uu-ctoMWpX+_c8=%a?lzI^}?E$FkbZ4_ae|$+u zK&(I<`Q!Zr?1THjtr!)R@0MEpNDfWg-LEp?e|la^;&qYu)!3wGdbWR!M*bjiOt zGt3BW_HR4^?mAdb0Yzma_b0dKDg3p(Tm?_R886d6EFeFw^K{N7+zCmeF~jz6RSoCK z&EBP-#GCK zg@$Gwcc5Zs*E)n)@2_KMtK}W?6%+p>=#HkF&Qv-8nh2d|lm0>XMtG8t(2rCOKnki` zCgU20?xs3EL`Hth(!f0s^SN!N&=&$#2)EPmwru5iMi5N`1SHeY;&ut21CgeE?;8pO zOs`dayC}2Si0c@*?}L71Z{nnpwtGX9W7MF!!&3 z8tyZ|KC3)>bg@8`d>%bd(sliEu}g>Ywv1OOk1Rv6mg5LpnuzJOW#|5sEHhb1JZR|n zt!GXYN4?+aOz(@59qS)eGFEW~piab5{U#YvJp{vff91q2CIyz6l~Fa$bEVrsW!iSK zXf8w_g=eeOd8p%yRutyZ93hGp8}#m7;(qyRCwM!lbQ&4g_YMOE7+hG^pYb0ghI)tk zP|7#Vedy5#0+G8&5#Gb~U~rBIG2{*~*ZnHk0>zUEt`I(AM&j4}DcKLq=upYp7r66t z8DL7Oo)%7lF3c1}qLBSPDZ?L3x9JtDtfzIY6=OD_;bO&j#;UnQMfl=`(l#o;`^)); zpC$#JlGhmgI_7RYz-L%yMcA}9T1;O%JTayo?e?o;{8ZOPWAgtDlYe@H*KR7Oy7ZQ`L z0zbgfu@1UDA)qmlaRc>>{V2^gsz@#;w|v=eE7<2Ah9j`+bn@Dg)ed?_A$OQRrWIo6 z&&yPWzUnYx6f|9#o=?TL9oTPUHrQ9*cn6$(n6VE3n z3TWzu-nJ(NRM9J;Mq&<(XT z?i6_JBlO2rZXd*;POK;br?U$ETa*d);`oJu{5D@cxq3iZ z+Po*mV<~wc9m7LnQjYQP-byJzt3ExLSqk-IM}P34ucGxBc^FUB9xcoJp%Y;h{!9AE zFgj~De)X2mvE!SeC@WNOl~vAF%F$RsTbE6ongG;8A{@l^M&+W-3&@7;Y@cXohG{Bo zJj$j(TY9pmvkg67ac{I~izmjnPI5Utt}XjWAsa#AIQF2*)7r%1_!78)*E7*UgHYWs z4eAG9>wQ@6rT5Cgh0W6EiTkD2S(bVm+T7d3!9ee{C4BknkKW0q?%7SONH}-32xqCg zS_I%(UNsw_*(nmUuF3vHBWMc2Pjw;o3T#C#5Y`>!QV{Zc=SUx^tWJ>woZtFy@$K=g z_fbJu-d8?A3plo~Ajg-?dduU!4!)+EE#=qCtP~0^d(Xdj-}j5UhsqGBkL?|8Nuo$e zeBOP{2FVUzjCn#%#m^RFQ4!{I8TR}mw7JdA5${`1M`|D6GM)Jn$xVL;_J9^Kgx{Yn zoU+^S0-M-vx8r`pL|e?x$jFw7=870-kK$N+!Zm;a^$EYg%kj`Ei}Ih@G0UvG_QpGu zJQIVauNM%SZ=GmP@_eN%C#Fk(uJ|Hm0Jje_iY0nh2LSp&6)%4z-?)RInR$;3g1A&H ztRf^M2^K2iH2PGZwVOHnVw8T2pu{wWN#BiWxvtgCU(F5# zXDKHF0efj#{==qt;zZv2%lDsQ)JSugVKzUHphS0nCDbc$gx06i>A&VUFQS2BG zFWrud;My~V??h2c0N1z*Y5e__)U!iQ*F3!m!*fc)t_$m1gaMiy6rB@f)HKwB`6JJr{*f`&z# z0p2!N$Qy`aKn*IG`XlQ}cV#;AojWI_zuIiVUs#T&hLk%^<-dnw56G?JOIDGI7Hb3|r zWJPdUbtc|+N_fJ*FP4KTP9f@AI_cDQLr-d=iZe&F9`E+VtoIE5H4DN1D9@MgB!Wv| z|D}lp*5MfS0@1d?#LQ2jFv}+)g&n-fYKJO2Qe7AA^25F}3h~72{uU272I|Qlk$N76N29a|* zxSqF1L(*uXIG;L~F0%4B#+ctvMH0;RP3G|WKnHiQPC646-`8q#{q#2PEW8Q)n1DZS z&#u>W94Yl)>8BA-oZu#y&Gswi;6mz`0>bF_H{;a?*%*eglbm*eEbt47-G(sO|`&gCA}xgf8q5?5ZNs zgG5l6;rdf4i$(W7owe!MRD6~~8j?p~Dg{;@62L#Da{B(zkzHX5Qeyit%Q8*kq?oV( zNhkXxVzKC2=sCP4^^6=EG4u%4|5Hnm;#Vrf66X`wGJfxExEqHmMoUf8 z#?c}SZX*(HRF9EITlj&;jYQ?zwH??+ka2Gz8F1aTR6OKfs)M}` z@~@GPFt-dnm?XWf=I#`pLrhu{#)%osYlM^y=K5Fp=e?j<2Oml%O zol8eZG)&_fjC4qe+2VD*-%0Jzk^f1T28C=VVpguAnMH$v+ye6F7QHaVHHwkJ zskH-L45Z0`)Y(h>7oG`9-_X%hNXLRBQ-rL8IdR6veT^E39=vFKYvkO91TmC+iaX$M zs1`cvw~kiYsMSmfxdm+_57C3jP|iE{T*gNz)9+PDRiFlHLZ0Ca_^Ta-8v9wH*U?;2 zw$$3r)M??uB$}abui_CU1P$tp{D}2k06@;lLh^>S|qWu>~jgP zFc^q%U+AD53?*)pK-1gX2{8$YS?#I~OzZ4Yj@tVPde-Om4ZO!^H*2U}Vp!lfHBTMt z1KJ0<6KAdC#Tk=KBbyPl9LU7@gYIz?nO=f9Y-bzv!oJp7#o{cIT=>GJ-$5=x?n4qU znb(oQNV2Xw3CZI1H4~jMTz%Fd?}fpc~SlYU1Y;BhI~v1!Vr1YNX`W-AX>Tb7#Af{7&wDh3v`@(#g|f~T5lo$qH%1=v zAs%=v%+`Hn1jTPp64iFbEMYj^8%xF1r6YN7sf20wCYTgAi<*{Cx;ubKxAm1tP=iM^ z@K!QoeJ~F1x%I>}VJ)&9JYSf72NRa3x>$#uS@WT$nGwF9d$vV{e#ZFKg`ppGcY1)= z-V@(NR?GW$=PNK8p`(J!@KyK8P)0(i+!ee|fTjn`v7p*_L%AG#5D45_wG2D^SPj?aRe*BTjs{{2$kwBGl&_yEHx4A6R40I56pDPG zY&8p?vaC#eKJ1g(3;KYmsvi25d2g~WnKSc1W46>Z06$TkrlggE{Jgi%*1%y;V6CX> zO}i8V7dijSclfN%&7uLm7*kXd-i9j-r&<8%V)I?9*Ts>8*2p2Q51O`TyAz7tt8Kr} zb{}`KW2tCAa{zjfziWdd*xt2qJ`Ws%u)9MLrfr(}`w;wj(p3FYe~CNw4^#9k*2oE8 ziTv8JJD4TIzF8*y4D?RODHr3`Qp-fMvA|ACfZMxYrKqd&p)EmeRakJwIX)CTETzUv zg62ADQ>K~sRMjG( z>Z_61XaqVGyqfWKVS~7dJ@?FSA3GB~V})PX=tSZ!_&G?pB5(TNMvFcH%7Mq5Ggu3G z;Axpqxpix&0`*V9G;EHgi$%36XfsV$2NC#XtgR&v{^0^F%tQ`;%`ktX=4l85C({#(_N1YttZ+8h9B}jwKoBBEr_WAEJ7@$kfW~I*Jj2;FzIrzEQcPNNf)jS z2bvg`pAHE-+)1-`!?|hzUaj^H%A;O5S^kqwetWvuwRURGnCZBDP z-4HHGS?L6F2D#8XzE8mn6ba0fo$bRS8(PXBXu}wsa ztPKeZ3*q*CAUJrKQ85m}2fHV^>?&(C;O|*+dAh5!kcX$C69LD9j}ep;-r)h8m# zI*&EglF?NeiHuVx$O6CL)3q^iXATPP%t1ktY17~4;LR^{03gGEykU%FHo<(Z*0O$R zRhZ!vL}oym1u&x0%R#d5phZZggc)DDOzSKMG6dbA;7D5Pmt4R}!3C|+;B)AfO1jC@ za&Db$Pb~#voNQG~EZQV<7$uDV)U&4!4M4MqR;yX{FKt2-p@5P;cSWSLV-XcuAPd{; ze?Fkby05hs4RrW{;a`F@$o=UWp5Z+q^`~hQa%}_jre@u2HKzDA4NJpC(j(fqF=G?*w_u zdZh4MBpL7YMPMs}MRx*v!5KLlopR`BNhCZa-(4jPvEGE80@Bo|!+|bqu>N@VtD{}hL!30m){Uk;@WEa>hyFbF|i35)o zJqX1p!rVV@3BvqhJH_wJE1O^VR^-4B)!$n@q^g@5Lo5(6pfK8L_lrwd~x%;s1DXtrbx_ z-I+W3aF};5tv1*Xjp-}kJorF=oJWwp=L0eKilf~*Xb_%pg;i|!lk`b!Oz9D6(t!0K zq%4ech0~px&Fx8W|0-A znjP4+7$WaF)zW%$bGe)RK;%sq&{3!8)jdRlTDt|W7sOEToa4L7Pe@@9|C}&wizoe|vPP9(ab1S0NuwH4KoO#+>&s z2%zbQH%5dVDm`x&D!tQV))&vFg zAv8Dc06b{<^S+0t(@xQ%KntpenOlJ8PLvGhssXvij(sm&A8ZDxgWoYKCp`{Qmh%Bb zUx5u9d(UfGc=cuPAZWhQwDrCp7lu?n8~iPWz>}~s&=$=i7{V1exzdwd^okbubARJz z5!C(h-b=6B>(db*+Eexm7-rk(_dHcr@vc)0zfnUIrse+jwMv^Y?UO)|$Jw4~A}i79 zn2_1^0s_UnHz;;t-*d{gx}Z!30q3cwtWPCE*zGP0t--_zP$bpnmujL3ZuxY&HBoN; zGDkUCd$u@Zk+rS+?E{gA&DDU(RQfR6;8y;uCrPD`m6CLjs=3AoAaea!yN5}ffX|@R z1zg$BcSMwlX?MbpJlhzc+-pZ0FfTJ}pGGlfeSVdO^3f02jE~JqTXcVK2JxM{Kgm6_ zZVp{6I;Ja$m> z{GKPLC8!~nfbnt6&+mDxv3O>5DD1A_7LnuO9A&96SUjVOhBT9O3oX?0VEJ1ji6!t4 zKO{2zKM7EniQZj;23&1nX%)!6(quJyJyfmI4t}1CKTRV|*UK$^BOMtS`t#?xu=B># zmE#BxzXuJ^!5y@!MJJ@AX-k{QR3SJXSbiX+PbUrSYcqAu@j!ncmLUg#0&ASN3miBt zL1>^@usjdSv(V9XOo)}y)dE5mZbt5&U~uVMB)fT_i<2nxK9TYFcdS;Z+Y{{X%s~a< zmjMEkeoSnun9}z=!eLg_=LVfX`z4_9Q{uGCoR^Zx_Dp>>hJ0kLEiu%=7vTij=wA6n zg0uy**8BRx_^@+v52>M}855O6Ev9J9spL!qRgF^|2-vp!iASS}kvw(6iIvMT|H26W z5%mAJquk&J6rMVGlV0w(9pSX zw$LtR0=Gq!=ipw&9OcJu6%jD><}PA2ZAL#7`mdE7h^B|ye@RIa$r}&&2vK2+tnE6R zdM)JipcRyw7eux9vu$a|_=g60`!lUd@h`1pO^()whMRnR6w`$(f0P&*+{dM2Q;ef8 ztLq(I8{&X7qR<&g>b*z7k3kr#Lka9Ni2V;=0QiM7S-=Sl;rD!7KT*q?Ade-gEy4RYVM)e-U?XzIReuXs@-FX|KnbJ!AA%+Fv%h(gb+%Ee zg`0jmvOs)o!;u!>)kO7$7(K?92vNLKJFw+c$d*9-By6ESO(Lv0oD%uh=keu4GTaoD z_!rFj{RM55q@Yipny0fXKky)BPNnlBr3Ll3(*$#&K6*OINb|B`T>>dhA`RpBUWKIt zir4na^lkKCg8mr&&oc1;woK?E>)LNclZ*rb!CNG0N-O<;emFm7;1AES5JDehFawYK z-Kz>T!DxX_+@auJMC`;WbTkUik7wONze5xd-5s{33tr{=>nPKNqXiziV6^LmFx@@; zz6jxmf{_*svCPn%BxQ1Jky<(;nBk5%zc;Q84H3@T9-1Ze_%DkSN{h5`l`g$MvJeJ5 zOh3GQi((ckDM%7P+Cn*~_9E4{be}dHmWKRWB;3*fjO8}7!ioP|B*1U{mh8v>&xl0H z3=itl;4a&}+()}oB+k0~ByEL@;dC|l$U|>cC7tGQY&Y>`v>4v7$IlKAXKS!cQh)L> zYQI9=5gGn+M%XnYx{xJ4YmYt0{jkIW&83gPn0vC+ZTp+H+!-P3I}wI507_=|ZIY7p zuQLQe{iC&elv!ui)XeQVm%aFl;g09jNrb1laqR{;7tSI8LNqG`&CT=uuPTCbaz1y_zIM`5m=W!vL!im^P27Y*|Dj9hk+oQ8In4><6YW$NZ%d3N#A86^l<{$;~)J zw{0v{z^*$M*@HnVrQ*Q8Dh)@|(eCY{jl%x@=^CSbKJeP(uL0qEn2qfB@BG?p&ywJ+ zxk?qDXT9|2nH&$l#m})@4x}ZtX70p>^KE>aSC1d*>cFLDoAdG*4QdQGpM2TIu5~ z*LmgQ<=wZ(Z);ridBMU%!`R0qO>`>1cI?E4aDZ+uLyRsJ$A`V(dv=7Td<5(?yGDnq z_ad}cud-FEGxf}UYY_2;ae2jbDo^F(ch!cDq8Xi9)EOf-Nxjia45@16ENPg|(owOe z=noM4GY88odVbrBSSi5GJRA3G`~z?Pv4Lkk|5%;(q%4azInqMQJ02QcEh?EEA}z`! zoC@v!6WQ9W&ks60mb4m!i6Xj^JELn#_7e{)RoB)mRzB8#c|B$7S4}Sc@#dv#k5~p% z)6oN-78SuEY>sz)vO%h1r+qq(x5&6kn@x4G^t{yB z(7RJC@sO2CRS6~XRs*in_H8Bh188>AUN zQ3fs=rw43kWS1ia1pfA=-y>s+FtN2sIMd=^pIXEStLQ7I_KR73YarwK;DPF9G0)}J zZ`E1;`jxs)m7O77(TS(&Uf*Z&V3MWqAgAT|S)+sNphHI>LiyG@;Y9BBg2g$*Cg;W= zng5uz=wd%n-f1=N9>qzm*GtR(d1*9n^SKA~9#wWp&=RQTY#I@^X_;-Zaa`}-`lICV zd*wYwit*jqs@gpw*B;%o4^mfvc|1w@D-V^St*!VAF9|gFt4JG0D~s7uB@A-eC)S-a z&dGf~EV53f37>e8ao1#~FkOd4?YmR>w2yTZ3wNhQcd)>;9?(vto=q03&`nghzsI5! zm$K%!B3IAFUXy;HPBZ@A7tQ_#B`}KQH=={!0YFv#8QYu0zbuGZ;6F|!PB#4PLaB50 zi+4Nig!M8saO{0GQY^Arr_F9;Qe7}w$d^9Qj`z*a!YlWs;D(XZL{4Sb&se~Z=+z4VPnn3ORlh55NHP-u`g&YlQtFSrN2`Ud-nh2UlHbs*w z*jY|Z7(FRXgS>?*%c&5BkqOr7+BSu0$q3k9hYZ@-M>+9tPIGBmy*oWI>Z>Ia1#MbJ zG!0cYx?1RUD^c*gC#g?3(k?qlH;EbW8_&8Cs=WuUts1bl=VW-u-Gm~Ni=WvmJP-7l z$vtpri2%(8X2?dsuv4mICp0?>JTq$M3OL^?PF@bPA|6}}jG<5!)Eo;~L>!eD^ zO)b@DcgBoX*0&~4RmRrVRlgl*P4CzMNruY{)))GX3E<(|e8=$T{E%0rV$T8xf@Zin zrtF$#M#S562$#VsK0b1OXY~VU5+18j7q zTtCR@jeh3S%WlYyQc{Io*u8-4@vIL{U@X%yi7m1`j9ynB zI8jMd+aX&dUSvJ+N|9ZZ@V+&&PVR>_xK@+W8hM6bQhHYqWDD6vAd2RTJSxe;!K-j9 z$V(_T)C{}0Nsw$GoAYi2vuCpBHO$YENyxY{dawd@qQlXxxE0LRu7(!`hKGHco?^WaGLR2=q~)l-z)bmx~-&^=LGNmun=33*MDy!d7h|8lPsJN_MPv)m@uPrk$scPj6->zYVKCKPS34 zz81L5FKu30V~bByU3R9}%)0hSaGfmZv>7hhEP9nGi=~6U zXONoFRoP{tJl%z>WgLBpK?`s7izp3^hR3pGV&<`%yC5i@w0lpi-HbGrj8(||ThW{@ zofKUq&xmG#a%a(7D%Jgl8Tc8EdgF3_@y=bK+GtvBn3z&%8~8)K)Utk|&s#Al$J=&2 zUvLgLr5a$8yn$))WH$Wp_;5kBM+jYrn#uJhTGN(bQWyi-6x?08b781A%0LGykKG74tjF+d~2NRtw!I+VZK{Xgz@? znOQf^3#00f2l&%3(e!|>3RpGv&rVq+1|=Dzdft7Iz$xUem2+7-d*&iwZ|*U z=OVwVGI@JGHa^EXUGCNC=In{kM5(o++9E4N*!9ECe_jYdLH>}XL{zo(c=9#5|44_0R?p4YzifM*?uf7Y+h618x>Ty@)cL4+|@uDhZl z>t&Z^Sb0`=^Oa07dA8m)+-4zBml3Kxvil-JAb;xiCE9Z|A;VGybcTzRo)Kz}9$lH5 zsZ#5LE2}G?fkl>#&4F}cXzXU?^RNbuz4!PJ8%k@hj@L2|jy}Dh|71#CzK&fF^ch!~ zu-{{?zF3;|&}3|ofK&#Y;4AM}l78(l7=^_Jbd*OnGzgaZ$`FXrv`F0g>!1MVE`vmd z)Mq8iRAUb0;zcB~$hvBims|PN>R?6WZFd@rxyATqj&UTbK7Xv?rWq?S zQZC32r_FY3RT@yxYi)Hr3vpVUhP_)L@>ok(By=&Nfz#UUec6_u-iW5YNs6Qv9|u8o z$jpqsb={inm503HFBYGw?&+GC*7dISRw1?%XMP`fRh!Wc-6xuD-J^v&ZziR%cAZ9^ z#9EGSjqFUojrN@nCyBZbJ(ho*%RO2B{%WxZMEgswrVqg@ zJ$xzuv~z;&Pn^FOj{p#9kK26x@S}1>V{%%P3*-@oxCSqB?QTz3kxsd(^X|hqsVQy{^1UilRR?=ejpr#oI`kkRj?lL7=t5Hir3J2L;aRezo_3ja#6sMCkR z45C0eoRy!M0_0Wjmf4I-9h$bL8;pVW^Y0@{GYz8Clk0aC@k0E+FBbJ{ZoCKR%YBtf z%a||v$1v`F`IhN(0~UeTSfw8xuh}3)tGdR!yBoNtKE!@Vn4}4%A_-(36?z=P;nK(>18v!lVWoPl|~;SX+8f{8#CY6yg*JU<9o)0vA|gKjNU(gBpj zNB+p*b*I*(ZHpcGB_q?2eX<6#g$I}YH9s}Vn)>l>Fg#{oL;AXfBk*KLT>$skJHzbR-`Gu zp3d(Q3C&)0g?qI!*bDb(otQyV;9%|InJ!QkQ+>*aX-H<*X~JWJFN!N)Wu>Up1Fc>S zl@-0-yi6(*mo8oHO;-hqTss$2QP7f}@|=uEj-@vbr{4OkVGwyH!$aL*|lh`e|l+NO*fsmEZ3Y+{pqv8&tlT|F@D zH#2rK;j+W!UYf^q`#}%nLg&l#RJ%F``rV1v#cY=c(xhsZ0q9nP{v8-xQhj7K2k8sC z*^P`99;kfHr?Rj0!_yDVxX_Gmtl|ELo;hejQf1G#bG!cT&qPxwPQd(HI{4!IVvS{o zrHL9%?JeO|XB-q-cuP^$!FM5S*krcjjNsM>C-}!*d7Pbi;>zxBFEXC)5NcRl2$lB_ zzH9;aH*vB~x$T?9cBdk(udV_A{^qj#B39n_km=xM3)OF|E~xBllU^=5@jngC9!kr( zNYJ9q-ieb9IId0S_11*Od7`g&lODxSk!9o&9rkx$HF2n%>L|r(p#5oA(${M+0q@vq z=w=B@dNBKB{YFBOeh)*-I39G!mson&ho(G%zk$o=c3VfU)+%EIkm`j`Ys&nT$+pBu^)uBn%f>xwxHQJbdf0skbtkofq zczHIKbgcB~knzzJ5wzOQkd>1kRvc)K_KX5IUCW%CBv9d%$?^wTZxybAn{k4Jk#C$9 z-%~6!vLSjC@E5iXfcYl6eg>vfwOJvzW@Cwp>3ALusdu{Hkn)t3Wz3rg=RT>K*(zIf zm!H_w%Gp9d98A*meS=A{GFqn~GdvuFHFB@z=ddLgDUtfe8{FikT7=A!{75s$jR-29rGK6Qa)sxXo8@15XBx!a-)d3R zTFx`vHa6z*I-%VuuRbV!WI+TxE?F~l&uroIH*-&Xy}Iqv9*Vw_v+!V&-V|D12}<@g z__-2~_}+&K{+9RN-bcLe1)QbTGGJB|vyFQQMs}{w#yo-jciC22h8`s*8Ok6NYYqT4 zGY8tdv7PzQfB{x$kvZ~1Ew34oXMXBRLnU%lfUk}xEMQYbIb5R^BDkP_9CZ2SRV*2S zJj>9@#j%YNr2BbjrUmQRqZ)zm`0v92-HCG&LshJ><+^2wgzEEbcl&OC!C>p9i!EQ^ z7btz_Qf-dW*le)9ryTm$z8m~z|8n;sCtDkDsMnra(0ZjwE8KStiz6_Ge`iL(A{tEP zuY7Sy9fL7qqg>Ac>)0>JK6`kV<>{EdgaA1p+t78({RClj3N;&t?az3e^^dJDLw zsPZ<9N=#Kx?Y+HL6cg#fo7xE4amcU;X=+)r<@kS|#(S|?)Y@w91nfDO{;sGJ*s4UR z@|{1dCDN1vWleQ_@UY{zKA=N@FaXJK)%GtJ;EQs__&cP(=8yLw0w~&@VU@s7EaIcB znMrz_F-lI^2Ln~BL#BUKzG%pSf?UMEYioaeHpt)~#d3-Q-jn16C3+jk56I(l2MT@2 z8s+cJbuA=KZW!YmLd+MBfv zQZoKs48Z#*aD`-@k&6xjX4!!MKzGOUVB^Cn+h!ROI)vcvtqrxYPD~_fqBbe7u9%t{}|81!MtR#8#uVqO7+cN5&E0n**6pD9O6vvZsdiS@OWd9Kp z!T(EQ+Tc>-mj_y;(4<93VVUAs-7K59{Z7k0S+H2_oEQSzlx~pz;85Z zGQ)*M)mvDACn2!PUvKp5C#Qbyv98`u_{}e+=Ni3il~r7dLSwu4sTsZ^=OCkZxdDN-;YjBudBF zO_ot5gBXK$|RwyL{^%efq&4zJd_9&#c#}5gBlo>8=sDMu}mq z6=_vbbhOyXyX?pBHGrER1aaP(LQZlR#B9s}GdD4oPCj^V;$8Uab>%Z6}K>ZmY3-A-)aXXpbl*AmY^cch+mCH$E z_z*%HDHy=E^J^6*l^}vpyYGc&?cra5>q@S>v%h)UH5XSvvlXRxELw#Lle(vQX%arh zv+kSh6EzN>u7Jr2aY9bd3v^%^ueT?);uuvnD$%{DJ<+7{WkPUX9UFUW^N|7FpCzi3 z??xu(C?#GMQj{o7t6`Cxw>dx$uk^~yf*#q7_U`&60>rn~flN+9=#u<%9oHiYZ$hn# z9vc1jvAaQ!3#}n5<7MV|#T41tpm8mHg)5-co^WQ{i}x2(M6%+J(_+gV4kL~P;$XYk zhJvlzFf}T-jJ+ifD8d@AA7X;MLs}W)KB6t+R#RA z;q3U-0}NMSj3D7mB4p7}0Ip8VY`obTS9e_Ne=wxI6l?}bWm*)m%#Z;3p~r<0Fx4HO z1N26n5NayVJ*i{L6lFjOmV-!)L#_9Zfe|N(z(f`qV5>;*4JL@FWQN1I1gw)0;Y<4& z>K`N#Yk}zrt@R1G*CpD8UW)Zv%z+>n_yyj{q^P z>&5=aBz^w~kQYj4XxCX=)vc&_#sJ@$ZJavb3N+e3&H*Y<0GCe*n7ELwnkkMk0tCe@ z&Ko~sf!RRaz?iojV91OEh?rIT@lMmNH_-+`>O9o@JV*F59@MWUuyk=;5Lat`NNw^x zMR-e*CPiUd$r)*eWMsof53os(N*P63~3Ceyxx8WRf`2+7ED7u&x<5G_Z79LHVLblR)wad;OAAzj zrac5&z%U(M3w`kO>71uR<+c`(sy|!qj7}&w8_fIlL$B6JX>mEZ>)hk|!uf7U9(tw6 z&J6!*cMPo@QLf^5f4um<#m$ezHmg1HW!{%AV9bnbT;~uF!n308@_``*?hAe>OGnq> zOz1g$*ZY~p;ZDOzgOwTztoieLOxLwa60rxx+^(*GO7#H9fb$h&7uGgqLvK!i9G1s= z>giqLxP>BR?j}YnHK&J=`N)muFD4OQ4AG(rUYe(qPga~Ktu4^2Q+Fq(TG!-Qdsh{o!pnR7I%6t{ixHTzxeTLo+5BwrtFO!K^GWD^F3Dy#OH_cV!Twu3mog!De zGEcj%Z?NR8U)8>s5z7UbjQ3RK$Lp=c?aA_N@EJXLq;nXs^EEPl?~4j;IwhOgTH47n zQx!{*@ftAtbzp1&G@lB`)lg0OGX0iV)|s%K4TQny0ozpoy~lxVk1FxatnZ5w7z4xt zmQB=1b>q#2oOA>U6?oF0!8XY&rTpyy9^>j}cG`*HIuk7VBbiJ8SMCEaVioAiWJe7@ zrN0R80Z#loVeAG(_4bYS%N-2ZeD4mxJj++WxR21+7STw)=3=!%@W9qE#J&l|5pDPF zF#@i^Y^S*fmTV05i?E!%BTcR=hAG$=K~(YRy#t^`S02KD>hjoP||9+YF68%T!Dds z{=aM{Dh_14g2Y0xxUb0WuL7ZEyxWIJc1PGtD<>lwoFt=n#HZpEf}_`4`NbxkQRF>b ztiSFTQ&|~&rdN^(8VXqW5V;BXmE1SlfkLC8MR_L@pW+LeKvOpeF3F#O@YovdPFE|e zHnlx`D)HloLYlt&h4?p#v{Jvj0g81~ozk3ATBv;x-{yPzfnd<9R&u`~YR$yEKoV@$ zpSQtSWN}zcpe*o>i~6ES4#+H$#(|vn^VRubKED|&&}OTkJOm?v^Lc9fk<%r>^o!B* zpniu?_AjkK${S5A&<@{7ivC^dh*ZFnxgA;zsVwm^w)Qg!m6XzqhKM|-@LRh-gmIpL zJ3wOCX0NybMs#nvcRGewVi>l16QY_z$T=Q9t`!@aCqKKzBI8j_L-=zaH2FQSxE{Wr z$hJ>t$qSR)(+j-F*C_tXq*0V}itM(QgfVALIr?2V)D;p74)YwLy;2He>(091;`@t_ zc9|f^_CJv<&z|^RQ9*sLD?LJ*MBRABhL)H#m&vohGPk(y6uduYPkF`)h0SbVx3D_a z9j6$)F9pz3Gx9F8`{5d4U_-~1ejC_3fz*+QSgXS1_jng4nQmC`ORgid&WeEBhPa@P7FryLqx|l}UYwnd<>FUvpc6i~_L864#a# zx^%YB(QQ$!d@R1J*=~TfreKYoVZeo%S~77K#9MSyHFTdHa5gZbLso@UAqy33vhp>5 zEOjd(&qJ?vU}9!a$u0<{0XB<5&ytsFnyLN}l60!@4 zYxUS8Ni@i|v=Jwf^s@^+zAq8$F5GGO1hou;Nwt@VLuuG=Gx8&{hZf^r zpN1a^`k?xuKsJtnK}xG*G0j`Ci9kE8pLT}Tl@keP==L}W7cjz7l^23`AHFQO9?k}{ zE1cEzjhTM5`<(A*)?r+_$c5i2rf-|ZM7nsuLyO2xZ9w?7wYlaIMOL1nyFd!|Q~mq{ zH1}HCJg^n!!5x&_IBjZE4O7AVr2$=H66pkn?_th<+0R>kwdRvLcIJVtGo%RoXB$rq zcUt7#p4>%#W1e_x&(WpS9$y8(56Nun7hNzOb2eQ<=4=f`3HoFUeCOtz7vs01&8o-v z&iH|x@)oH8%!AP}dh_q~VHbQSk&U_oCf&n|_XL`MFa-*S20#uezcjUCkp!! z)Z#llkD{>FU}n$XkM~KvkfPJs7?^_FK52)fVZ|&0_<*9wx1l+xt)cza4NCdb^L4Jp z=!X%CK@(k4eyHIkA$z0jp0V%V0_|);qMJxH9<(8nK;oVR%a)Oh*9(%DLKzcX9l=#* zeg^l8Pk;eKuHv_gx_iDdFJGau(*p=DJ^|{Z6?8ucb*1o*0X>YPDOLE-dbn5$XUqFl zu;yQhDUr^PR5Gw$AWS6e87V2D=~&>vehbtKjH=8?@(nNvr1siKPcixcGn%a?8iMpB z_sCXn!1tnBVk^iwit8&p#`fd;@Q#$eP`gr@nygLm@dYQbw_Q)vS=a-MF6Zt1cl%=_ zb;NM7qjo{Uad)7VDeaxH@y;gsRt#&?upNTfO+0jbSQYZcRt=zIiD`r_-06%4`i64| zNu+l;OT6_O4Gq1`56F~W>=k7;R92pCS<7o4N2kStiv;{Lg^o8-R{sjz-y?b5smz5X zfnJy$2r?G4KI~OO45%WSrU~vdNNiF=N$dF*QtmI}xa$aqgZ3}g1_WK6*y;$(B9)_5 zUwT5YZ7OKkfu^;XL41P&}iW@**x(m&Gp0p)mL^+o=<%fr^2? zibp?+*zm;)#?GBBM{~Xhnn010QUO<>Qp1DWfm0>+Mc8qOECsyBMoK(6ET;XjL#;y| z8@hIlK1g_H(zjAH(GIEfjD8sA51?xJoba!1t6d)atUeqT(d!sr)eqZ zyp<`^@|_Y@4Te642C;$XQC2Xe3y9gh7;r!> zsG}cy*aND!_SjTDxIH9P7ybJ6Kjp(=H{f2mZhE+hx)&9QO5SxG7NC)eGE{LEmYnXU zvCm8*h^?@Ai3Okf&;De`2K!S=e_}}2iQZ!cBe*(GZD=W<_p+s&vf(MBZKVCej-QBHa9_Kv1| z3!tsLvpQ@Gaxdm^t1E`4XPU4D3ke1YlP>Sm*=&bQcaQ9i>AQM4%soRcB$z9_*n?(l zK}Irzkg7IT#!(a;JQpE?S%`V>R~o(+rE`RlA$+pY<|ck!3`L;8~>Ng+<4kv zSWkc30mw%cCQHtvTpmK1*%~O3yw40E_+I8!zWY=Ll~mR;UhRGiq2JfiTZdX`h(c%- zDW6djunEa6GeKYr{qNMA+`IxQJGdb%xYR@_g_L4TB_nYRDsk(A+cxaL&o}Q8@!0 zhvV975T;r@e!Vx*?nyY+>iJ%8%>^OY-2^C4mRD#82#Kq%pDO3-HZs#OkZw1_&K!O{g9PlsIJ@`%qp7h(r{#7k-6W*Nn;&$Af zd;$c3BYmonB9KKbLFwzr{$gGB#fC;`9;9jkEqoUxkX+i$2T(&aL`JOG?up^}OZl<- zX|=Wx$HmD;^O0w({W#*kers95564&!#kK?&ufz9cV$Z@}QbBrgC#mE}p`}|^xe@Q@UyZJuhY2XB6^NpWf^BxHn&$^|TTZI`Egw@DN9kp!$agyxLq0$*#dY{9W`PrSc_!`*E*As87Q;79Mq8W|Bx*K(xW{* z{{&p@dtwGb>v^o&Rve!EhAL$I;W0V~@f!N}526FeQ5m8hm+_q~-aW}w_nw#uIOw5iz@uz`rFa=$GW|;=QPEH zq*09=v90R(q1&9*I`gs*hn~Wy3z5bdgx3q!4!}kqtG`_c05gvsX@c4jUhrKi%ige{ z_g5k5e^q7{{yZmAvHrZY@u9djkd{}#OA(oWU;+2NSMiH50^Ah`sdv8$b*eqAC&81` zcdBt%RW^KI+Ffv1w<+YjA;oSOvX~V%6Nfrt$(kV({rTqlGC#Ro0Ea>Z4$ioDD?di~ zJDqH_QRN_C6iXIEN}BGN&+jLc(uo*S)+-ZZi`xt45qoZXsQz}AC$(Y?=ZEV|gWviR zx$@sap5f_Nz7@wfl{%B2h%gNK^URM7B!V_$>05Tq!!u-X2#s_G&w(349(xo3(LMpN zET<7sn|l5j^qHe52Pl#Fr7#C2_PI;@l%r@M_tnXUzeAw=kBu5yzM~gZx)HM{fVq-P zgPJSdN7>(40tq@i(8MutSrivo!m38Htyy>u)UpVFCfGuBLlMv(9}gHzmra)K+=`2p;Q5XPrO_ zJf-acZhA=kI=E2;WKOk|nMEgYK^k-|=NGYugm4;PvU;PE#Sb&)sD@#P{rP4O7HS8q3Ug%66;%_hB zXvR7}#|acUkpa?Gn`D7FWH$Z1Cc*p4gp^PrbfK$C0A^?uT%OsqekXWL5zz3F@sAiL zyA=0&m&ED9U8DyQT-t3KkN;@#X$g?IGsWi!J*p%T@{ULR1? z-lD{xqtUK-_dQS(^*e~!_PhXboA-OFQjXlipCl5vHyJO^Mn6!zGX@D!v=c4ZbR&NG z7j;-TGN;TtVQP~N!^dQYy~BlatlP0eP_93Aj_K~{8VU>8Exa$oL7$tqQ!D|+XHbiN zlyBZY2znZipMuxP@5aOfilhJRE&WYE{_PG`swD|7lAQoJ1_qkc{W~hppRoSa?|*`d z_LuL7puO=-#3=!xOZv;~kos6$bdQM=LnR?RmlPzU0MHzoDB+U%XU}U$3fzvBmGu8% z4|yy=ba*dZ4L<_FgLs&g+hjvLcZexIkW_CZV-rAIEvthth+ie~SgW3ShRrWlBJaxI zzX3?%Iu7mlzkHlz`|AzzpKOE@)pE=O~ipD-B zfQnuj_(vl0Ls(7 z+DV?dLa?0!Y^VFyY3TjVqy9nToV3`pH4CwC8E5H#b67wyobE<&Qaj*E>shav^w07? z<V6agu;VPrqY8mwJf#Y1QAQ z1qe?Z`k2L~JBsI+k*pU>4C0<_8b>ViM9CmUCObU!xy6?Vii=tT1}$}T0#?@QdHWF6 z{E?TOB*q_#&mLJm7mWtl{98@|T3C_pa|P_*lT>L>u3yXh_^9526YjaEV&>Npr-7uQ zk#&1|-4B`7{{_vzg!}~n8se+`cNTl?{O~+x&}1O)_Ec9YzBhrjr-+4Y#jg)Xw%P2} zPy61J3H;%DZy;WI#t5P&8Jr!B^r>a{RHrd{)cxbrd7s$9=*MwRo4W}pT$$W6j{itxsvUh5AH9RB4WO{#hWk7KE2t&z;5GpBbwsC&3d*txSt58WO+RO7K zBJ*{;^p1Rt1nly}5`eqH%?1Avv;L1epL%g-1iX|z%*_A(W(w8@cdW^>B$pp4!?wx~ zcG|S`YhILRr;OlJ0dt6HZ#)SvXAiHrsc-^uCeWU?c##GPq|mRs&75C*Rj`yl|Hu6T z5(Y=^rX4|-M3p5(>UPpfGtdVAUyg#mf?Q$oW*K?Pj$^7R*01}P2 zAAHnvnP)TUZ-4MeM2E3a_x79)@C^BU+oxNj62HbT9++V&3}J2^Hs0RotPLhJPmdQX z8(%E;(AG?oXu$k~vijwUU#4CK%}nj{{l*9wRmPf}G8dVqmXD!&(wyJuR$-f2cW!Jo(>l5`S<{d=-XuQ3b zJ6}*(@q2=MK=@XIb(mwf`%|=8*Xk}?*EK1Z-0Y7+tR%HqZQ8UW&^r9GU$Ur?;GouLI^6N@b zooYiX%E_z4ucq|_34f8+ifPw(+}p38yDeHO5tlySf3CR+RX?|y)Xy8Y*>JgTZV=2; zOFFkwn}*g;waql$=;@e-HKRL{4N9$rP<&<_Z<&} z>81E2QDiZI@7kyA7Z<0q(oD_bu4aD-)o-C_Xc|95Y$T+suG;!jzNjuwYSyT7&O2H2 z`NC?UPuCp1>!pV6)CaRQwkC%$wyU7gs&Q-!da`#4Y|-1JjKA>BEeo?5Purfr(~vHJ zi`D_J=OA)y6Csy%3$Fl4lG*5N1y#r6`o%HO#d;$f%abCsZz6ttwcZymqi3!H#OcZZpl|smp1q-ZbWy-m|T|I$Ytv-Fx6X&oMZd_(OBG7&C(j^gQl7Yus(DRp4{MA;)sjTVp4dCo!-&lUm zhkPL(@U}#aJ?#PqwQm0eA=I~FKb*iyx(H%lyL zspteCkf%En%yMh}afBG_U5|Ry2 z36!eVf8HnTr|d(i^#yJ4p2{sI`*OhO1k<}ESn1a(`#cY`9t#7`4h*KLR)|ewQs-!#(QuwDGoGkUN3x3(Iv~C2xmwHPy_x5|x&#wH6zcLNr~f z!@fKrcdvz{Gch);x30L6(Rcnr#{=u}kEz$OJl>C!5J~4;>MTf8?iBk4u+g&AC9OOO zZkqYu4>~j(V)z z?)W12_|jzAD--P!!!*2Y#i%DD^l$PL;(96{bv2#R{Y1mdPw=8HhW%1@M}+~NJe%}8HCXHFIh;d2PM645e#*Pv zl;f?r*epH^!|)0s7qNDm9B0vZ$*5XP)7mj;xBRP7M^MYcZD|qap5`${N5izweQ8`{ z+ETw7yx^K_y(Y(x1kN;%c#?8;X*}1j49ZHzjqHq>zl^Hg$*COytH)tUi4|ye7Gjq zZ zZ<6RGFI>1JJWdj-q;rfM#V6oA$N5O0aWoycaL_vlUr}g%fRB7SSMfH{O7JFy>8Mx* zcm;&GB1$FG7?tEPj6fuy)gyYGV={11(g$sXJ5cC+D9-r7@)0liG`Ggd_A=RuU5G+s zN1(`oKmF)=V5@U4agPiJT=pjW_Sj_9^lGo#%{`&HLTH&Q z_Pjjf$Hk^s5a%6&!z<^+%wpW|-nqIE*)qX%U9lg|)#!=0!@qz!1MowD`UsS-(#$tvjn;fAKmjy#zL-!j9ZPC}gPKKd$q=fz z)yYIhrM8#1^#*YfhYS3rN)u=c;%LS00^AG<20fkh>jYOXFlCnmpNq0?B6snA}uMT2Sw_qkA7 za3+O{pAGmA#vPew>!)aUea$+Ym3WlIMxU}y6)(s0rKA<2KRsGp;LsqU>i+c7nDOcE zt<7axUy6YKsz4BmMOA02VzrC%pBX`xQST1s!M= zoOfd;y=X}Xq!-5eYIo@wZwonH3_|oaQ$i)7_@ba*(EfA2C}{8=-Bv?=<5QqMPqJ?p zboq3&-T8Fn6J)?fp?c)9efeX4p4daarIkZt?#jN%Qaz)h=bi;ogXHS@*6tzdXVlZk zlAkzcgQUBfRPyuIecsU3>*|Z13VRxpttBi*QO1^VoFsOYRgU04&=wC;>_;N)R#sPf zjYA^6iq~fYU;n*RJ9wYDHOfEYe!c1)^e*cDLv93be8L50oQZUfGdQkcqVB0*i&I}r z>xJLmH3gn@xn|7A3Z;e-Km%yI5dYG?8%4e!OVy=!2R2z>SpS=8*A0yUx32^htc7bE z!9@qCp21a+^CDY?i`4@bj^UfbPaBbg;*UnGDPPAwr6j@Yf;7gc+|~ksSjs%XDY3xFKzB`Vti9svZMs2BN9^p8{?JIy%mBgfV_Q6 zgPs|vtxjc0iR31j34oBQI_#Zq#pcLQq`RYPmFVpFt(LK@G|Gf!Wp)ZmjgQrp*2lf~{=wae>>imLf#AUD;D_w{v1yb z!_uGGzR>7h)-$jOI%7-_`K@pVcFry;oOa0vrdZo&Pw{`G5 zC?uzb@~KT_dd*5mZ^C^dsI!Q!?V{>&Lxr1mz9Cl)q9~K`cL}dJtw^Z+hho2#p`&M+ zY7AFjuIyGn9Y_y+Vq@M5x|>?FGIIs||h)%b~?t4x;qt!?Ww371W^XOzViU z^YYcoB5O51-N5#{Za#o(q5M#)xiR`w846B^)USu6C`;#CmId0VM;cx)ZcJAUS#8{i zqf<-i7uIo!Z#F^twgic)}=9U2-#BptLJ=oKnY^jj{#)me-m8bho4<6k1;7f zv;EAMWP`n|+>f<+TGp2&&}?nYO>p|5iU)uAAzJ&IISubZn(x(OtsbC-vrdy*@wZNJ zebWH+go{JhN1kcjV)dTBv0>~AE$GbSXu-!^y}SZ_p5MKBijN|TxKS`$$QXl|Z8>&F zO|mUt`)LCDg9xv<&E3JAty|!kF#JQDkl!Q_(}x{FV`CWNEHd(!p@!=5?fLo>QsE=# zM4=14y7c!@nMe)GmEu@$)&Fu zjOSuwBroK0(f^EEXCwxw+YoOgkraRfTk_4p&ruL6hs7}?*2q8nDt?pq8B!nO<^gM| zBlMkj+CcERn^#-w(typZ^!+`@dd1#NIqpquE-pIBxj{DD>{Ad?YezCE?jLXAB#b53 z139$mKpuJUuZtXuI|o%p)q^Qhj5xYNEk$f8uVZfYx+SKc8f>3+$e)t%dHIi8{Tj8l z5LcWnbx%Pj|E&P5ydu2ZuS%L1u#M#_2!t%@6Ii|I2r*0(M$LC)pJ2(4e71`7LPoFS z-V8i@)TMjO!`hb>zCPG}S6sU!@hw(9vtPbkyQgI#6i)y3oV>%4!}44`9!`Qh&vIQ` zq1M9HBEV}it8_hkCvkmnZgX{ThCA8l7;fS#RDI{ECHli9Uvhtphg1K56w&T-A;|Kd z@#}|>|HQ9TD8PzRu)K{P#(M4cW2W9+h1J(ryrbdt2RJ~hy}iQkNk#qTo`+ew5;tQL z`ry>eLlNX?tYZis7i*JNPs}AhQDpw}1#k88w#KA{+Na$0uggftCRZb|7XFuMUX-qX z7Xt}Y+=kz3E92-LTxuNP*{45Vu~?p_I9bZsQo;l_gw03u0(2a^#(4j`Pu^_WBELRe zGDhWcfMEcde3QjZpoCXDrP&(_n%zfK&ig4!d0uMWUs*z$l8(YN(ij%=&8G)m#Z@p) zwPN_o$7>&}-3<3zYLNK<7Is*isSa-Pj0PSCO4Z`+PrNGyrO9V?wXox1P>x4OvW;_grLENZFN2BUu zt0&KkusG%ZGuUf$uP2M1So5}P=`XaewzK9QWF2G77MA=Jaed|WB48?GZz#DYvw|vh zj$Nu|EL=1OloXU}v{QSDZc^sKr6hm7KGw_NYQEWAIGXZh>78 z)fZog6b2zn?g0`_r!6SBn9Lpo*|)&~h0i!>utF|vqqOpv#llMzQD2GbS!Xhb35?d% zY1quLXhOE<%WPJuYrOXT-PT;NfDdzHf)A(M!}=M0a@HJBVk<)cKct8n@0SOndaBO` z^cJ59pisju9_54HxHoXI3BiLG-+H|{=CJk4UgcT#b<=mToidd#oA^ARN;^MF}&188i%VMCO z$0@GoL1cO&F`JI8cgR8=^|tq>X>{{W$^P}HLB5|HAmOjx|5Fq=yW3IoBr*Z8>^p1s zY)oxKdq`+{He;b(rGp}BM$RX<4hF>Z$`T8ePte*e3c~|M%5eTw5-5G~H`WNp3qK2s z6ts%_@hWzF0<&lH&d!n^=Fyo%(yk#qtO9);>#>^l<7~^&1Nb$yIJp21MLcSV2p(Bg zHq`FtQ}Lwkc_=Co6nQl(#^ek=Lh&H6DxO;ddM*4;+fc z1DD|?rW+B12~FBZ8+d7F7u^xE{#dr5FM`4ocsi1`f>dX0SKjv;S`D;VGCrmSxG`Tx zucv_A-A;Ms)lM(MC8T2J%{S%zWpOSpm)3XXBjqPNYDT=_+1Zz}=y;!$U&)fjy{`bj&eBuA{E+@WE-oi?`&M*! zLu+tncDCI9WM^mAHMLWw0T`|K*{&rga_xG}pYG8pr|S~3D33G8=!BQr*;~p(y{crS z>!)?1$VDhm88u*#OUk1tx9kL>)4tEeKW*Tju`U#`XK-4NZ3n!Nvr-%q1vK0{?2e=S zPTK@+oj;y-z8>3sLQ{hue^h62%$dk%QIf3Mf;^M>%>B7mvRj2Vy$@9+UW9LarDt@n zvBJ}8iKVky`F+COtL`2Fx5+Egi*L$3bSiRKv?7FTI!X|;5#oJ9!d3yBn6RW8uSI5B zqbDQk^E8j%F^|_0 z5f{gB=u@>VC+&}xGkM0z0g`^9#dA}Sz(6wr$v@I~d;*D@+4T5|I|Z=mR(f<4;lkM$ z)of@Ij))7jisRk!RLt!!dvAx`Y@C zxXg-ih())5*HRyEad_-5<0FKF_{(d0meVzW58?jg4P~?a06CGQj#i}Mmy79e%ITo+?g3!d^_fa%baL1YbT{;a`l6mQ<+I4 zyyRIfydZK0U~%%R88FaS8D%}ZQ66W^k?EDUmE9h8xBgI9r=>;j^ZdZux)h1HC^Q5) z99fgYhtK&tff6fP3?|3GEQ80-z(CU$;&mJTEuX^Oz+F|YMe%A)pdUZy`3_nMXPH2=>mPIUsy4{>(<0#k40g^qw+VIgC99;Y*+Ko zw2RFx6M;j9N`a{Bl4bUqvuW=Xq0RE5RwegnV~Arqw)3>6&j#AlmHk-7HaJXAyLY%| z#B}EO>Xbm-2VmS?1~}Ttf*U)Hq(C9~3lZ<}e_QDe!vNPlCG?Euh1d3YZH$#}rBtVf z|5#!j5^%wk*=^3qdVBptpf?J*mR<~s_H64^SICJ%+u$?j;azHkomizUFv0yumj@_t z7l`R%*sqttUk)?zq~tUTejxX8%dI2oOHy@y+83K#@%kuH&Kk9`c;<>Jt+H}gyRe)U zh7q4m?pT+dt|-WJe8b!)EM13`V;j*0%Cuz%8t~c9QqaLoQQ|$}JXm??Gn>uAmSvr> zmZw3Xjh{K>cxV}|$4j>&kgAOWE}UZG7+LAs<}Y`A#cF(Kzc1Ujgh9pwr7zC2^Ishe z6zk>t^X}ldI#t`w59`(+`>`eSEgF*oC>eojP#UXJ~5UZ+ZHio1y6#jDS;el&DjWxDMej~~*Xy>_^c zuoiU~*mOV0AEN5%!)k~4MaCH6ADn#vBR1@dLGvQF@CrBfF$YaH>m&(H#;5+Lej7ld z#C>qdL^bFfqmah^FY`(}8Or24g8*xa*gwrHpGybV&%O~!ZPfmd{$jSuVe}h}(|i=7 zRk6uMn=Fio>m*L74qjz4QjAJ#XrZcbu^rLubR^^UE+tTJp~0K^^4XmsHcGbA?Qs2| zBy2Fy1hwgG*Puv=>YiMd`TU;=4Do-2x|vCHg1=TIr}JpFl?ftx2d7dLc1Wk@up^GJJHiw~9}zoi$Zh?ZVYlPvC@v{>_xr@VTL$woW@& zzs}E}5nrCqGw8n^Jn1(bTjJ+*Z0B*8M{9f88sEhI22C$C1x3!_DXP+&ym&}x61Yz8 z40_@QFC(IsAMV&)haHW@1Sl7KiL*E%8=r^yG}vEjKgNK2dsfLEPOWAwT5e}KrkyTD zh=vv0Jdw5MB_%1P#7(Glzbq0MpqBE6Dt}u-u)R97RW8h|@_!=a6C9_+8a+!31uyNb zX3e{(z9b&;ac>u0jN76m1aD{5J2~6>@ICpk3IcH44r`W(+~B|LTBiF72$NDQMxyZ$ zVTfZn{F_n_Q>V&}D|<00$)WNCtv;2^<0CA*16|_hkBR=Bs}TKt&?bO`7G0yd_M?v> z_zp5=vm8ZONI8XR4&&vFo@!RNSQL+u31`X2!jpS$4x>f0ykKTe%~Jibhg{E^D0*i1 zUXqHM1(|f8_^aFmT#Cd;_3M8IHs8ILoYo!m`^?X%ECuF0M6#bgj z7N<8{@RoGj-wm-?J~dq4<)~nOS`R!XBS)lt+u*c1r-C1-><*`wR?)Q!td6B|^r{R^ zFLy#>GbJKx@5(EOE#wC(;AO>(s;|tX6mN8aN~A9ASs_f z^7%bj1;M68HEa<{m^&kCZxtxWD!{Yt7>S?o`n`A0n$}8QE7e3KHhNg`j!Z~( z_d+e+i@SOP%4m7SL>*trA4(N+^Wwt`WAIs`jr_AM4~`{TR|Kz=1(Q8o2SCE@;(rD%%eY19_H5P{FG7jJ9dov-J3Lo9p znGMsfndrT!Czpw3Qh!#n{ko`|YFNuI0xJI{D1}s~K3fjQXxB)R1=}U;tvHg9>qT6_ z>}6#H8IL0N?^&9VlU+4d{{*?=&eV^LLGdI;WlhgB_5i|6rqpK5{mHQT;UZ!B6FgnY zR>FggO*vJ}QBL{MOIJ&#a5DuokgodWaOpm@RhQTSG2F2*p6DkjeeZ_wR@xGKV@WT) z#$6qE>KCuFFF++mtZw!CXgYr?zD()V#r7oDs^`;)p1&IOMA!XT!MSH;WJi0^Win+i zTxbCq(Le_G`P?$8Z2sMCt;bsa1r+8c-m@}55;twwT3h0I`(3$M%QE|b4%1g5I^2L6 zs==Jh(jZs`Y1=P6?g38v-qJuXBXuF7r_+!=D_rMD1sc28;hkx3^4UbDGmLoPBjX)% z)f68=#NN&@sz7(~kQY|lT zY%&v%HGZv<{U`a3j+{oo1JYU4&lqdFISa`VU8oJAthi1a&?2 zF?b=Ke4pCjO1q6LvC=qdJ5~xkWIeSNR`FlUJbNE@pBS;qo;tmC&$m|Degsb;Oq^Xp zY&2h+EQJKns(vmmnsvMBbM1L$SLr3>P;%vT92SvM^BSFiX|?OE`MJlY|L@D4uLHK4 zuoDU)Z3{-;0`=|^FTJYeZ$vZ=v_H4{Gnk<&+5Hf_UigbwqZHrTtf4H>K_6G;I`Ja) z>X;*Pfq~quP+W}OCANWChpPtZ4>O=LLaGH@{wS0x6tukCcykcGGI;f~i5d*e84Eq% zb@iot@R+CR%;{A3yQ*)yM#eMIJhLAbF~qB|%bi$4K^%lz=+37`Ftcg)6>^#E#cQoh zu_0yoC8mNV453)vo%%!hRrKq6n&1k1Xjm~Unr1CR=>Cn!bPW9dc<5h}yn6fv-A-JI zfAwC?fk5wWl*ZZ+M0N?`vYKAu9jopC6I2%)o`uKh%*^WWT8OWmC%acn4{pm0_6Fe4 z0v}tOedqq+bM4>Gk>}{H&1x8CI09?=*)}u4Io-*p5cb@5{>_w&=X6ZJN*_GUx0W%> z!Q`V;4~@2PZ71C1 zq)R_F&vzRv<$%%QiqOsd9aQ%krEtD4_X(j&f7(bQF~gwN)9u#8U@@md9;1q#zB{s%qI2!ICeZWxu1-{bJ#7j z{zs(*lJfV{=fDR~zcS4y&WJ5DrjK!Ft)C2TI<}MHa4}k~oS`Y-{gENRjJ>YaT1cL< zRL3jGb9T6flaaS6x@)Ggj&^dnoBN!_QiC3zd)xNjnb>y>zEk6|`bqbCvCL%PqJD&S zq$BzxaMpgcF0w1z3ZD}6;cA1gJJvr{`D``HVHXoIQNbeEKwBtyI_w*;r};uFK%dW7 zUHXmA(;&Iej0*gdZaMPSJ0X2iJ#uc&EndDryG(a64dNP_O!gw`r4)SeFfp zQ$V&p57VrQ%p-swd#}|HtG%A?Tpi_w_5#!P3RLt(Tzi{Hlh9&?tT${pv_6`3`Zp&C zHq-SrX8|7uW`2Cpt56!zB`9pJq;^d?M@ozKK!lFhb`FAC916WydvB;*ZO$ZfGue|z zQfjnVRGC@!#ap7?tsz~Pba%2!sWXEcqaV_ZYeuwW6QjHu&wdjzaN4^){S`DM#!NP3 z%I{O;PP}gWv6LbEBz8DHFQ2dCvvc{b-xGHVG568V;#0U+>B8uGr7JSI9@JP_rG=PlG?tsZq$EV8h!0j@g7^K;6(aVR&Dm(#12 z56kS2m9K?NK{%Tc)lv7r*UsC8@(9PLE`KIpmtvhoRhL4=7%fD4uVB&rYmHGp(ebii zkYjq(ZqIo6LqEj&Y73@(Z^9Gl)?q`g+jk4ZQT<4$D&UG zO*HiUA@ypFp)g=@)@vs=%&I>9LztcZL5%OtcWz2Znfi^__Hxr4IV6Zyo6u??e%gzs*&fT@pr(ho8E1xpquk>dB|a>k3sg_#KQc?Od^F!3$T{L z_b>Ab%G&9RmS*6sokq5ovz>=AbgN{3x?&u#EQbqP+W$(9h~x^DSPsUDYpAR0)^nYu zJy(0&!|TzQk8A{Y0>!FCPM0ru(!AX`udY1Ka0&Cd^&YZ(Q7;-3LyKfcVfB-nC(V-& zIrV+UYoj|tjThLCx0*-4WSf;ton>ezB4+Lhml}q9>DQSz4-;6E zHO7ZAMw{PPdRVo#kS#q!>H-=_xnqwv$;?Lmzk%`PVsNd*O~IBAPS2JCY!rnp;sJT; z#k&v()6}z=rx#`lONiz{(J`W;m*6|z1h-<3j2cF=iCju)b>0-Dk++3f=cSqoIWE3j zOwMT?(w`gAzyDbD<}gClk4YJ^8pYnOG2fXY2q50?#vwTLauWpdjtb8s@6DsJ)1e_L zfJ9>tQ|taeOOEdw{CfQ$|&1c03D9Bhe(8Df7JNh(!;zNbwN!}P!@-fzbMY^JCsu>W~@B0@h3i0?m z^}GdB=Sbg$hAnrHw!j-U34b8AQ}EriX9eRqy)d-zp?g zWdMhD#Vu~@Z4;{7jIwV~KF2c9PSPJp70Fuizdec$axWbo&QrlzWxSQmKInJ8-iAX> zhBv6D)O^!X@GXlUh2~@Tx;ai2%zSE1>X0S8zwqPW=Htpxrof!nwse1`42mto;E3!h z5bcM@w(lX?8|>Nu>0ZTMGVS6X{WGg1N2Xxwftl^rM2anJPAWu4Yy#QwfS?oq^NmV} z52YYaQH&h%Ad|+{srg*Z0pTZX;fEWFNb1LVD$mlIV%tFxx{?>`sarYyiNvGr#qYN$ z_?bPu#e8`W2FdV)8|m;*d2UbjFhb7 zz3k38op%YJMD<=!XnaJ0v0!i56dJu8=Xt(_%#g+#UVoPa@8vq(&sMP$z>qWD!!omz&z{Hv_HLX9AY{- zLpoL7-Hx$7>UN>lnb_&pT#gJsSzF1UHncVVX4rc&l^>uN!A!;Z08NVIUQs-Cfutt+ zzobNHclVw~5(*{w;TN34cjvY!`MkoPktaD;T#+!^FMN`h<5xG8RXS)H`Y*}T`V+Bg z$y3N4=^FB zIB@>TeW!iqz$+?YMV|IX{QqL_EW@f?+di)d(jiE9OM`SHEeIk=cZslQq!%EKq;!Lb ziu9tpQ$hr3$wevM4bHW6?`Ot6@AFQ4nwevcV}ICRTYSDf)b|G&<_jj54Do8H%Z zycB0OZ2o2X8}j?={&XVECq7DH0V(y~8}gKe-k+xp^^gVP0POuz+Q6Yx!cWXoX)FAC zdCy#3am?W`k}>)+A^{Cc`{=PmqI`Hl$(xx&=2+^djR*9`oAHlyvrkC!rtg`6z`4)v zXXyS)&MiGDmBWR=^Sp%ok>}c{A>YHiHZ>M+k5~>AEh%-VTui3+TU><$NI|J1mwv+w zssegKjq9GBmJK}gY}O0{>u4u*2?yKsddf`>!AkqpN%PnKQ7hN zYZmA}z8!tZC4W*_)n~&b8tuD-cT2k_!2&oGIUTSkfdPG zMLluBR}mLi7c4Q3zfBnNz4qWNB5Bn8_%ih?pt#X}(yk41u4v(WzISwsu+J34BsPtA zM{*dxWE*dO%bh{<%gM?cx$kw3PnkET*c@J}W!}muPYb0idybDr3Ra7BND>M#hXDJ46CqMappyUFMB)_Z-+efnwVNztxJ(AiKAA(z%6zxo21jeR3B0f{k zI8JHX0$@su#FZ-yzUlQAxnijz#n)Rw_#%+xK6+i_UfPO;Rx1WHU%YDr4snFo`%{Cs zNM8UcK_=Y01LGjeduI$OTlhhO-Tb?xYAoBIT7Er`LloUx!Udzw#SRSIuE>it$hFyW zP&hhI1g3YD-oKp=cOT3%?z;2CaVIpf*EX?n@GE%L_e~bh3c{%!ROK5XAboVVH{!A* z*Gx{zoQoUJ^cGGGK3#U7^-PwbIMU(@%cM*jPl^`5tptV?UI^pGh93DT?-Q|>q!wLl z)?SMu&(jUPk0W+jOO9DB9`Td8Jm2@Muwh%7Ds)$@&*R-_kulgAFUX0Wx99v89$3m_ z?6WoD*@tw|WaQARzk`)55@qb-*e+D~bXV9%2lzZYtX!i~?M^r0$>$Z;&cR;zG(o5p zjLa_=aohBo;lfJCrvx~d7E4UsO|&D#_q6()96+(mV8IS+va5IfY+;81?-g>bR(Pk8 zY3AQ$tq-@|s{Ymj{0c>As{g8R4VuRQpLMD9r#~Hd2Q;iFPkmfkVG9mKa*T+WBfwg!NZFcH-ndk2;OmK%t&_y` zMK{$J{*O;(n}ywEC9aMOI-GIaiIHKcY&Bgu&ewq>vt`Ha!C@9BNW7>Ah5jGXCgmRIKJo88wtYP@AD^<=!rwS=hI`qdwZenye3XroZThmaUYwxN;;l%H>)rn}A z+y9oS|EZi3mC^1za{tF?j*)VEM(eNdYZ}M*R5FP*qR>ji?8842 z!S-Rvq1;f02gNM6r9?w|*y z-k4A_-M%rG^+&j7(KI`Fd@>?k&b@EnFF)rFzNMl{@FEH=( z-5rtfkFph$tg=t0yR`2v zjnrk0m%W(n3X!`simWjKE21R62P;uCVvRMqd%i1Q<80k^x#>wfrC$vg*$9j+9YQ;Z z1uoeOdMqPeK}Y{6ME*(%wbOg35m#fm&@X6e#xH29isk$-?`-AP^iz>%-RhKqBN+?R zgxkt%#ouzxQ`0n(C&-?kRFI!!C-h_~mKaWHv8BnBTY|^6>!lP^KvELIE#NwXM!Ki^ zc#AQ|EUKPW0HVjZ$~cmY^Lq487G+nz^R0V=i74W1vDQ7> zeiWzss^ppWWY+n~XpOP*k(n?*uA`F*sNE!=c)R21LG31LKWt7`YK12kzw4PbfBa6Z z%y}n~e8O*h@8uncCrCmmYHBo~*h#Tt6>akmY19D>ngX}habK1rYr|pO)fIurpnHIr! zG6mHVzUV@e39k!KsYrAPrMd;@BR%owXPVjVAS@B4oNm!3(`4Bm#^x=0*>bi1adWEr zq5f^-dPk#yq#R%}ct^6@uBK+ip7A&eT9a%mahh2j7yNCJ5|DEG3_~>=C-N8&b#mWh z422_h4&y|Ri>s-xPV8W*qXnOa_l>ip8&8;btT8_oTa0mi^ldIsL3!|8r8p!T+3f6m z2{##~B-{{1V+LzU*zZFRqg=r#fLz}6@vB`aF%&7i%cZRw@;GDTDKlWHX*ZiFc1@U& z>PlPO#t&0NGGLQ?-UHM7qxyl@@pnPYUtw?T*&FuXWM5_k*;h?tO7NEtK0ey}({TNS z{&+b~Uq97nC6i1;E4^EZ7hC#*@1>HKr>BXCVpc8CG#ssLlTU3pTkJa4%U$}tbjuE4 z?^Ki9AmXAmAGy-U*zmPe783iS&eKh)v($72FB>sB8aJOd4QQM3judpcytBGk-Iw2SuhL(wp1n{ya5JQ= zIV3tDEOB8Y_A(mz=usBZ`@E|Sj}{_m%L$|jq@gt}M1%4-F`qq3aGzVJxg`%}N2CM9 z31hq6SIo<8q4*bv0~h26zovnqAio}O6<+G(7nn}j{xN{V8Nx3R&@h;s9wE2^#yy0rL8<^ZUksa(+v5G^>(JN{8TH_C)Ol3A3%l zD#VMFg%;|enS+*H?oXr3;7aGvaE1Cux)@|mI(`>N+Sapg!Z&}uNRDPCk32mGcUfOS zMc9n%gkiK7!e4hpwr)Cj<;VF@b1bvE#1B^}iPoEKgI&E>(XeWcMEXEb4u8)+bC@cY zaBAwqg_eS-){Q4W+dJyqGZ(>~!Ak!M^a1Y-bIH_YbYC1lNy=25Temd8?4K@e5qmf7 zSM#)VLbm>x@||BjbkUpTK%L1S^LzHwqB-k+0cFq^p{$X;oWnTF8k@n>;8!`%J902V z)vkkm=f@1!qTn=cd}DxQl|c8tuRm0SxO z#rKYq)hm{j<4P6LnfuLZlkC-UU~Y*Nk*DkZB)i?;*Ro6c3gldt7a1PDcP>7ca50z1 z4|{>w6<)YyzJKlt@4t7&9&lOux4Qy3aXOVWI9D{7j^TUCqKkC(YmV@6`d#l8K+ARr zw&aQ7OQOwDWD`Z zf*j|b@c!M`F{>rconc48GEwll!zlxW*z=J4n3Re)P6`=dEm#6XLpA?UK>3p}{-yiE6#?R}Tb540=orGJ z2s%buUo(QOex5A#u}gdAb{Us8S85@059$8caNzz1S4~yxZKndAsc`kWXFj2cZ1A+t zsbX40gK2bYL)Qc`m9llLBOf3RmZW-0B}UD6fs({w9T_o1OXO2p;E{BC+5GaZSzPtd_&VuEv~R{Oq0JXo}ZZzb4MpWyTz2pTpyKa_Y%cLeGeM>;;|sa7jh zI1YVdY{`kR3ooD~T;T$$5dDEHaUb=VEpTQkZ+vcgZ{}}BxWoWz#ZeH+{(ECTzX=wN zCza%nyh|qQldh3I60u+EK}c635b@JfRN}ae)r|Aw4I%Tc37(hGERrWc6IjXZR|)Kh zSoFNYkNyzN_)pUQE#~9>_b>HE4{oTmG`&e@-qyHZ*r>yJclGK6lOu-aX}5R})_YGQ z;rZu{*yyJ!FI_U{f#J}9&43Z9uvFOJSyU$I(Ei6q10cTK26oDl*QYW@oSE8)OS5Z7 zd>i($IyWC(4vNi4FDg`lg^z}UNVFd$TK=QwiY)Ux(GL6}EI9G+ZV!MuiS|d9y-^Y& zIaMcjygkF}7&X)ABWV3?%T8aPWD5>~a+ebd0o@-t2%s$J9})e-fXW2(uMFco=1;aD zc;TRw?r#Q;>MsV)hjIs`_U?##&CXa1G)RASv-;>C#wks)^1p}7w>b|rolH-+KPNU` z3XCmo)g?inRD20xBsVR;_m3)*KEUH4locF?ir|SwB0`KfDIw;-6y$A%+$lldR+g>xa?seq+R^Yt;g6>$N{9SzD z_|L!*!L#`tI3iRA|6ynkcs3-zr|kgG#(=F$QK2IB&-($vv-#D;^#`5Yf4>NR{~&<< znNZ>x8MgiJu|L04yf+}v`wvq*#7q9KJew-@$QOL+e|mc%cs9RPV)qt-R?HqF;*VqQ zpRs~)uX;dhaxvsS^x*i%HoJ$whI{~n9RYWs+F#8+d(rTqqRZY# z*rM6*z#Qmm9Ny+UqmUf_qFhA(>z_(-BMVKS$iz(iV>bl_+yJg(1;1RyV(?dK{O>U% zT)VcUj#ZYARrp3Dmzc@CUb+4Yue0`_;dKyHoG0xR40tQQ{KF`Fn_tZ|zY0lnx|EgS zD*H=e)P}cUK#V1Br2LQ0KL)}pEnoeYSDI2vAoAM`;J`*#b8AX4_M70a8gF#=^Y4t~ z|H9&A{Ex6W3g|Acr4k4hK4>nx(fsc`TYw8Xx@U@mALfkU1kK=B%*)!(%gPK2`M%9b z)=~JiSo}@U97`}ms7n9g%>xcE;8FXO1z{>j#W?T)>4_@hHGlKpOz+)FfBuoQ=kf2? zkJvZ>6!iH&0|iZZ5E+*mHL<4Nw~S-eEa*irZM?&T!S)GY0x#>$osgm%$11&Fx@Ftk zBC~U3Li<1t%!WzC{uT9Fag5HJlROWz z#5^0d>Mi?|&QnF9r!WMg!f1!GP%!ZF0Ay%yIuh-S1vlPwS08x4RbkZK9>n1VG)) zN^3Qotk7LFO~&`*KkPd_jX@eP^8*H()yQo z?C<|+4F+3lgZ2T!h!p630P`ZhD+@4ClvW2)i@om!0ss8zTY`T-5;MkTtsnLnS3}gR|$tNklB0|Itf}3@j86& zdHy4BWmthaaiu?5bj5!+;+W(IaAuDMI0z+p=d1VX@0C zC3$ri4?SHB4UoyeGr5KMZT>Uy6O%HnHYF8nob17WZ<4C~=0Flh3s zq7ZqZ2)GvPV1>qI2)ke)C$|S<_OMhb#?m=2_ulE5FE(qQ0|R{EC(%1bgsY_ZUY&me zl&fXnB|9B}lg{T~OHgc32il5N{pN+Vz0xH2nQO&lpn7Nz@d8YWPuelL6!TOwGYsy> zknl*KpX_h}DpIuxJ^l}n8z)01N`6FvgbS+HV8XAzmc!wgC*ZuS=mV!NYoXyAD>2lc zgC3~V?KEG&M+wt^OulJHzXXg9&5^vvrG{mu^h249v%%G z``Ek=1~MWg&h~n!m+r1XW}Rq-a&f`-pal-e@u5^+iZNX^b6)-GI+ebfvhLBRV4A&L z8mp@%hFWr)&uzTamof=wCU;`nLy(L#0r>{5(XlI` zQ~5lW^On}W`>boS#G~OxPZx3AdM5YK4~}8x->U7r=Ex*{8+}VN#J#nK>}DI>-Su&6 zB}3(4koHSp!a6T<9qEbUGTxWS=6VJ+8KsS<@CbxuIg4R^<=2%}uU3AM@rOFpZQ<10 zQNYiX8HoAEw?|EjaU=TKb<4k>P>w6ES|kL!(~YC<-gR8*BYc=HP`uEp5kbTOAEitR zOz8lJTJZ$ysozWkSR(t=kW%4d+<|}#;Bb66y---%JQ6^>sy2hUK^4h0deeTX?AgnW zGeB4Q;TjPOHBA+CFcMc-qAK|`dy}*9JO}HLpqura>ZmIv&{c39wWOKB{iNHJA@Tyi zMZWV));JjZz}EbJUSu!#CqFJ-;%+%hgQ?0>=VC=VB>3}l;1GL;pREs71bY%uaL;j5 z!-Z{uN3OjLxOU)04OD)YPP0QK%pTQySY{cV`qhsP4y-8r{aemvojN?a+SnSc+_qdMU7Gbj!f58*rD`RynXb&2s5N77-=S`_+AXRT?IhZ~f>N57 zy&keDi9oq+`#S>qSFcXW0&!2xQvY)H22;uHc;nWSK|xZXD*R!&lQ+_N%tPj0^5+4d z4Vr54Jg`s}ao^q?!$6%Ng|p2hnSI^qd_^EKM-Zf@v+#iLurhzW_@REa?XatC`)x{D zbK4@)AF&ElPWY~6Q-Hz3+r)xW^J2319K3dmt?4SSToUr45N@DhnoJ1f#)>6Cx6~@s zQt27*=oXNDSgRr%aEjqiD9tR81p~h7%(c(tu|#ZVQoaIaU6GtFcTYDL{FyZSH9%|3 zV5HZM<8hb0LIzAzT!R5K7s_(zIt84;SGHnmCPid7*rLA2I<@Kneh_6r83Yn=e@%TDVwUZpA{Jt(9(cLlup`m zY89o-EzDM0#}l|UvT--l3@qt#GouL$1;NaFaGLheF-gMk@$QO&EmbB`FlCPbN&qkD z;DylIvJ`#p7IV&kdjXyv*OzCitvn~R*Z0Sf9OLz?&y@aZae$|LmLLbhA$bQT7*33WNJao7`5U zd&YbAZKfu`O-WDH9-&}C+1 z4Tsn8X1^rsXqa#+kkdYb@8Ch(P$0L_#sPnBG=tiElgH4|Sm!otvsVE%&Tt#!HKqGZ zj+|Xs8wsyTWD6-opm1+Z1=^*Dw82JE9T;IRhMAJSYb*8KWvyWT$XBNn{*v^(<=J4G z*H$&KJok@qjI=~6|4!{Pe{o@JDjw++wBL%fvDp!M<_TXW|x7~ve3>nxy>otsdDQlD z8rW)eAi0fxd@^l=l9V5!FJVj8UXF?mXH%V`_f1@s$4-AZ>CweREr3reLcb_0As`LT zG*$q@7$Fjp2QgaWoNV_8eu1F@^dmlj6BAQpWfJ-a3?1EcBl#M6hapvkU2&!>;ZZiW zkLs9=XK+w~{f4@GC3TsUshn9gP5Uz*fx)&xO);3SI70>nXh$@plup(P>a$!dxsi~! zP2z<(E~*N3i6RZ*v40vMV(*P9Ln^z8p58jS8LD{EdJDI93opDB-72G;@tWZGGVk&y z4|DI%co`_~u_kjb0L^lsPG~bw7{?4hh;E7djt%ZGs#f8u@SuJnjci&l^KzGulL`6( z!5j*nThy`&bnrE8AknNr|G^7CHtjlWGQKYd+(l1mm&wp~I2#MOnFFI}rO5(V;KODT z_gB@qgKvbmNm+DXVvj3^kc^fXirQgYGvu;Yq6bvB-3Z{2eK12#lr8@Vvb4|#GwrCX z%8GS-iB&T1M6DxFMkNCV&-VC+t?-Hl(dmSI1AA~>0#ZGg~ual>(3tjz`uP+>!v1tXl_Zlc0^3eW6<=ndDoOwqf=P<(twUks$*fDKc1? zC%??zwRnta?@EhxK4ZaWS2VG{S%_zwR>uTX>u6GwhethK>8>?D<2%6IZ#!iXi)p^e zoJsc)o))cYn+=2C^(D@ptCNDJ-rrh)p)B!f8(ggqF!io(4&(=ya7UY}0w0!&q4!EK& ztAw32(8vJ28U0zFYZL3>K?QDil}V2vM>H^9=JOphBItFG64Z4REz$4jw8ziic@8Er0kS1x5Je$2q zJ7Q2x`xI^%=pJ4ADcvOO42h1fFHiw(g3n=2%Ma>R z61aeGD7l%Aj9Hd^dK8F5_;0UryGTpa-Vwl@!J~JgjzpaZ>af9s($Gmik_BUO?m0IM zN<%dn)^U^{GK6{Ql%_P1fk}0+*kHgUvr6i@KtmA!I`$4LOrU#_O-I99c@F%eEOJMZ z8%yL!z&OE+m6|ij$pZ&lBr zowjKV^P!AON`>pylTM6Kd`FW0n{aO)dajW}~87sDV#;GwYn z$KbORG6EPpi~d)LK-)`ePa349<6q09^CYcUGjbmIQPS3*H~2$lw|*|e@QqDLZ=5)v z*3|HM?pxTsH`i&Feix7~QA-)pFRK$yNt3SIqU)hiv=hi1>W6iZaNNByuzGIK+O^h?6C`1W z1jm({Gvq8+;2Zl)?N(|)yB3kr8UUw@Hzddjbr01H*9?u$l$R*Id=422oL80!<&?iT zPl#(-j#^gIrG1gk!=6OQ@r|2ucuW?kY?w$tt10C+ECuZc=-mUkbW~6DF4jS;*R4% zc#w*A=e#sj$R$7%*;i(u1OPENvQ5MzRMd2QYAaafscfn0?AxWG!M(wQ!ky9w2@A(~ zb>5^H@`su|lCw$O<7mDO zcO&r$ZRVSCA!DaSYvboZ#-?9|rSf)7C{nr>X+!^x(7w5OcylwG&jrY_<0#Oy*-4ny zcCe0V1mGcKK5#(i6puxv!x+bROvTkM59x^jpx>TXh3hKN;MW_U#OoToJP5F`3%wg- zu==&!Eu~2Ki%_xK?jo^C-$@}@rM{$;PK8L5>TuOntjhUnL&S7Y&3y&fdCTSkY9oGz zR>AUuIut_E!LxC#8sRSqw!#fkr8i|$F#cK}-x_G}YC8O9z`cXPW>Vx+HvW>_iJ`G&p_vR-#xZ5&ui$6{ zlds#i65KynsOkT#R^CM1Kki5f*a%{Qt+!^fshQvE;@lfc#^&6xcgyZjg7UK)(-x(1 zh>6W|sqKxWuM|!hW}2R1&(7DEl zptcRFuP_N}jmGA3*8YZp!hZfXq@dPBicM9)lecPT9APXg<5g;M&p6m^+ z!ZdL7QVQ{Qn3!j2?rukWMlOB_1SvQ`IB`n^Z*F%k6vKs<={?nb|j3Zf4D7Uz{P+$=}x2%77yZ9=P|{PnCyjv zsBDpYI1`bM!_QSiP935ho?mCeA<|TJw&{d_0eVuH7yNJ1p?Ahf#q;Euir?hIT}2)y zC<&8cX(of0in{mf%+q8st`bssl$XDeos|Kn8`%LNgiGhTJ?#!^W>K7=(^uGb5FuZu zu1ml|^eqj5VRpq=Zz}6N0)*eM6i#RZus2=vJPNJ#L_1E#T@#>!x(>d)tWNjscxjgA z+xWoh(W*b}EL`~R_|PiyV?G_O`U#ZUbT61`!Q6#n7`{qzG z2!`K|17maAPN*x!T&a#W-80@eFbPo7b4Ph9Gtmv;yiu~G#9ozZonSt*Wv+x2uhs{R z8&!w?Stg^|N3fi>=@IBMaKwX3%7jxZD4)`UVzq0+jbUVREX83OFfw)bsPjWiUX&)i zJ)1#|sd?At2wsVQ8T#|iHcVi9B>c=2Jw28mGCoSK2C&ax_wruDVnct1bkP)P43>Pk zzC3u$@v=QQcM)m$I|UrwBb6}&>$9OOkvVd96d20}XImVT zObAwhU&noBU>GNoUS=Bc6D*ysD!hCrKX)yH=eO8N9iP+uN}v~PYs*436#=fB&+A}P zz(JnOQ@hlt*t|HEpvaWR4Mw`L{RX*fLWgq9RuEB)3I%RgE@2%6cv2>@_O<2wUdVJb z>=Zoi9?Bl5CbM5aUDb0bN{~XA2bFYZLoK{Ch1bdBdF#uYUN~+l-4A%fv>p9zkJ0u( z0u4zn6H-IQeDV%Zo;*$#Z-n2OzUjg}2d(FueY`RQ zJ5&`x>~)Fr7+D9kb$`58aE8*{t{nVyA$TF7^1w|1pdGCuJK+ozXcgO>#UW%->Ex)V3rwVQG?5k*Vy!>ZaZAoFY)MUtcZ(j} z6NER@4Ip3NPM!my>vc4}-+}iAUdK5hg;MD-#nYSOvA+qR; zG7YmGC`eje0v)Lx52BV;q5S7J7G(iAc^hrV-`xc;iw=bZOWiN)vR7_I=U{2M7-Hp^ zX#B%Lk}e2QnU4k`URy7?0`B0YD^4!^HwbmI1MXAkoNbpptpM8qphL&gIMSJZ#e#FF z&YFMiS9HvW5J_FVDzuy(+;%Y`a0FFjF-ulj6IIe(R z*T^& zIT@o_3wZ>(-jKxzbM*)@C4IFDkG0M#N>@Kq+`%c; zuEH};2dfTDBwzI_x)pg#6Yqp(!SpJijx^~_%Fm%pQRdwcf8i5QzQfOvN`yYt zVUJ8uZjwl+Oa;RTm6yBfk=jYepCIZp5IOGdnU^OGrEs=&q?;rz*f^z&tvE9~bRWdG zZ#iKu-Pxckylb=P`t$lKjT)zy){Wcat>kWo6k2HQ);fRZycH^FC!h!02BEEBto z7$QtD+EEY2x9Cj=w?T(s-f7vv*Wt=vjan$o-<1=;NN-Q|WdtPBQ%BXj7^`j0L;k9% zMRboJAneacyY+~#0-M`Oq+1qXH*wO*K7lp(n4099hkIK9wQX5ao9J8VV2txrz6eB# z#^$nnWCyHc_A(Rwf3D*|#5(rwKr4DM03@D!3tL)~ zL4l5tcc*y9Wv>E7Iv7ciDQquI%u&)`t_BTe0$QCln>O0eWJ+5%t-uA%DA{<6Kaer2 z(IR!=Bb8aw%1e*>ZWPzd)PD&Uitz9-35imumFxUKE>L(UeQa#21Uy!Dti=yFP9?DX zNR^LFDrwJiDB-EhdxS*`)pydB!?wbs?lEX@SlJH9yzDx?p<(-??Y$G=rB{6-OqD8e0$o1nM=aC;?3n3H^ zBC}O9%V_uNGb9u9fHnTSzBpgF$! z=ghKfbQyH1&zox(yBQ&f#+D+AhjT*Jjff5yC2wO*g0fr%KLt)fz5a~xFkVdc`0@W#9Fd7nv| zftt18m*TpbzHDAPnxMu1;G=bCLw#4$!&Ko%27G(a&nF0rJ--%SLqeJCU0&$CIvy&*B$ z+1Mp!+M`uJyFI35rU9Y~O*aDW_Ggh4!s-!aj~MUw`b1 zrTf8ODb0Bfy3NHeQbHR&4{YubIfX4EkvVxTj*#15rz+A+BmUMgmBrPce{1nqS0eAX z5+e?875pXXTK4G&2;G1v$>e{lg7~=sq(Xkr%DlQp-YdI4deTW8yb$iO9_AEN+X`-> z2u*fdcO)?N$m#ETFJ0!3${G4g#F3#2YyrC95CC>v`H~}()YSNFkgQ*90Kn%`e~B*K z|F=XJPuM|9#N`5XU@sA$;!g4Zg(#&GVWs0FseylnwSV`3{~@5jm+&7zVGVg7JEg&h z(eHEB#L}fr-8tciP3I>Xe+E{|41o~PRqP5!4-B_8zZ&p;m81uxHOi06vVQLje|#XV zxhHJ&r}T&ckM-|K*HS7mzi+GfK5U#BfxR_k_lW_up75qbk5*9oiRc9rbd#VxQ`i?@ znyVc9_b1K)dCSu2-`?tfbqK=v{??j32it>JjYt8B2MDd=A7Gi{e9YfWe+k=r>xb@m z>3Hw=bR5GAgxiD{ZiUYu8yL^$Z<*NM|2Z--pc1>?2UnyCK?`!ZA4WTHlRwc1L3ong zOEmxe+Gym_z{mwX<>0eFq*PLO|5nbSB8Eo)A5zZ!|3~tB+5R_T!vBjOiBteR3<&an zOz#t(Ij&2ys24y3AjbE|W!-Xe$T3Q^ISmtXuIl@n#lNkrC-$p1+A@+#SZ5hye>Bc{ zeF@yc?>k&EM{mqr4VFraTi~GF5?v>Zviz-J9i$KW&xEqS4}(s$n|>2Ybt4i$$l~?6Xf^cYN>|F(t>Kavm86Ow6oD%bOU$++eA!{&+D%_Vx_j zCyftrE221`d`{+f68x4Mnk@S!aV^FRDC0Q1RrBWUXB%l4r5yuJ-cz}##2il68uAB?x+z60Z6s+Pc0q%=;@{6338 zc}8Sfb?=`3qE&W+MGSND`)@N7IsFi9goknYcEwT}%194A>ZcUiD-OX4pTRYarBs$@ z=%)u_Ltw}t$HCvq-R}wC9{i*+MN0d)#8)EaIhBVo*;5%u;04KX@r6>Pp!N zLN!OkGpAxt;3sPq>$QvJ?rU0S*E6JbYjfVN;#uB2$qA*$<;z^^BgcCOT~^+QqNrC( zdhQeHhT1y|>s>B3v2WW7OO+>l11WW{pYHU`{Gi&?YsuP)dTM9Coq%eoO2+%Nx=5Fn zm@6Y*_7znem^6c>jvEiWn7I+4!w$9gm&ODuN(bq@V{Jwj5QkihiSdL2NIK=?Cm?4afCld^+ zuioIBWgmx+;2Fo}gR%e0=QgTYz>=@n&M2RCO*K2NF*v+m86~`T)_tB{0e35TP$^R7 zP|b`qr-+^giJk+D7Gtu>2dIf1EIo-^nOb}Qv};}UBY+!03X>`ZXHT-W|r-R`a*3d zz@74qyWJGc>^DAqit1kD%6E;gd1Ge${OIa-uXLyCJTo&S4>Bdbzr2O3=%LN1Ikei3 ztzWxUbUzWIh!`(%C?puY2>Cubww-5~y})$|3uSIO zQzFympl!Ky)>ld>q)Q!ZSaM+8&k{Z=#zz%cuWl5zi7URf95e5iEV6>p+jU^Ia$KAa zgwQP|cEHqJPNcN(6l2c~9{TfD*=qP_KvMC&Tnq5hN@6!v8?=$jX`79J@6@RE zCUg=-uypog9fK!dYVk~N4onTraQKV8==yWih9NhH_;Kt48y-AVBF9t6^kXYIOv+)mw&qgX%$-9lJ;xS*en7D;-c0ClM*kVo%TLqlJ#xQ(AeuA; zlaf?geBuY1{)Kuec7c`wyg$wY{m&7C^76ZfZ8=87sbxK$U}e%jexY=8VTt^PR! z_G4b&E_^PhXpPABbky{r&1dt`7U`pUUv#Dr(BhZ@eh6%k$hm#J9zjYu4XDcZ$ExkN z*)2mZH1I-S`^1YL4W-Ua-~=^oHpc(pWiW~*aLC!nrtugrRnuuvkXJQ)6RhnL5WEp5 ztg%wYtd6iaJ9@m019Fb4`*pn5InHg}O?}h+RT=_eO*Qs&PvJ^RS0}&?Su-iqWftQN zjJ4@aP~Yf}`;#J4-F~|z1DW@YFk9mjVLt}<2{-i`4U9yNDeeJ7;z)X+o*|L_0!-zs z*~Wo+fOFsN2z|6+b1TlT<4toTxXE0f0$&kqH0xD)L?7DD&w$zU$5UX-4cNjc*mW;)A>?yoQ|8ji68LRJCu(94aw=joZDu%EXxE?!Y*B>! zT$r2N>z8M#D67UaYp!Kx=DcIednwoInN=Jw)nNesoQ&^TpC0@m17pccy21Ts#zArN zn&A-0=O$*qIPdMcp1gYGc=Yi$*S^){texlt;E-2;fKn=IST@& z(_lkDG;MmzvVHetxoKFVa>Ih&=lYYXXO)&N<7zX#ik|!EPBob9F|4!noh;V{3Y!*7 za`?tw#^rEM@yV@NuLG_1w<|kWLo$UqU^dq~ zx<|iU>zstof1iQkQNtd8=w zHx=mCiV={iTKt|6v*L&JAU2SlJ=ojrc~i&D?>t$zI$F>sg}h93)%m{ z3FH<@O@BR|LaLmoU*c!h(Pg8@ZT`0LKr2+psZ9x3=oF>Lc|`gVvl>kfv|F#JWxVBI zx3yhw=oZ1$PM7Jr|8%8Yd^G6J&L;p$LEam?ewzN!aWRqpvHweZQk5b49z`j`fe0j; zg&l)6*5?^l+wV-4@X7|626Xd_4Q@RQIXyr7vC(H!7(gVra6S??v91dBDOQ!6YJe27 z&+XT_1^(cCaPQtZ)TXK~OGjW^l9V}u5)M%%bijwp2CmwHv8o5>RCkSUk)kv*%^+X; zWZk-B?Bi)aNC6{~Dq0Su^3@T5`bzgaJ!D)oOf+gDmN@l@{Pa)YI2*ucy(q%b=t*F_xK9ihFjF!Xav!(aJKAaa znS^mx-Um8_7Ql%l)6AZHXKu02=K^?)y|A?(#SBpAbyfTT%)_a^Spm$Qz5p=CU3lAk zib`iv0(!QEUdBrhv*rLs-p=~ftp--*xf!L=>nHk;cGR{D=bsyCW0rjX#_%xyQ=3nA zZ=ak?<#$Po_D2f(H6_N3hI#_8QK=slnUk4|wXGBHh3SJm2W&y#A4oSsORbnGW2&9E zZhH+pDC%@qJh_&6Z&WcJeZOgJ&|^Hyhc@)-Q#0;Wk}&mMJpal1XM3)|VNNN5BCm5q z9ykq&6|`IviG6F(92hw3!wjkciqsf3WrwiR%!@{@)1BfCt-jEmqgYn17WmzkI`fKT z*Uc3WN&TqpSsU_Yj<8Z0wy#?&n)(sefqCcD(v_PgAKwY`k>{5Cl_V?i2*MUkc%SG? zDDrD)v1qr*j1~QiR1Uu7i`tu=KA|f$5DGWH$+H6(*?jda1>YiMuQhjnr(TQQ+8^ow ze_Gn*{K$(ce%IFdxIts2`iU!xUVQ)`v^euw>dF*xWx9uP4|T~Nu)bIm$1C_>+X1V4 zfdu`Tqdm2y=+rZggM=z(F{O5xeA$xl!adqR6&&}xj%IU=zKPw)Ei?BzRek&22;g?m)uWO+FSka zf_cO2Dwv}Z35@;naWfVcUNu-CHj3K?Qx${jG*%p~vX)OnVD{E<`}d}kIp%nxV(kM} zRplbQdKy{jN1R60arBaNQ-u3giFY4MyvI4WOYq)l7~1 zC|z^=>5^s1I(Y|UmDA0jMbX5Di;cee3R~c(N$KXyD7)>fgDLYpwEwc7xFxyDyTxrl zIcwcbrVE#&*uXqbv+Qdg*ZewO53P}w2I@m6>WWi6)mZ|qJTL}cv%HwrhG%?-*k!CF zcl~1Vp+XYw?ZYpqH=Y46Pm`_BC>*O})9(*H@_UP~ID1t#Q$dg$^;`SL=*A6Djn{oQ zcZt)oMK?SLy3jCTiw6!z2U}dCelMd)M~wY8{KWjKytMDS*58GT1+{(lnbQa`_5QU7 z@}k}C{MkRvQl%WbUM}J^gFa~^sB|x)r8uqHzhz_m)~rj z?t5n&UNGL>8~e-?hlkRkCc!n=r}}%{6Bw5bN0NlG5e*~-X8W0B zoA#wFPfEh$FGjEWTsI9Yr_QEI?DoILNPdUVbi==ByRT49Q4rkap|tW9)6-QdNp#8k z>HV{nS7AQKoY5}ik;KGTjc&GNPNgXPx_3$8uOO9e@u%Sukb?0?iW9;Gy>>vx^^e-4ikJ@!~N z8n}8aW=3xs^Tsuh)vM8D#zUDRelFh|$a=KB$o+cPB2L$bylE25`?H-=z8suiWSTMK zf-$g>r3Xz2G^})w=ZpBO(mf}K z1M#<-&|)2$^w!xocR&_9nVvwkKA2u>l?O_G?MYY`vjmE}Wd z3>k&c`GiCczkz?<=U1*Kwm4qT^mWHGc-QXpqbNq<(*$c4Lnlv|#r6pQO8GJ7N}3zM z7wIJg?ur0BNOtkHxF*gHQfGJP%>+~`#dj5+_{M7mwaGFUuE|qrq;FYSknV!>6G$8!9e#IhiJ-J z!@jgBla*I&K_#Jy4eArsWdDV-w~VUt@480+5>f&J0s_+A(jeU>&1TbEr9-;efFMYB zcSzSJq+ui7pdiv9-L=V0zL)oN-|zdJ^PF+cIG_B$7!Jn8`mHtBoO7-9yK4EVsv2YP z6&1fqR6B4tA@xpP)u-JYAll9sEiePOqHZtk!6+8qHo(b?ZwKco4?{Xjf7dLVJdEqt zf?5dthj#tLgyvv&6o3j`o7vX6tZ(GkEzZx;jFZa3+?P}Y6EUr5i=bGmn2UzTPn_)h z-B3%Z@DvM!;JGO2N}urScj`5CUo5yM0h1PMcJe!1XB?S(9Fa5I2|=7yJKdSG530m)4)P}p@tH&JU)HYeJDXl{LVL0L_zjVUBXFKA z2tS}%sYlV|v@W^m-st|K%mRAdjXkLJcX2YX%E`k{`d5dVnY4lM9Nvt9(rnyk(QlNt zvQIy@mJ7}jT(S5bwa-ruxx0P`H}wGYQTYeLk5PPhiDDUV1HPw@XKM`z88+OOE;K`& z)I3-33_vqP4s+z_zAmD`%;SpdU0p;fziG;7dii>^MEsg^*-}jizs(Yl`qG0Ghjs%` zP;?WJ(11IN8p>nfjNNN3P-J_lJVqG40%Aj(cYlah%4gOu~Vh|*RI>!d`p<8N3)9Pw*VM_x%T2A-+gi2+NU-yhAd4; zRO6M^+YoMyG}>#?XS z022gLj!L4b(aZ1Ji!~Xq7c*M)?f`~!S@&tKQ>f#X$lK1rxD{%liddDCoCtR$9GH zidknInc)(}#_bQe9TlIN3ijX&8rlqn1kyRATcqgmmRPt&pcSGlyh zyYxQ-0I+P?Sy^>HS8^rajn5mQw&1yr-WG7TrCaKYmJl76yD+?daheKIa{o{wA={UT zWeVYAj!B9@)s6b6qzNN7|1LyTp9*JsED~wZ;5tR_o$^i$l|n#a?9lTj>Kg@YFv0T% zC>})%+odMAPTZ}#6eH5_&J^2P#@pK{k+zFeh3H~8H7v1x-U08KAhnz%mqOLX43QhR zjgCpkQ5MggUQt97B%tWSoGBJ~0t}i)%rRe;`J5W-mU@vfMWq@fz${!k$0JvMT2IVn zrZ|;~oSaN+uso<@Sx~6KpYqmSNOUQD-}CQ3bJgvvmLTHzwP>GI@9wffzCcAXF(Wku zcK06~Srb2E*K2c@IbbC-(T`=Bn2@?zJqYhv^IP)0Mx!0mwKwgMTh4zPlg$S5W7RG$ zfLxnkgVOtk?_@X1J~6zW|E!|iE(=d$sco0clqG2W+^A#PrbJtBmBO#|D-D%?d92RA)U%}#>z`9Uxs2Pr)r zzUvAzLRQn8EXG_6Y69iJzl|TSh5EwCRPP`I_(y$(I~;Yxf}Op~8gbbMY{J z{eM~vYOGnA(9P=ZHq~#wgbT5U^C(3Gng>yXolWm<;%n#XEyOJ*vSX!MMQ9=M1bB{l zX6>3maPIDD?OKReAk|{0B6IAkJkPDFq(++85;UjjnSPqmqh{|BlP!rHU!mdHsT`_& zDGr2$FA8VEWpC5I@2H#-wfFYkI^n-KG1sm0qQ7w?2}0fG9O8ur@L;tEY%>kLQ%puJ zr_^Ps1D28Dys??Ipy6=%Q2R78THooE7}(oWNGm>n>fZN$t&yaOm6ki_Ahn!O$#cT6 zMQtO*nbOl?DNniQ=4Woey|SM{HE#TCE$v+5r|Q)pf|m|Qii;JcqyOaejp$i=*QX*4 zCD->-IC}#Mk#?RThM4uO{?L=hVFt~VgBH2H8!VDOFZ)kn4dqOyitvJY3G)fT{>qmf zS!@l96!(f9c37e$5ZPHz74L1ms8tXR_DYEGJpPwC~SgB7>SJ^4wQCrm~G?{-B8ze}6G zB6NOP&-SbKb-e)pO0%Yn~-R zFWe>l?x$0$ZG{)aWRScLv{BtS8uUKOqI_=DicPPtF($1W${&Hn& zc*b&C)fq8l@WyS^Y{Z-`jwMYt#`yME;%LFpjFdv-GV?5^{?desVZC!E*Dt%BX3Y{^ zkhcQlhk)wVdwijDHL~y;^vJc%0x=GHwiiGNC9ri~uiIpSM9OJ)Xl6kt)blED!HkuM= z*kQA1ZZr)|+AxDlWsQrWdRb+#3H zQ8A})7TU#cj!miR{k#qghw#1{lKI0e=iajuj_0 z^pg#xr1c~noK%n5l?7~*q+D4P1*I3wPv`pJ5?;};#>feSXziw7yDm&O<$)M$ydU4a z+s5H|KIGNeL|2aCZpZ20yATUGD?b3z&Le@-8z#San8QFW5n_7Gl}2nJsi6`(I+P5N131AO^Mz|epVFJ{apnnkD}jCYN`BX9O*%35?W3xY-= zQd_m!T^#2qw}(~Be%*=aF=vfskZ`VgF>>CZ9z!Jd2O8zN4O2DgO^vU^eH+*PD$FGt z?08!m9xu_9KBg3)Df2mB^1@uroH21k;2RTB6btdNT|SGiB6f{@tEqAxvH~B~Oc$RN zV{k2>azw~R;BFYhW%_KkA~>lWwe9gAJCEV_^90E9x^*o9VViO62rBw7B`w!GcZsC0 z*IX`mQG(?P7?fp|L(9-#y7q-gvM!rpTxo9?3s~55E!Lz~)IpM5r4(S!(`)BL9ipbZ zRuQA!f_jDdFlQ{6Zyy3jWyY%)y>&5t(MXJIUZZof@YNogul7SFnCp7fj}@;`z^qxx ziIjxS9`{^BdEh6?zxqKDI(svdZxZ;?2y7TAonI}+ex%SX#=jP+IlfcOdFXR|7zCRm zCoBu*4+IWzd_~KQ;~tdw)7mUcbbpph_2DbcwKYi;W|ug69ihvo1wE5Z2r2JZbWx1e zyV!hF=QPdH-mmK(()yU`yy=B+TB(~Klbpe52tE*A>-wX}WM4}Ydg~^@&Bc!YrfEW{ z?G&50Ys_^NZv?3MkgLQ6_^H6tyZa^JjUU@F@oTDmzFnyF_;jC__Rl7o?y=eE4Nk={ z*dD)3UW~eaYm}4fOaCx<@oQY>-T4D$P%lmSY4nq0bFXUgfr)JQ15g@={II4PrXVt6 z&+D3|SUJ;l-i`^9Tu4cBHD;Qmq)GOx3YS*-!$y*KPNOYCATGY@3JdK?(-q&5G{dt$ zR7QOXP5s_@AeXAWBvO&tu~6ozv4v|5Ole(mXXg~D7nA08Y zwA>_r_p{y7=Cr&C@yTt9nnre|6DskGOYwSf;WsR)mqC|@jurnHO=*!2BjKtzii}}` zMz_;)G@)k-!`3MIn?A7n?zFrSTQ?PCPKS&AW*?a;-@&|Q^35__2rV?b#KFt2zI1&S zc!x3fVMHsMAe-uC5SGD~RTXzT@H)M=%Zynv8EC4PUFm*_Quq%Dqe@2y42T|{{miin z-_@Z4(YUMYy0UFXxFC5o3DsMT8g84dM*mQjybtL<@7u-?cGEphh0k9CepESfa;#X@ zreH5j%TXbL$n6wITzUA~qGSy{JG)>~F_BRxe+%Y;AP0@~81uZU-0cS4liC4~Z_ z^hUz{Gvz$`CdswL`t5|Vg?$Lw@@?L19qUGr)vG@b^4h2gI;l0(l_onpy8&-+=pai` zlIMZ{N=DxgXX_|1ZanSK7X3@mh!g=}N~9){!UXyJtW+XPfd-rQd3>Vuiy)Ca)oH-d zW$#lGKu|*==`c8B$+0wN7AN$+)pnJGT(U4VHEO2Dl+yZaujre{3afz>ui_Ff68TjU zt^pHn=qP8~`Q?krDk66_;Hf_ZHe0B13cN(w5kgdTg)*SUk=wD(Bg)k%C8Gg%G5O zdDEdmLec?U0@IC(saq`q+U83@)(_r`_<7$t z6jY@Zvryf6BHGVp9_vl1%9bXbkvWhliI`TlXsV@rz?5ac(&dO#zB};2#W894_PPF& z8TXiZ*|hYjB1`uR;@|Qr$!A&Ek(PaR#pX!FF~hDbW9BoiEv#mv;4OqZ&E;v zPZ`26+-!D}oz^bNb}ifAc*lg0B5mCz(I-|LJgEUvf}u(ZV-JXK8x8tT6Y?^yVPlx$U-Ak zu)-3G!yt|Du+cLjw0{kjY88;~mODS3iESA^$24hk<4Qt2dGYSX!+!W!OmfgT;XeBA z1-qWTo(WQJP|@p=Y{lZFQe`mQ_F|T|n+-wCrQU)dN!n{~w^_q+M!5)`ndC{oTYTJE zZ=((`$J(-2`YC8XCyn>nQ}cOc9P!h9BnnXwjqi!Xr|suT#-+)@iq{Be!<$s139#&+)AXH_6BohG!{&C|b|{dbVo=xCg_Yd9d#MX(c`> zP3(_Y4qnYGBB?#mkS-Jh*+u|xo3 z^v`4lZu)i-38Z|!gZ8&&_{=C?*yZA>r9D%p!9s&MP3{V1C1G<iyu{zlc9c>6LKIl_^v&Z1wkCZox)i(JN;)lXdWAL+PyL? z_==B#t_$Ng82Di??GbB~Z(QkhHLAq)zEJ+^@o37@&qslDbLd0BgwEIB*>am}Y(_Z; z>%LDm7{fT^;oY|+;W%*v{Y~?aUb#~PG7QQ%g3Ovu&Ql%&6KlSWybGQTYB}1KkQ2YK z?b-B&$96uNtPi0dr$wI}IhypV>iowGz)(MrUY)PAPX2*Nr^(q)>%d@S`kp3}^u#aZ zp$!&0Jt6FI3f`bjTwzZ_}8)5h8 z5_PdlUOS!?*Bc+-J*snfeWjavdBPm>j6tHkM;jSoX1XRH3)}>FP3xIJvtkp&smL9P z&p?;9Q9;-DCw-ytBUl7Zcl9TBx)j8`;A{`{3a!MEnEjeA2?`&uOy8BZz8RxN{?yl- zetAuUk^fE+E^|pN@kxK@Q*f5ayHo#F-$Zj~931+s(m{zTvl8y}yE$s4VU@MI!MorA zPBxXOKy-ekx%>j$NmERDOVDaau9nXBg241yTY})_CeG6W;`Fr=2R}x_xv&#DNcM89 znQ-_&I4mCTnaHae2T;yOmFd%?=H*xX&xC(5eP$lbn|EwDxm%(}+A@<_Ni<7#YbvkJ znS4DJDZ|U$BMZo(G(+(UaI|QGUUuU|t@C z4}!A-PAggYEzYHEBGRFKD?h#@<5;S=JR?zee%RnH0S>rY9=6?H{PPSSL5vfk|JLtr z^D9sV=OfL7i|R{wMHIb)Zin&l5^bC`Vh-0@o!Is<|Ua)t;p_LI{}nW6&N&U`>{NgauH;(*Mh zQo|uVQ@Ip*7rlVpxEOR%@WXB%O5G}r;d|A8DDaKBb+v#KcyE)h?<*{Owo*WOT`#4( z7$91H;f3;I6+9RVeYB0Q^?TXSY9gdMzrzcFD`WX^zF$}Iuyuo>^*17dl~^Pv$d&DK z#W?#K;V##Dry-Zc*be&+Q;_c+fywOCZG-FOwk$NKFc2k`Yu~YIQpAu9!=#j{UYXxJ zBQJ7pa}6yh#{Kyeeo|T|#XGFYmSfG+bmeB1mc@|iQnK1iqR#`ve?)!!sRZ1tua!NAc%v}3k&{S zMBp29RGXdd1{ibJ_?+6G)}sw2aOvoYxW66(n=8orvlD<>?Zv#*WAL#=IhGGauj1)7 zj71R_{E3Q|WbBOlcE)!Y%9p?$O|8M?+x%8E>Jj&5!?_q}$T^h~dr5*#DajMKzSBSf zQQdre(c^uhqkIN=q)6gc^~tvZYGlOaeeAFOwfK zNcsZ_PKJUEH}GIfuIOvz=yBMuXbZfDXu(X~)x*aUb2j<{4S} zh*J{#6UDTd@3#Dg9x_H;;xVBU5KLRnq;Rp%X7($K=7Sg)8{f1{0o~iGBLbx$vsH4? z@z!@&jAXS;e8~A-VNA&3n0AouUB2xCy^w0l6 zAwXsWe8WF)Mpw8ppt!twH%{b%EWuK8>QA*Ww?1)!uGFMzcC#PkzdPIoV<02gRFqh{ zMRzyB+8B$Zfx7VS%%{ttq1FDc_Sz{az@%lYU)#$Q{+QHXB@M)C9nMzCo5ZJJb0dhi z(}gQxUOnAloI2N&4PG$x2KBW*zyz{EXft(haIKPIr5vd77+69|Hsq*O-8A`2aWH>& z$YYh*w{mC$g8e+S>Ek9lc(!2jva=KlL}~dgriAs0EDnECZ!-vV$y@!x$4V{$yiozV zy>A$!e&1|&X1KS~5^~)Y?d2KsRTapD(>9wG3s!}65Wi?}(*%zwV*1ea7vvUYmqL-! zdVbKM8fPQThZ^{HQuKKQKY&k}BY{@%200IZ4=U3?oOHaNn(=P!BM-EC9vF6~4)B<9 z7)k$M0De^(j<@$aqp^N~ZX+YZ^MJq!g0_a7uT-#U?S?Be(@0X>qEEEY>KNBbDO?`k zbVou%-0<1dwC*jRDXr;oCCP7)pfW;;4J;V(E5?fkb-eKVqR3RuL#xbF`~qKcAO!St zU?)q^yJ`9|1wo(t^aU7OaU1SSWtya@{4g;-q@kD*+hvc++Ga`l<=Hl}_2az?`C>=( zU?48JkeWv=eLZybaN>-3awu+5WsGpTzkE=N4RUW%k#OXu0%xPHqcb)UGn*^^bgT&C z!QHj|%0P!#N=Uq@3vUH}tM`uQM<7zvl@7L$dS{OS?Nf?Na97187VxRdw*P+WB`^2e zS7)J&C(gI8E@t3T0Hr2vD>Q@d^0p)Pe}83u6b)(=t!${@jnCUAefkr7`seX0tUxd# zW_|PqXB^=V5h`0=0=j*&ajTO6I1x+4{9ganrXj&}M$mcOC09bgwg+Hxa)x3s*W`EM zSafMxCz>AX%wmJ!>lFSnJsHe6bc)L_G_(iz<9 ze7vM+s0PQ2x2T6t#AtcwBhxRb1art6jyM(`$KY}&xIr9j-TKJ+HWRb?CbLA{oULJ_ zA5&RF2O~|v_sHjg2{2Kis{AJ=hLa1Q2f`qEFl$hWm`SILR(q(aN?O_<9=l77vj)eS zL9p(1kksdZL47Je50V$e7QR#0YZVn7Xk36wucUOF){bdYk3$i(qCQUe z?ly*zbsJu?{aJL&w8;x;MW0qN+T83gHwUvJ;}(jR$Zm>+wwH+~ZiWr}!bDXz7E^!E z!xko7)mas$YvKh1i`uIkhQNP-Jz?Q!rP%WBr96M6-R^!!EL;TxjrNaOX~%ku+1IvX z{xNJcl(<+na$X6rlR@@WAxdI)9WAl0(R}mD)lC7fyJx%f#5=k7(A-oOaL9+=DAoWx ze1w2|#y1X5s9e7=im6*%LBC&XhF=_Cog{hpA>X7!d#hi}-22^25!)GwCO=fk7FT9G zotx1)mVi}4xPv0Bb79Oq2CqP9^1o%fGDWFGn1&ze9?lO;F&#{~4bKSRX^JKNi|#`@ z>6wDMBuDB4sZHy<*?$RBRK3y|3G# z_sJOw++fww0&JJ1Du-WkLhN--K8ZOIo@~fo7&H`!vKE)D z_OO*zMobN9i1Yb9Mh^Of=|_s6580Wlra`IJ$8$!e13|sag%V?ty~14WH!0&eDx?!R zd5*wFN4#{abM1m&x1$X$T8#SI zlFgwW3=}2^wfJ2WoAZVJV)JPkuO#>S*%{`9fY?e@zLMcxmE9x3uR0wKQGKya4B)?d z0=lr9k1{AlODvJ(Wo`03do!FPn(I|R?+ssj3CJxN?pB0X`h%wNSzN<%WMP_d<;-b+ zK8M}5H!^HiE~E1YXLwLsRV|!W6D(Mbs@vawS16IW zgMO(b`CvV9`ORXCS0()o?(LwS1^$Ah&C0miIb))SCAW%b-5>E7)%ZiY*3ObjMzvuF z-1Wtx8nUSP^?`eF+v}BMLT#*P7!|6bZ`b}7nT?R?lElD4joa<32D{t^xIu@tU^Ky387_Y?EA}eSeu0em7^+oclpF zX)RgzuRdIESQ1}i`{O7=dN99$P|j@8>ojENi82-)rg>#80yOS>$)HBYmj*bb20;$B z!3U^6MJ>*0Xs<@Yr3Pd#e^z=bgx|d`3~(jocx%$!>L>ud$)D%`F%0(BJ6x1GE75o^ zvoizn&qw2C#NJe7mvfH!_4ehXOxwg_VXt!Hob(z}JZ*zL2Z_`2#%pBW;;pWD;S27P z0>fxS-gtxbKU%xNQ2Cw{s;2iyl=)VF^u$@AV49v+2&TmHs^$;cCe%3>Q_Rmw=#AYY zBYr29LB1N^RaWcPOC%3p7y^95g2t=gq2Wt@=bVn*)5Y{F`A<3Lai~h2mTNxHel%_g zJ6gQ^blsaIp`-WEK?$m!{(++dJcnx3@mG=zW{7*$?Y}#?N$8$$_|;!7$ui=h%fN5i zeoFz#24CqrV|@uGR!qb{QSJ=P8?2i8^EsK1D_n8^pd(k=w2JUZGlqy2VzVPI?ng-Vt6R_lUuD!=idqcb#q!gj z6n@X)+!x2WbGW6gTu|JQqx{{te@{;pXytvEG?KZT0FCzgwbbxWF9+L=6kdm@V{%*e z#cu$6edxDmLP3@?uxUFa2Mz{{K8`0EgoJ2Zx60nA1aOM@Z_dDEtO_ z?c;7EDMj&AuQv&Hg}frtzeh9a<&1-AKDTQ2o9ljaLnr6^_(W~9yjj+_ zX$oH>=Lh@SMz=#DV`%>3(+vzQzioD!fhIusg5_%Y)#g-)3BpY2rb#B8_58LDTJ+!4 zkP@?k#_OvPGM}ku5#Cdq$9{@$3CWsw*T2-hFnqNq;MUIZy__Xu(?VsKm|m6{u6X?H zSdqFtAf54u(1lnLb?=93+(6s#ER8O2e(NeZ*!*son5<0SmSCCz*U01C{Enh2)cjy6 z)*<560%3~r(qJ-uuH;=3A+s_Qhhg}3-^(cZ5*SMK;fIy|>a zKecgpxubg1>DEV+1!f@^Lq!?^&)kJeJ`Ti`GDuVB5$>*9ae&E~tCc7{4#b5=LB9j) z+E)wGk0*;kkwqUcK?^3g(eHN0$c!!OR#f*K)`$D}3pAL?#XtI&+GZKUdh^>INs5sB z9=_GnxJL?~x>ewKN6TLtt02m%jjuNLrctULiU5@<%lve7$hbi;Y{1@?ynB6_Eh_^jBwY7;oUvd zBw1E1vQ6i|0lK@u5}YS$a0O_9I1m}VUn`9Mt6BMX-}heuf)*qt=U)Oyo^e!X^%_8M ztc=Db6~VQ`NWq9y2tg3R21|Rva(1U-`)ri+`5!7D#No_2;5(h%d03Je8cXLkiHFIn zq!8u4ey$sbeU};jB6vMtDTy>wD69N#JplcNSNT+?>2px?T5 z`?6V9&T>5Z=-vqfWG~xv@RlZ;Wk+$9d^PTNm%B^?hgL!%72WgTBVQuL!DAXXv(6Kg zCG+mPU-6$^4rz9ivKGcDz`MVr?Oe(MKs!mCl5ixsXX3+SDmJk@qF2318{4A-pKqWv zwcxZ(Nsdr?eo3RaD&62XVAeLhxBcju zgc*RHg_<)5Q09qC4^9BE9J#S2>I&AjVVYf%OBuS*+=ck&4xRY|F9ntbg&~Tj8BXv@idP3xFPk<^cri$3? z&u=F2IZ7M0+XTGJuDtpSis@eb4G@+)7lf5}b)=V@JI?E1WG**8$5mi)ci>^ z|I2Xu4R&)e>2`8dL^kj_NG627J}dRkOb{`3e+%zE=CsNQ;`i>d)!1Z&PD|WhS3ggF zj|Hf{Dj5xCu&!T?zLFEW{tvr~w)(Z0erMP0^S(zZO7v_9_x?#WZ35ABtYXHu{J7~S zRMS2cEuk-XZ@>UVa9;XmB6M0Qn>R^pOo)TC;Ey6P>dFw?|mDqO43J!bRN1H#l|qY7E4ulhOaDnGHUI0q5wPK?`a{Zed3kLuwfiRcx6@FB}uM)+>JV&CW81%?Toyr2;0B= zz(O~MF}A9k71?Szp$t$3#8n2op_f@5tb~2QA4wU-eT3vWV##>=Xv46_H!}>lM1MPf ze%eF*w+4ufZLlxLlY_X}hNw;!U-A35GL-NBn|J^!*{RwPX)X=rAQ4>Qu?=Lg&Y@||CIQD&m)mI+oM`X1>;s^c4Mcx2sN~XSi z-o~6R?80N-U?0%A)PUdW!Wxvj{DE~< zf}Ux3g*ld-rmkyN((Lyu(~m9CBYt;BgV<399zhgj#PDoBkrknNKsFlY5p{r9x7^7J zT2Q0wRECVi`t=s3BVxMxkOl_aPFM3b_~3OAC7(Y(^A&_zmLlPOrmE3bOZ4oQAaEA) z!`x2(FUJj0Q+OT?q3Whj^vNUemy7=B{Mi>!K(jpO7Hy0V#S;9HNM@U-gW$mrzilQB z?+pDybO4UsVK=~{QLD>;^_-_#$-j39lhgWjA^~uPBx{p(F%Cu}R8BggY;qf$M?$%R z2Z+nwzgivSl|M-FevaU@OHmGKiBPB#4Kc`>A8XcLPyi~tlJA8?JkAqwe>!l$&hLNI zaZ)D)KU_^tc%~J9{ZD+}{pH`e+TM@U_IOYMUO-Mj*bp~}jx>+?dMugO%+X|r*R;W| zGy`}7o|26P;8`icySMos_qhokRv@L=67ujP;ZupCx7lATLP;kF;U;1^8F6e!zYDm? z*p;N=5|2>+H*ObZFdeb$Ybp>|5=hA~t_oAM)6m|GaTEAM-;*9`t`!bJO|dr&QpeL( zXo@7@w_2r#ypsFph;xpE}9{0FI5+BgfXVei+m{2YkYhrnm>pJu&z z5q$8{~av>pz7VfX#v!IzFc2< zCIm5WWi%Tp(W8bJtUv~jsvxXIXW-|iYYEH7uwVQ?0-;8ThIc$n)ZhnMiKn6mgEPmS zwvev0Y{@0;`Vx;jr=UJYQnC~;R*HBFRvp3(4r)H_K zm_S@Ud@2Pmc;mGHbHsE&QNTWal$GU{0%<+hD2p(@R5sji)&CMG?H;IQ7%to#8+LikK$|YG?zDTL z++R-%HV;tD#5YNjLvj7O(zau0@Rkcv;>x4Q5UNiK&jb<0)#Y?{V<6$kT_Y#pj=i-kJe zT!3R=#?G+f++K4n5q3{EKXTVX#muSQZMEs&Zm>huq)Vr*ee8?I-*wcyrcK{)jIx)&g$Et z4z|}lLB3w`Fzb^8&w&%kuQsFK&~?MZy~D6;tv`~ROdAx9gH4k0R^2$QDbj}M0)PF4z1iq~Yum_A0ao38s- zzeq0FJhLjy1y&i9(LhGF)D&w!^pOw_f?0-^1$;HM`Z?zAFKzQSDw4M=$~WwO6m$%h z&@EGnNSOz&KEdJEFIf%#4t+$RS6c!zNQR!p%N#;>&jwW|GT$sSBg+Se#H}1v8Y+QN z$Q>NE#9MbZt!iF1gXZ7$SPY)hL9#c3Kq+0XM)QRSF|7C>7_zvGv8uxxdm2ztb?*yq zzg9fa~h3@92Ix*%$X(AsGD%7zF(hNA=MLa8>Cxj6Wu} z>&>is@TN)n(T%;7jCvZCE_0F5h~+*AVDVSh58tr(&AOfTCLM(v_7+D7J%Fr=hvI1Z zjm!`2!~D+A$a4Kf9U@E4A{UUIZ?;Y%?0tvq`2#`trv+b8x`pzIs(+~he<8Tkb0DA7 zl`yG>a?mpt* zn=s}+yiwxD!QBzXSO{t;u02#h5c?0lG}?&-#x=u#@v-sIPP9ATD!_(kYX0CMm$w8) zG_(7wtba7y_~#1iB|bTkK}b6@^@0M{X(0ZmBGgU%U7lWjF<+CCEAqz(0LjqYyDunR ze$r=P=!)~W17I$6r}h49#HR6>5gf77BO`3@0F$j&f;nrUhIVJlZ&&3WUrz>IcDuRv zow7=gFO>%Z0k79~$UDTM_wadmS91-h)b@p6e6C2`i55VKe-0ED z`EK=}YX6OO{;yXi=ls9l7!-J89rvs5w%@gD<3_4_wP9F9%T)Nlf+3) zI-C?Rw{;Zpd;mLg7o{0d@3q?H6MztVvf>LHx(FN$2plNosboSO#FjM`cAhj}91ogw zjP1M&Qm4ethcuTrxm11O_nC*Pj&{qy*FNbe;1~4OGwE!@drI9?mtA6P%tC*FVCDR0q{SN|sTZN_pElX&d#G$^KD_ESe~us=|kBZKoh zSf8p=$&QQi-14t<-V$U>y1zXt**vJq5;RNsG@z6x`WGQuXI}WnV?3!iJsX*joC0{x zoi`&b0pA}Nw~^RbG{XU6h;I+O4&f@_P6dP%Hm18YsE#ox@Uo%@Ers6*B{aqL3IM7X zn^B8psUJV`<}nT^+4N_#Zu~6g3gl0SP;N|=@v;7!i23gfMN zoHAG=!V#NrPBza$2`L-M`^oRO_?lKeDj7zD1Lp9W;HEQ*H9aj?xaA%M79k-+ietz) z4$*hfV!?6TTYV}VhK{Bky5+`_AJ&i>Kw{6hvlxzN?tI=*ej@}VcAP@gbbTKy^MF?O zb@ILd*8I46N?*Ljq#xqGYp*#oMYFLw>d%48Zdl=H-Na~S7MI&mWb7%VzX5nSxHFzD zmU8Lh&V;!Hq%L4|uPV5-e1y2^fh29F2IC~7&vhs9))SK|6VvsA4>Ryo)@Sm8G827& zQOwc@KUa$W)@_SzkG6V)?9wke0pOBzFm*$^XtG8s!m; z?wMtEkVhp0NcZ%YkY@VeavMll7h zZ1KM7+LhrB)Yw^)Y#)_1B{?}c^Old-$-gm?nzVy`cu~a4*$@^io$TT!_eDk(|EX(# zjRSv^S{U~KR?xcXC;;7HB(g5lH-kj!t35NaKa66< zOb%+o)3!(1v+Y-dMG5 za7EKE%sa`yaU1*e|2=Mh{Pm|rGyLl}p~o_hFDJ`@M(Zw}>IyQ^G4GWSzFxBCM$3xI zt%-xFAl&x~8t6#<&%aaMgjgV4w>YKzzG%E)^|-dxu$VikAHW}REIm2~I3I3^%e1u; z4R@M^*2WkuDK(%T&=IXvD+Y+9@X&b1?%mlP%-!vrP3#0rNNVgFZ07sdH|mCis7CYW z-mH7Nm;w{?od2l#k{gt<= z?NS52#c0CPwzZB5sU8Ij^3hvoUek?O7pOSe^HFDGU1V5@-uubQ& zb2zVjGgKb&kj14TWB+(=y(E?NGNa-6XwCuIzRPdpCFz4q{#B{q)N|&^!uDcpScSR; z$0^g(`vK3u>s^iGEtAjnA2ThlK_DA6?9QjMzdrNHUZKW|3k>xM7K!7}S^nZ?>x-_<@=QbKb%l7pS#7H=5 z2*5I)MU|STayiGiz7DAO``#UW+Z$>m|H#E@GgBpiR zMs29KYi_n{CMTv$+G2LsB@TLIGGUnqao@0RLG$TaA|0i4#FvA){z5k%9}$r4Wp)hX z#y6unhn``1x+(n1$mFWtUU!t6h>m5L$s5I8J}c7LqrV1lNMo(qG-S^%GeeLuDx(2F+&Ge+lY&PB}Ia!SD+_|D^LbDZA z!4d~T0H5AUxekmEw$&mRV4Y~VUu(oW&X>ubY-0+i?`KhSA{?8ma2KwjM|s?DWuITV?Xte zQ++M{*7uj4U0Mmp@5}>K*q2GFj|nqGOt{A*cEXC+nGXcX9FPzuZvFM^*|r%Gnd_+Y_AjYk1~+{CFaK1{WsXe%kmSgID7gp&+-pJ%W(|;>aGwizWMTkwMy2jT zV)vNBWRmY0AwZw`0e?q&y~G!q$BehH8W$OY@7lps74i)96Q<)D#C`6@bfw$CD%M%? zVthDfE4(lWQ4}05zR*)&=bXH@b_;je=9yTLd_2%mc|fq6W(V6&H{!BQVSu@R!q2F^ zp?S3gN^ib30Go{w|my`NZ15li7_*jMPN{`tn;KrFLRh?SaMl)uP{kP ztz}rk<`vK*AJ!RTygWDlIZagzPKLUR_GibztW%d}pOGkdyxdz6K6N$NOnmip^rJc3`u!Dbj^{x*`|#(eDoeN_dAZZgl(4|`!*tkN z=)-T58%;g+UKhgV7=cAnuW|p29oLqW7g;ZK;IBXAjJi}CWt0Xm>T8GD4FlE>X;y}4@YwpXpqd3mX_&c*uJL#qvWlnG<><_=h0 zLqOjShnHya@CUy1f?=Sxtpwag3s&kYWzKLgLh)(if(=6wLxIU?RUWfG1po7Ozp(_7 zyBMc<2jdnw!0nyJHf=9uR?J<~jtM0-ed8G+(c)=DSVY9A&UOq@TX0YGWH)P0zgk-K z4E%a~en>{uYc?V1_|w^+`(V~Cu051d?V%3b-AbvF0cCUn)`^6cWv&Py_nvZMEC4fL z{7UOjuNaiN=la&v@)-{gE`N4PW@5@s^AN=+cGZg{1bKp59fLwGbj1=9Pl@yp#4eo} z-lqg|71$iZfxW)r?5tOH=eB@aCe{h8mke#?ui6ACt&tS%zMd0Yn~b)e0qwE zWcAgBJ2{}At%G|{fAM@~1wY8}d)&y_aFf(Jo*XN}0=9O|o`2FGU2@+-AOd5n0m zL3K0vlm}q^U}~KK2uctz2vlAUh<4=o2j)=#y@~r&A#jloFmy`g{Vf_%1C4YL#Tu9I z3VP3#vY9xO*UEjLe~Fa{P!jSAB)|zWN)xW5r#S>nVD6guuL(5)!qZ5oT5G8e-MV+9 zC7%r|U*Si-mb<}A)O3vGP@rcW3fiXA^K_E=F4OW4)?2dw#ok*6)%kVVgTXBUF682_ z!2%?~-7RQvcY<5+i@Q67;O-XOF2OYr2n2#V!5uC>5Ba^_Z%@^~YkKC(eCevED5`*m zb5HHP&z7~;`J8iexi=8x(Fd@FWe4807!=dIU+WkOXBgs^**!zamd?Y)b00W4>(ouw zZq)1exYdTp;Lbcrr;rf!06W8?GaH)#c=of#6vQmcKC{*$vxk1%ZujWH1uUAVpX8vT zg!y4ZSZmsb^k)lT~pXWJ2p28}taO2TzGw&2# zjkBL#&q%3$A!)^&;d{I~Ub6n4&v1)IG`^Dx-4(8{EUs>(ziIjave zJKS(L_?9!Slc(>pIcBgq;_!NG_{}1I-d5j_6{i}F>`F9NSVz>I|TGAs8_UHO@zJ!D6bwdw-Kuye5Yq-f&3U1fc(H! zc)BZf3zAI9&MX~LFD?#Gt-eQ8XY?Aq^x%wpoW8ffISlpXS+QBCA$FMJNCR?PX4khW z#n7VqRB%L`pEdv?l2PJqw=W)T+UjMyi1xGiVZHNq*m)UXmvGwi4rj37cO=|!Jk5;( z+SvZm*b(jWkCM_e*plrZ|K9KIM_W(~&aL=scK`^CHNi_Hjk;%0ppSedwt5 zltYPT@u+x6K&_c{gcQ>lu@Ya5Qwor zMP+CHAVt`|WcZyU;T16c%6Rk}nTo9`ez{wA&img^!{(Z!Vw;=s-fc8{zexU*utJET zrEc4Yd@%xkX)t5LMdk@yX>|&^?Ciak)u?!#T&l^;fiv_lx0eJV<+Q{D`r+@Ua)Df+ zq!H0ELc$F@{dm|r&@87CPx(H+dhq;cvZ67R#Ndq=x;z<2T)EdF>U9LtFYot;kE6-2 z` z7RmLT$Usc^bkw3VnO!nfrbI!F3-(1fLPxXpt^F6GThU%KZj!?KpA@l+${^4`ezBth zfCVAq!qGcto-58un2p8ETJ)`$z;>rQOn3@GwGUd|cuQ@de1g_;8qvSey>bDY?omQHMFR0MAuC90}B2kxVTBVNBJEtWEt&a=dYi@y<#f3> zfN2#~@$V*5wZand4{!l+yC`u@%idc(et>GqihjM-1oIa*Ylh@?M$KXXi^5*-IblP& zxZ-_~0=ZBl-9EoNb{~LP_;5M4c2*y-;UT-(<67xn&yk^HmFzC{JskA!t`6lR|48`# z;qT-CO*|!A*oidZ*|7pLj!a*UU1cyu|FXyz;(Xit;WwOIAsJYEw0v{RyZYx1)Kl7= zmD=y?)a?p^#^=N!!&&=2)s#;cg~0)%>2ym4j`) z4Y%=jf8&AOZ9jhd%-~d7Gw~{$HITpjmt(h|C+E6{cI@(w@7*dIJ@t}ddc^149RKe! z>7L-y+i%LJKGrVwDC>f?R#yO^f(NvIrG);hm3nnN?p``Ilv^OUR^HLn7!RG1X8 z)r<_2TtBGv0$8R5W^DsFEM;b9;YN&Pp{KbGzKTvw(?|Uwg2&~@tl}qLM*v}B&@56{ zIs9M8Ix5)@LR=Y2kL91wzi|=NNc?sMj+{WvXaf{lr(C|F&P%u#Z|p@CrgNv)>z2s5 z%0UCz=D(V!NsHv368QI>0lHnv*{iq=X1njQoMF@kC(3&K^Gr#xv_B|97t+d;$ejv* zgdkWHT7jp_VM_g>&Y?|Z%G-is5^FCUQ_Ek}z_1iGnAyG{fQ2%@OvObotMX94yxzq? zp$E8sK7BlD4ME!NQm$@IcrK+{p@&l@D<9{*d`L$m1D@3;VE2u6%yA~j9su1F|ELqg z`0B22gLvW{{*n}7f6ve2x46V53w~;-S!V7gs>UvxnRK!$vDWmd|SPIrOh2y|#k zoje6bCuFNMY+TcFCYQMmgkEtccpLJ>MQsKi2OHE*^EdE;c7ZI9H+a>Z%8-NIy{=9iwRBUXUx{4dN zYPrw<2yeUuhFvLu*!cbh@xILQeH^u$;Dh8JVPNXqKx5G3nY~h=esCGgF?V4eLX*e+ zjK`y`M4P59?(lOp8|LxdK(mAjC49KxS#k8XMUTwQ$-t48=%pH^t4_XoKogKAxIbTS z6U|!Z$x4vnRy80>#03QV-L$(o=}OG|-3aDhgnyELSH7WVz|Ek7?aZ5mD{Qw8X>KHX z@kKe^&U?p0T};LQBnVOYl^*0`6aOw6Uy9~7p1Cl=XS(qlUcw%r85Ad;f>==$WT9O? z)**oR#%>NXI8-fz08#gG%~Xj5W0|_9hduPe{(gV#iQ9#qoy1+w+OvC~Kf?q1Wy)5M zz_@EaHe$9jZ6&xYS?{0By5BM2+W8>C1G2od#;(_sf4WiRn{y1;sa64Z*R_tHu=WI; zZwigNh~gT39NAXDzlDn$(zNMGOkaM%d#u?bVAhXkGFxWr?$+yUhr;Bunhq*^cHsFu zj>fWXn~7hC-Q0-5|AdrYLDvIc{Isd_>Qw`t(IGj_FX}EIpjCFqywtb+NPgTlJ(Hjt zimt`acYMUSFpSC9w*XkS33I!z+ifP!=;i-NU<63x2?u9@V>xUlDmf}o&7w-1JF*{Z zN8$-*L2J}xbNCM6F+H*+R`;LfIsMrWr>^(8h97#ynGgy^P6zB0<4>^M!#c{x=Y#N~ zm!np>IBI+VUnE}Q6|-a-!p0S?zQganQ}eiZ*>rY{%6YfB55cC}^jMFxv4ZWd$>-x@ zgSN5oc!j$~d9e%D^I6FvrdEyhA>aEVYf$HhXMBfr0L2RfZh+~9K1r4~7Y9M;dcFq6 zU9m?4K+9nMPFE6a-9~m_DmE^nFaJS?GVY{Y<(8#-IVq@U-fF@y zv3mNm@byi(6qVXh(kDc%o1a)h*r@MeKNQT;l$qV_H+J<0d*_g(;SFP+<%UQ+K27|7 z_)~}obGAO=KTVDas*xlOY?Z^MlpwI6@`_r`2p=g94xCnDGD3li{VJLJD!jBvhlJ@U zvviV!qJ+ho742#*Vy&P)ukYMduRY1dBgWehuYM|?)w!8~2e zAlNRMfU_w$>}$AHP_EE?bRR8$wu&%hfrvM9TW?P_^CB4cV$6uPf%N$)IC=cqePWK} z1c$~DhdzFL{DBb=95vAwX1(v<6P;QSV1E-j?}dHEyOFJ|cf~H0zWpZ6GuhiPdYn`3 zV5V@{fv&^TLl1g`Swm%R#O}Yny48cfvs(N#qD5#?%&4E&5GuXeeqsqtN$lCQ;hs&$ z-svK0&9qwED{sMF!+j;Zd~M%kM-gPL=HnZxacp+EumCkmp0#?PQMcfH2S7QLs$HWu zc%Biz66ao}-4xH?DWV>rDz(N8X;y<-r?+;leAh`E zGUP`&u|>KzgK!ZJY0v};Z$3j*zT`T6yLyX%8?pp~w`B4C3B!;m6L|m#4dayS;^>b* zvJOt~RV(#PSVvM1S@tFX@c6J`3X;U(Z5RF3$Mpz_ywf_X8TR-_Nk7D@femj_I>-1s z3?{fyOAXeHLN4@|r;tQm+qSR#BGU{wj`iU*cXz7F!n*rav_0!)w7 za_J?#g;x*(yRph`7D8|mkn7rubKN7@&-Z08``O!RG8x)HoE|X`mTR?^G^R+ZX|gcz zm?Zi}RPzS-eS}k}Kp)8xzevB6nr)~=Szf2;&m9ZCCa?yVUvtN~K`**Ecu!XG$y{E( zf(*^t{!)iJ`)-I^L$)7J&&GXNunw17`mPrA6nd*<+&LKt@1nGvq)#R?AS*bTsykk3W5^)Y?ckjl<2$W%0`eFSh~UpwXaPnt;LLr3Hn_0>iZx7bj2y z_@G)NKFA?vjJj-XBbzE~9P)2nzulr3m0<1lP{v!V`TXKY1p7@yNo^Y)5SOMom2kP8 z?*OWZK|&c77ob8AQDLf_TyfCez^CO3ZPxfu`z@w#GfUTd0+1$X8#%rsdw=ReKQe*& zXKwf_0^{fB(|NES3k0&g`avQsv36Udz6h5bspY?j zQJc@G*P|^buyqI2;zDviJ_%HvDy-r0HMxsmdP&E+t_f985$V(zR&n6$ZLRYHP%c-i zN6meF%+rpUAC3v5rkGr3KAMMo)356-=1;8Ox6@y}>yv+^;3;^*u`Tj@4{oaDME9f~Or2c8kD*hKUPm$<#@>*w) zC^YZuI?%(C1t+l|yD_WgSQ+n4JU$9H5IHbLE~X+@lkSAssGEN3=EB-2z7ghgbzt@S z`U_`>l{(zxMl($M8KGHyjl*O+zV~F3T!H~MpEsdQwM@;qI^|~=?%y;AxGRY8Dy!#qhu_(*#N@w)f9=qGOaqNdl zkOt!PUF|!gbIOtYOq1t7EyntB9B4DTi<*9-24TP}Z1Jv#SBAadXLWOuSZj1)+Bv=c zsS7$z+=p-R7=#*~EWZYtl$xg2O3*lL1SKyZBA4w_Gc8NM63kf}DUr;XE5|FP@O}uv{iv97AIIhAOB~>5T7yDE@U-oDc$T zi7($NqXvp3?aXL6wOM?JXmqQJBXY~i1#_`M2HF-{UCM7+My87wad(-XSVgJaX&5ul z-VjVjeBQ>=yC0|$K`JXh?MM+W`?QqSG{9!acq&%K2y9^MR`St1nd9lqqactpXF6AB8 z%Ym9h{6T>*pO&%SSI`%N8{1}shX-H7>{B9wRr!}RMD)8xu6Z7o%@vMy+CEjO#^(+@ zPcDF+(A=R-E5p`^S@G`0mr{R2i)|C(2UR#xI8?PV=?X_23T|Kb(;(5Rg{wK;A@=+) zH2`;2zjL*Nynw=?SJ%N|Mm?ay<_a#q#AAR=P%%O65o!=e5`_NIotm1k?}-fLJ}%OCdgObjK!LFOA`8$}Y3xfgWU$ybU&1gP|;Efk@znDA~oZN8`tw0PM3^K`szsgD)u;Rf4A2X zP^#RLcl45C5AFnVd(`3G@<1ygdZ3GMDEkiUFcpVK=qavpI1VJWMs=e7CI6>LG$5e!m|44uSlU=bn$1If!V6}8eyg>$b0tSwG82cPS zNYzIz!-&w2Jf|C}!7rtkGI)u)6BG&g)nwzS%k&soc&E4eblXbQ-e7r;>Z=VN%BL|#Gha^GEvVq#?@LQD zlT4eWQAm5{wsHX&$Ba#%bOmxgaoQ2Jca`#)LNvh_S?p6YYZGDSp zt?nBrMP~iuovp^#*qPUvsS<=My1CkwYmFf zc@y88?vEU|A9BBpGUCo1LVt8xBL)D{O>VS&#t4({ZgxvKxS?0hg&V5zU=m)(qQN&e z7y!THQc7(G5Z3&7u=IgyT>YPMW`OE@7zQo*b1hlvZ-mre)uB|%`Fi)mR5GCvO!Ux# z2R0?O0;no=)U9zm2^2y+TTBuG4g-2Y?iyX~7~L;>OFR9h*m0aI3OtjeI;J7JN1qvU zi?wd;1ZgQNFHwwwBdRWL$2~zPWsG)%L03V8r}z}K$7E=oT+C(EN<4^Ko7X?-!$L_J zWdK6AM!CeHp%9h>KYdoUb8%FDTB4*ZPTB;<^&fw|caE9{)r|ZG1FA=C>S&<;J*U>p zxq(B%-~&SpIJCn--D{#KFYItTd2SCZGBbsBV=M8zr(0W( z{Khv@uukE*+UNe)orgvTlBi|{7v$^^8?q@7a{1bG#|Oqurp3zi)v~^rQTEC z`$q&YpDwqkIlT4d?1X1Y4mn}@VZ!URQkTrA6Ya#VHu%@+$UdRvWQ@rN7qWjk1ib7W>ByJ>sulU8$h zQop^#T@3m{;vHH2>I#o+BBxkM50H&~=QDl?dT9sHI4p-!0vkCI?Y6w>uQIJ-tvaXq zcZ+;$SQ2^_7jK4-nrM>n;-0&G!8OlwHmhZ4dsx}5Wc+#G@M0WjXvX|3jfJ95)4O+y zbJQrix(hN{eqU?BbmoE-VgN+!rNe2`$_pTx#A7!dcgRq+MGJC-D0_Tk04AEwO$5y? z^xY;pP0I{jHi3U7gUIFn%S0n=aEJoEi{wY)E))%wv*TwcDvU_zI4aOBquN0miQxSy z*w&vM4No$jG%_~kvu3UOhc^nSRhk8MPe>-X$@vXO3R!l}JM$oL1Es95&geX1DQ#U9%V>)s>Pqc4D zx-j>{soFIOAiFCi3Fz!;z$Z4JtVb5p;nnsK#UZd{2_yqzJw3lPN`}4NdrZ3Y`?jX# zOewuqW=mC+gG_0QgF>%$rB%Ay{4;To8N(=Cuq+K!`H02h@xSwaDh3n?_(F>1>gEs| zux3$S35m94vWHTg6d6TgPPuK+RDVBtP%AauQX0~Jh(Q|?8Jt}wg+DUvyuXQb|E8rS zEuOSr;|MqB&Y~FD!ExsDpNjYb`_=KUNoK*{p;ZB$k&qVb@$$t z;ZmT>$jdml`M}h}5UbZqa!1Q83`eDS}DO_lLFJh6TQHh+A-LV;?m71I=`ow<>{zaQ%OyR-2ZZ6(u`@Y94}aZ zIy>Y`rEE4fUeQ$EU=*K0c|$sH$K1)LeIVI@#M4wcn%#|*Eyn5es|#Bw9d?LPI3N{o z(eXB7tERId4bosmOcTnQC1@lA#r!iV3l;h9Ouhx9&UojLmAE+WKy~G|?88D)Vb1i0q%b zgZO_^cL>Ll3%q;>4+VW&BPnSA<#0h*mJtZv?U{KGMc#uwZaQ{j$=#aJydp;0`kq#7aKfKn!yc@2D^Mw6=sEe z{cE3xAu`lXjFyU$!Dcn2!^aKfxBkv|=CwWyMSYW;Q(K2CPQk*9O3ddwun43#Dy~(n z9Eq$CM0sI2QzCI#4_!!wp>2BKPoiZG|K=)f#hXs-0L#D-CV(dvYkPKu67j1 zM+kt=mr{e4GzRlmBwVUqWs^#Fghr>HV2?hXV_X%w;`y3Z1mJ4Ba1Gs1KxbJhCogjw#j#LQ3#oXMAm<%SHJt;< z$BhuwfIq5tB+=}<8r>2V3K~iXQc)sJSDBwqySEQi93pfo(&T>bsEVFXo39#_Dwj1p za{(?`Hqb1!Kubs)b$W<+d3CXXiE$lqO*oFK3Y{yp<;lF7=HT_#2TqK|^H?cY_gj#9 z(pLpf;ragO`y$qfBPN+zMv!^0y}-%~_4HAs0E~t=rSF!{7NoId=Y>ijh;Gi$v|;UT{PKPKMJ*tFx6S(&;ExauZi=lfl9XA2CIjj;?9MP6 z0j{E;t{R4g1w-fgn!ukRNZ*k1D&|**ty7lVIe08-)lggXmX(u>?mXHu(%RS)EtEF@ zblvI5G8xqw&0x}Nbc5bH8$CZ+#{BW$(~%aXCi%Nb6h>5Z#BRUU+q>)Tu9kf)sPZz3 z36D4B(kbgS>S~$=zS>&^h!_60o`yWvy}C^u4{c>u_3=BlS;`GnRqCoEp0-0BJU3y# zjE2}pkO;s}o-=&t+$tnH0icr+_&lg&C8PxR$B@;zG6fHKxE8A!N37qG65cQ+{kBgr zzsR}wTJ!;OL}Z$fZNT7zwg-KgZAA=RceC!&r#GCOWc)`hlKTi+%5eyb^5Riy%Z97J zbPG`mL-068RO$&P%z(2No#7;|XEnNXe!Z;&g*}9#$1WvF4GrC#>bn(*C~1!3E!`f?#=-1Q6)f9JYv5h{kGS(Em!5)+OusJY7QHlPQd&dDU&hG z@;kXLKVD?v=4N%kNYNykg-(vZ7^X$r`2vd8ml^1rNa5(5vxPYUb-K{!NkO#68V7=} z1^)mR%T=ND84A1ekBce!Ar7-5Hq9H^lvlKzl{DW;!Bz~W*y;v|2XeGo0IlGVyxCYw zAauHcsn8F&0|jJ8W{@k*ac8Hx)8)A8jPd(&^AX3at*Ij4C-X~WLO?uZVF+4qtQVXF z+{2;lPxx3El&cVFfrOJI?9OA$w=YMe!9Z805a$yT1*Bw{i8OMreVqa`iO$rWfvRGc z6eR-bXR6(~^fQwT!==_wmjx(HXAgi_v(JD#`#6fyvVS+un7@_l>=p$uBey%^E4RLq zg0}8t0}3okKdopb2(9PxF7(gxTVov_5-UunS1YYlTwa7*EEu4!Oyr1CdT=SJtUb&j zYgwyKe8DYMRRP760!XN0fTerR;M@)XPL+QFk^_S?%J;1}cV$<#wy_gZuHxBaqQk&E z_iqs?$a*Mr*Y(tlPA~d*Gy1=8KzrW($;|;BOFqMJZW3zmN_cNAt}>@!$AW#A<$8M2 z`Rx_9?u8%`oB?Wc=H4j4&s>`0su+G))p{kQK`$!!0%v=sTK{EQApNLkF`xA z;F-4~#+YjKsDPeAdkWCo-fb`}oqLv70SMzJMQwWTaoWlIK3o*5Y8iI9!)aJB8ra)Z zGis0O=Tl?~<4}oS+9V=v_S?3GEpz7ajJ#p2ext^sRU%G^Mfg1X5NOSW)l>85R`Tob zaoVz+o-o!>U}f*U-485+2TN(X59ComRvlg(vky2E-v>n~N_;UUFE`PQ#`n#t-u-Q> zTY&=Sok4qZhX4BY3_RKgceucs@gb)hXN{v{?RQtfOdI8teJBhgt@7F*+7Z#TzW63$ zS0A$-@t9T=x2QLlk=MpyIIX7-7)s$TVa1D*WY^d-9`tl`M-29tO10OHlhAATWl`pU-x< zY&EU?fS3o0Jh9;Z4TlEDS~q^z%fr-uxbp} zuJIKN=0vnp}}@sUuB=Yh+T>Nzl^T zE}6uW*t0t(O`Pnc28{6dyN1D6mLYws`AYKD-!Ym!XM>tmbAkLe4Xx3e_JEKEez>`_ zL`7#K!=v@6M6xwkl85X4Dbi%8w2v}C?$T%0kp;M`=w`s35Qj)nk1hcE@2&eH^ryby zck}j_j!17X!h=eBQNkm5Ucdvo4Xoi@gGhYIt1meF?V9PuB}fKMQ-a6lR4l{DrFS)Q+g*TI(5qUh)~P+GDbt zAn6dhGxpFubJmvWCpp4=x*Vx9&AXKyg%S?+qz6m!<4^@d9+_&08ZI`_Xu+59C8W-H z72t+9sNI>dt0rUII_?aMzi$Js))pk{UFvCAwpz7JNcUL&?!CR|b1LOvOwM(mU)Ds4 zWY8!pFWW1lkbN>Mw#m)^3*1umGT6Z$%%rD`v!#BPamPpR@_l_f#dEygS_eRqm@$i1 z!VmcqY;!9Nt6Ki~g)+krC|-pJYUbVSjujwYGVa6G6;qrE zxo2fV=iIt+S0;vQa?VVHASpM8(hjb!B4f#^X595f#%fl4zcp7bt*Jeu5yJvl+O;LY z!Mg(nf==(Dd2b3_OWW5-(Xo`B{iM_H4+*r|`;0Zm+L2~QY&2gbNO}y)>($3Zq*iY5 zJZEH_?k<}yc_aIq;aZZ~BMj;MW^?(Rk0M&Iu;hUmP(q@tQUaI3BT}Mv&2v>dtDTxe zbR`p4L$hLJ)k^3vHp*0x=WlsK)Tr;`tB1vG09LC&?~6GmQ{-{`3|PHpv#b+tPjFhNMg3fNmF zF$8@zb9_%1xsu*#p)(Gf!k2|dtK~o)kWTInOW@WlV#_$(&7eczUfa(VwPksd91eZ( zX(jlkdW3b4RKR)&bSP#jHsYgg11qQ)y#OpJGGq@x0Ya-vO|KaNaTp@3>s@O;7$Q+P z9)G?PxQI%9AHVr4TfF;T0IXEC4yw3z-}#M;dHdJ}%b494*XzytZzR#@G>Q>PrB7ln zQ|;bvl=e~n2dh#Mx4XsJK5@1qs#TOuZCkQ@<(GhP?&1Qy-CV~LIT=urRfNTjI;%kqK~gQZVJ8g$!z6D@04$}aJ~lJ5Z!f24ul_kvrm zm4bM$fDRN9qwrMTQJi#lCbdbe<*l9Q7-3J9ZJe$?rv;G!Y$I#5R(euq$QwgN7WnYD zZ~L(lEjH<|4u8Z7`5akll)vt<**2keOpZx(!F!)Ael?3vFwPY>>aOmOs_Jj?`z4*a zP_wJdLy~}-;K}nWURSo*>OHbk@Iw_vUi`Z=j`4Y=XW!mdd)^m0k(SK4&bzw1^#nXy z2c5zH6>v`5JI>$=!QHRv=;Z0OI$qT?-3&+*ZO${kZ(8;#oU)umqg?o$h!~}5q(7Lx zMc#l~{_PfrF%eXvpeJaL?6sbi!|F~3OZM(^^o}ExG!Exc9{3AJJ)2hN3&6c?*&$bz zGl48@Tn+kW8I<<^Bt%Cq%Oi&Yv`qQ19YrCioO|Q;>aGQhj+g~|f9}$EKNF>yMq)Z7 zGRhQ@cAJKWQ42AiA_0u;15TTfwL)GvHM5&W#sV}JZ$X(G>z*yS0~Y2XIgxdR4roqT5C)L$L-rM%tqbU zYC}V*GAxLQ3^>mj#($f5-7DJtPJn5Lcqk*MWi~_#CFubtKSy}A+N7zU_wGeRwdTH4 z$G&)%Lc3<$U47kGOv}c_r4Y`|jm{FTyT7AOZ^bjyr~(f?WY{k~rR)_-6D6byerUBt=kS`K9oKmQzeBPW-o)o7%6hhyt`+L8(r7}jPf zu%h$c&_^L%+NFggvBcBj>t5ex=?8irT*4Mo4tLWJ376VxY=U=t8(!OsNiCfvPVI;}|d@myLvfdj|TmIbi6=)^uKCm7Y?wLla)Ti}251B8Un*7s6S76e#-p zYClwQRB4DBJ_rs}g8Luu!3AFSTs0NfACBtBNNm?LBfA~tX-*cSpE5WG(X| zOZc{CZ8MqZDF>+}@M7(oxtKBNTq~Z~1l+V$X2q@ZHuI;~g8W+FQDGy)n55?B)cgBlw8{eBh`+0-7ZYA?b^WuW`|rT=6lDAru;XAv#f&Fv zYu4?le^5amxt~@u?`M^M(~`0np!{av#w>K~mMOo2?Me}(RU-c0oa z4Ial}U6mFp3yHqMA*FlFyXxS=5mXj|JmXH+z?(7;Wg`wvR)0~LT~*%YVn*^MAP~*dxSFE z-FZ&=lC&E|Tx}Vbmw%1oW2hh0<>1H9mVfQT-{TDUK9hptghN4Xt-N=}opl04^~2$> zdB9EouX+6cV;&nI*i7fcCZ3KBfOjGDYPR>ioexk8_?E1pzvDV*Nq?1?eb;N-yc$9~ zHkG@0m}uW|k#alc23_}gILmGWDwR2Tq5Jd0+5s}AY3xuk&yw4vE9f?v_qj6H+34F_ z=bsOiQxb16?S{P=08gBI3JzrLJ48(nH+uszyQqM-&*1>p+$g{Pe~u72`%Q-kOLwMe_h-{#Jd%wNNFOrkm3yIevD%?B&D zQgrNg?3!1RxvmzR+jQ(Zrseq#>ZxOd?v8sFON+C1tJ)5Eo7N(jVqmswz2S%Zb>prY z2Hp)`8}U*L^Y)zwTj}iFF3FUR`iFI^=c1#@qFF5YIrNs-D#hF`n?I)5~y|{|a zrA1$cJJ*@2+rQ3x?o}?f?p4kIx;3X?cA3%h(QVl&N?#CqyjJA8-bidi!qfZ~%Vcmf znR@pa8tixsEVQr<*r!EOw}n+Uo6aNeBWJnSEd{qz#e$oHkJsaEgccc&y?mL$?rCNz zC*u~Iz$bPVx?9ESQ1-oponTT6nVd|@u^#O8G#eF0oHS1tS?HSR7GUD*yXVfECdxap zUhF9P&iTqEjE~3=+qXMzaSt6q-omzmf@{eYq81-v1458=miix}hZqL10Q{N?iYSM} zOt+<-P83|a_YT?DqZSKV7MaOf%c=UFLz$4%p~ElMRgUK(L15SsK#H=fZ1_bac(iBm z*vbc3)`x26wQ%~F5~=cxbXWZk;#s^-m9h};(;@mrkQB-4aucB0v&d50i{s-{Kdr7a zeNxM(U#xI01I0%n{S-HE+qAe)R$jGRlx9vO6qC$=V<7c20SNOVPX-D^eZF`g&bH#d zvaTn=)IA6;{?&oHpTR!LcSG>&6A! zytdLT+LejpPXQA=>D&b48t@<^tY&!~@~j;*pMYz@2!azxdlhvtTmjVm6h0LFR5uU6 z4qZ~QZRSP3J{64s2d%vwe*hg50m52>r2ZJK(gBYN9%6XuH>R zBYqB##eeJ<`|#zRWdi=CyZy4qHtEqPe-q?x>sL7qYx~b+#NUvR6{XwpveLbV!q)7* z4$(Pd3Bq!2FRFbH+&hmtuJpNprFb4sS)}R#cR?^I;1eu3K3iQTjKt=5(9Vr{>6|oc zo^HDUtc;+jFLfCo)4I*$wV%#;}O z+ZA$kTyLfOJeT}jpum1|0ho}B!Tqj=_pSFIF>)eGPG{c?b?m5wO|H)CXty8Nhi;vF zuCqp=tJy6=zM&_KK0zv3y~H_?cUoZu-lO^@>@rh`~<4?r$ve zS+J{`$?yFQiF)yv=%9yyZ|OtOFt%TJ-knyi*q-xbVz&~#C=lN)$2Q3JzSKG@I5{0* z^^#)m$0yJ4*$^f{X*MNFCzSwLf z9yc}HqE$h!b#f0l(pQpT+LpdQ`^0nT5r>H?JynL_I!fovg^8hg1L}{Zs0FX7{~oDI zI`C$lE;1+J2?Vh#S*FU3V3eo@qb|*8VgImr4QzwQ^J1Zv!W8Xc;sp@1`4`Kg)en=s z8sccZDL_aalqzwIt})D!`b>MX6YG1o?m%Pd9yKk5cac=l7G5pBsxlLZB{(Z3tOeAX z^RKHTI6gn4ni==GtW!0@3gn<7jE_;*1{wI?{LF4$bX#7Onq6-ONsc)$YUq3PVbtC3 z*Twdn1c)LM4m>PB-e>bYz}BV0?NAHF_ozN84U3mj8&j#o5WekS=F$o~rYU49EIVmV ze(khhJJ9Mp#xo!V%W@iKXjx;5FA@)-M9lQL*@1rVmR_#7#Z?};+5&vyc+Kt4F+6dz zll^X-#z(Qd(4imSxX_LGkBI%A#pXEuHcP#>*VAV|;doCxhd#c+1Nl&t;UxQ5Y)7lk z6<%H0qzPVQGMT>K&ORcTcX*bw!tY0M3tNx9MLpIsPb>UHYjU7}69CS_F4v*Ip@|_d zH69wancQ8Mc}m>&_yCU%0zA4k&IjNB=C4OT;$A4enC86ynbP5obCIpDG$Zg&s?#d- z z&U0#^9sk5ay%Vq)U%aOH2I$0(JsWV)&8i$+35vvLmhIucs5~=KuQVB$pj3mDtyGve zwf3c*XNQ5Wnzp`J-@L2`;#EhwaRh_$G$SoE0qvQ{lYzRC-_8y=RqIW=lX&~}!ioFg zTtvWRIQ@Gt`It2z_ou0y`}rD<5tOG*C9vU>bbDxvBK3Q8L#6_=en*RjY4tOBTv7gs zru{VOUMKxm$q~NCV{j%|uB^jDA$;Mj$1p^=mr zN-TzJnj2_R4c0|85T0R@P)JkC2ff{1f7aDb0l@<8I{s({BFMdd#>+|n59Bu=67Zdo zmS|^BpGQ%PtA*}#ME)Y(X3lu+Fq@}1CiHk06Xph^5B@~LGJBM0k!D@ha(DkRl-1`X za>5qnt0qWmj8C$j$ptl%$CW03@Mjg+`by+5G3xtJCB32hia*MY(2TxUrM=MCt{=q? zl6Qkrf@_(VKoD;MHIEdKzV2xs_0sy3vHQzWmXCy<<%FYK8Yoh5cThnkeEwf0%UgvM zg{Bp4e=1h~zPBjNd;Dy{r+?#~TJ|aHS?O!7b6gQKuTKJxcV{fivnobkH5Csmq?bzV@b>BWd;84nFxm2`Hi0ew_)E(lBxv5WEqvB&csJZU8a;D z$Z!lIFN8>G;muI7?`2x&p7JT+G--^?(pHCX1{E zg}{H#9O1qJjAZ$78W9Ik>M$AqS`)Z-bc1l8AWxa_@l}DR6#p=;0n27}_)vkxYewiH zkedkJoHupeS3VwUvQJ#6FW(i#KA0``5j-DwicCxK^wyZ={7`%8hi;1SbNKcu{S$H`FwBZ%rLJb0g~qYZgD|N8dGIvw^TmI6XasnrdY?vWS7V=dAOL2yHy2# zizGnu-9ieB**77Js6GepK2F5?Ol6Bje4GpLqT&ucNZ&BE4G|uC>&)*Q{B&CB(`;+? z@X$<5hn=EMg=OzAk8pQqQ1GNEk>`n&3W;*XN0CUOJI;?ZeO%lrF2LH(Lj4`Kf%<0( zscK)*1_?xT0tp7IAFnY3Qqwg-dP`*I9!E?uAa2@%@8yozn%)=F_sxBulQgF2>uzbz z)HQL8-V5cjFx2z;Vjp_=f^>K4+Eej$9sGq-9u#(+0q&K?o8FhCD4*rA9ORdS>HTtp-c=R6tpC`$X2Eqf z_;rn`6oFk5XEVSNL4kmGc0|6A!2hdrVZDIC_g7LusG>BMgx6y42#}|#XkQwD^kt%@ zrcgk9N$*a;U0^Y%IjG;NfTK@$bH?RsHA*1T1{#4W33?cf1AygJR+)(Bkib1(h1n52DyVa!l2^(@=WX};FBm9ikbd+l&-izjYFBKx0qbqOPV5K zQo5_eVL!3F@#0^AV#9{(QCmzqDqe_%lJSMKjAcLGS%ikkKQ;#B4D0uxkU01~icv-) zc7M{IItTI+z6CezkJpDz@>ISV*iT$t9&H0qkz(&FDybYD;F0ln{{o z1g64>jA($WE*$8uOl1md<*O}b7iaGlH|`Yw+JC4Ex(}U_F{i2d!VjbvOFE!@kx6UP z^=BCG!>boAZ!OnSI%_N5?Lp(sAqrj#& z9tD&#$QoW{G?w0ygP|E*N;5$$)37~U` zpL`o-IL!C^7fJycEfKpGc7m3iW|zC-6z#3hn_qFVNda}BiJHb(>PL0Jxc+p7Vp8=y zk9Urm5zkaP?bSe}@73-_%VS~yeDpF=V#CZ{gDR)Tm>;y>+=%8&68M2`CkJ$cU6Dvj z2c5e@sLie)!1IW52v!Ph9^`t(%7^Kl(tsB1BBefTT%L}Wftpe<(9kUa`U4xAedt;} zi%TJzNVBI;6L!aU5TqJ@TTXF&Vk17AP$a}$uxA!cEGch3{k=+TiedR%O`I#ZcvcBg z$O0<74*Zz$iE3~!@RxbUfc}V4E(`X%G%pgTLCVrp5qAk)$yfB7uR(45@`^!h%!d7) zI^^>D1=5QLX@rThUV_Z~T(Y|sBux%P!YlpZfo>=YMyrHjW43CL22Tgl&q2X1Z!t+F z-t?C!Ze0vs`HbKg^t>g^2HoxWiVLmBbvGLqhQYMWNnYLR_h)~-l>*vwZ~q?*r}rk7 z@IGZsU1h|jmgi%XK|v2TCT zR)400h73@kWA6kyrVnHvFjq)aT$*$265aGV+8%m39NRZHJhxu3) zmJo;&*+y>Fwuq0yw9lJQJH#!*gayzj*eDx84H7R2jdA1k@f-~wGJP-3}qFCMT%$A$(64=PW+PmL|gt$;my8@^bn2#;ZJOO?DnvddKFcO1O-(v zV*rJymL~QsHVgBa5^R3l_rbYoFc+i@B3GNNsZ0*qf&&2J7gruV+H2>8j)R6)iiBWM_ zvFY@S0Vb+_?A1g}jTB72mj+sOlu>7uQfymfVOgU{#%uKrTH2;HJc99XNTEpR6Jh%x ze%F1??7ZI@`1@MnSq-m@zuNwg0u7`9$@OaeV6KBf!6Dw_ml^d&H}?40{evhmH04#J z+WPg%D6nqv?x4oS&_3RG90Nxv_vNgKw5}hi*%s> z=n-BGk_0-CnbJT9auF?wLwow((W6BdB}H!` zVe~EpiC%(4?>z+XKIe5_&wZcQhxdA(wen%s%37FX-?#q%ZAboBb_Pp{%r!V{^@Y3g zdR=R7YL%k-3HK&*#8=s@h!^h`Xpp=cbt=48#|oj@hmb+n1G;jVt`M7^ z?c)!no2RpW;UxX^dGk|mVURM)rFv3q@nUBjwvPG$CVdUqe^Yr0&fX3mtNs&!bT$~ z^ex{OQRc~h`%jrzBBgyvsbAquOC{0Y5;d|wNXZ&{t`X&7G% zUrQjN(3KL)+%o@3at1b=lZyWm_ia;D(MgV*;1^J(l;=5pHaL@07!dco`jl?Nj45VR zY>Q>#S}0Km0#lnO^G3kJ7b|>=Y9~RrW#6ZY(@|*}eLMSXPJQ-;tA50OHFSaN{06YZ zEMA-?S~_q~gbBU$6e#6r*Nbj{(&rhz9|-k_xVAnFQaD8YlAbpj=>{9_4Ie-Ymm5(r zl&&~$epiRTg{yemM8390nJJq0=6gEnlJK*K9&~}w%W4#i5k}UtADy;-PrT2z;OC#d zl?$7D>Bg~G*q6c2DR80bV}{*A+Q3pOoO#{!nCy;cL}ExsX;AExk5iM` zZWkypR%Dt-K}epOGp9DfV^ktPaVQA6aWAg*HN*qwToKJGAtG=y?zdP#%4mpN?QQ=N zT`L_`MLDh*vB15$rA)^1&Qjw1qA!%2LnM$|)j&%*X_@v&qN@OMD_e`NU_1KkPr|;Zbv3Sm#t!JsE>@&lJD{g&4j}?&R zv?B~i%#8c6#<^6w)Up{e)hPV?bnY^!tWxIoJG&1pWI~pEuZLC}JrlHycGg1``wZYr za8L*o1=9+;1nUg-qx<$F-^N7rJ6lB3Dd-?(y_hS-^~nky7fv_WLHpfk-aa@E1Y2(0 ze8v;cI!f0gAk+VS)UTl|e#6^ZqgmibH~COSOxdt-9-r0v&qew2kl?B0*bM#w_L-f_>QK=t5K z`Z?tMa*$feJjydlM>v%INmFQhC9MdN<)0TSoJet~KiXc8GgufXreo{+QA zmfz5YGQywD*@P@yFLPruqAXdX9JbrC|{O!@MQjt#3!-l?UYx9`*_-7vNj zJ7LOw7w#^!llX)zx>G;QlAzP*#1qp{#c4@pNzl)h=1>tH5THDYc8zo;*^#7i*)SPc zwJ(M{w}_WJo9r_5piB4N^cT_-IPXjx47ZKiW_ouV5ue<8C!R3Rr!7FV0xH0?n1?ES zBQXh~4HK7>Geq2Y9Lg~qbAHO`oJl(ue zMH@)3p7mQgTKm465T$g7Y(Meb%+ljmO`@qx3PP+M^Y0}3cQYkbGiwrfw-1q5jv$%( zZ+hE>r78`693CR)R;aDtFDPP6{lMKmYL!75kwQI`6;9QEbHt`dAw@o{y^V`H!yu-N zt}QNQhmGG#GbezfeRLbDvp28YlP-jbjrz{KWN~%3O8LjnI`CsPF!tZIUmj)t7Ca@#oH4#FxXQ5{l#;ag4k?ML<8ZvKP9Or@^59|K(na-t9%stwcI`G&p?sL8SRyggIJ zf2r3-0>HWD&CBVtao!41W}-{ZyVmMrG1EXKz#p&D8k}N&O5#dRK-g9@){lxttY{wE`wnyfiAQKE%OsKwVFOZ(&ED1yvXTx6-XSL z@R*$VOLl1TAK!mL{q9W*jfM?s1vz0(x5*!^@lX+iq|DabzQgYQ$nnWM-+B1_Af z=B`+BA816BCPl;+5rEaQ-$R5#Z%1H`FsHf^B%%s2_D{_oH7H!_6kCk#E)v4v_eFK# zwFn`Kv<439QXEN+vZPfQJ-V#OOfQh zp5<0iNRGJziA|d+Nwgu+ClqO4+%4~mlw;MO*ViaF2SgnNT8z`scTQk?tc&jZ@{0rA zeN1_mJIW(1SKio-+qL*u6Pqx#9tDacFs*1y|JJyH3e>jeh_fqc7Zku|0ypciF4U!{BbyD2YuGrUxT9+idb8$_Qxxuk(^6PVf7 z1AdQ1=C;A)^N#DyBR5n!A7anoIbYOK?K{MTMrp|tX96wb$U~S%VX0d{CQC{8Z^Fhs z#w`MS&h$r+5$_wTcgW)(U* zXJh&+gdsfYbX3r;;*_{|Z?i3y4BU+kB?PHWwiWMK>H>#!o1O5+7qmarr0=WFBgWUq z5j!!;4P`0%nv9lEGSUtd87fC8H$%d_E5}a&|CZ?mR8+qxZ71RUUW7X0=f?22=wnqR zp&UwI{cZM{K(*_!q@qf0UhBh8L`ukfO7G~ePw`u?5XiN8&^71G(VP<%%=w#bOr@O! zE0WJhlifmy+^694pfroDvL8QhmyC>!mM01AElUafu&f`)i5LS+~mUHRDVl^2{9aOb5nht7nwKI<;Ypzqpwlnk~;>IY&neG5Qe|th+GQUfYq5{$1_YTSpOXi0l+1Tx4qF4wmnLAsKVi z9c1Wz^&qFvh_Z0f>V=Ehd*oIMm_hz^Ce=Jy>7&wcbpGD{6cF&n?1S|BtZ zI5fsTO8G2Q1LLXi&vs0?0;{K*Q%+*!tI98C9wQI&7CGx)jCWDIRAm7PvJy0Bve}TA z&|j=}7;g6CMF&-h%*ri#>}#b{uh1hdm;X`}J`vdP-VMIB+3b5ai9S7&|1+uEbu1e? zCW#~~KwkgXsmf1Y50UaX+d$qB8P$ekx;W0Db+#h*<&WK-H%p!Nm7>F>x32h6aq~m> zW?(_W5n+x0IGVQxko)hb%&ycOt{Zz z#gb@nUtr`>nOa1t6rr5tf>{yeS>zu7#-%G0^kIn1wBIBbbRZnEw-4VowVUUI%vTAW z*??Dn%?w`M#Ax&t{#LAlVUrjLd+L+R{l7-e0)fjrHB=NFH;F|8!Of~>cG2Mb&4FR9 zg9nrQ5A2^`7+9PYVvc)queCnY*JrI@ZH2zjei39If^T4YZ~MBeKZ*t&EB1o?|U+&B2CpydW`>aD~{3{lU`udkFUsYHra=|F|QFH=OS;~Oz5-r8?*I%!1 za*nC&BrcCw?Z3=V&J)ys=!T1Y_B^*3#rAToOQDlT-~8D%=(W@N$G)Jysb*{s+1}HX z)(3SOsW=&gMg;oU1$n6{Aw9%~$LX1|oJoA>vvD1;^-%*V?3yNu3_DB5a9?%~z0k$o`yI67PRY`Wq zoaRM%skhy)h-ecHiCZ|^Q`14Ui(hd~T!U}!-7@e|o8i{yJ~Gu@NZ`ai<5zqbNd=(TK51(?d&0}tB3*h`A`!MaRDsf@6jY%X(u?!98!Tm9*ozkX_SB>#%7 zTKq_^{;rh#GRF{EIgBAZBD)hBtlZ)m{4LfB%N9>5L>MNW*4r`j2sW$!(A+8Vg#wbh zp9`rw|1HQRn5YeE4|936;40Zr@^pvRoVxG<)5D<(`e2XtoCs0!$pTj4`IMlPE@KIP z1cqyAe{RdpAe8cccT1Bec_4iu>&QcMB(J3`r2bp7!;L@8g!+qZbi~HNj+tNEf1{nO z{$y*`6C#I%32jRvDjjM|z3P<6PwzzOOqV%BO8IH`jDDDW*07^cD>bhedL9WF^|qNcNJ;pFc$vUpJp;(tb`d% zt+o6a_WL`xcSHq5IkjIx zcJ+hGXgNX82dGOQsj^VBDXZmF5w-c;+6WT;X}qs>C%?vt(#k4 z&J4=-ZiSxa92lb7V~z7ndJhF>?Ay39#SDfH>3bQMDqv|?~pZa{CzcS5Pn zhEEKS8t+bjwp=$FZFs$`36yMq|BKaPr7?3wFjuumM4=a}iqv0r$PG+h;FN=t`v1%@ za5~74zg15BBLNa%`T+@j68^&4_g7VG&tW--!u1w5UWOeP&R^xU^}kgV)ZZ)D|FOis zN038Wijw-A0C`8`n5PF0eHgi1fBFk-y3vV~`CtTIR^2}Oexgiur`jQdKVvsaYzDz! zYb5CiX4EpjOnI=Oj%c*Xw2EzrlB66Xgox;)*A)9tsyH<(C;er;nnHsV-N~!vx6f2S!IM>8qg7bl zJZI5^l(CN`?*{UHBL6Ke-De5BM{lN*8b!?;J#`ZX(XRaI2X0g>BmL-e zh+n9(NBfKzIzNw@>#4AMT)Xs`NLUCz?>mRB>>}R;Intb@SNDnN!A%kgPbrc^eVN>& zbvDy^5Hg4^Mq;V()j3=ZeatciwEn0?qKe;N`#@vXJ3`DBOJs+9TShsk{*C9|?aH>% zZLi#1?-=EeNFV+~Wp+RMQRPib`;EK%cG^4N4}ZW~lHRVu@^9``_xL{8y1nxqxn1>Y zR3TH;o6K}KFS5h68TMT~w4oPm!7VJVhWcVZT5`CA zYFI1YcIv^ES#HnTj@Lt}+9tM!E5i9%%z&$75m;WpcaUx67cmEPu8Wdf) z%b^tVEGVEl^%dMj`M*U?`R_nP$A=4_m}gx_4)PYHv}}dEJZ>>nFfTt#rF=Z5*K=4^ zZaGF8R#jZ%%!0=dhsl$!@RL@#QhzXFbV}0+h9yMqlTM$Djm$%6|Hv3}K^yjm)cl>L z4qvvfB@-E_^3C&|UW&_1^{X7y;x_1sD1B6Io;YtmE|H2*4CaF+(jFGF{kjc?GUM{Z zm(yfy@B>N9W3a8Jck<8$WgmG@ts&MbKZS9EvOVA1t{KAfd`oluFQDoyTsOt|5hSrx zK6_upSk9N;w~gN1()a~hfI4247t}>wgz+I=7Rbk$F@BpC?T0+%!;h$4`>d^T70>PK*=Ol^W#T#gU2M=ijCt|Yv4ZuL--?>qu68+5DXtHcW*II1 zMn(O=fS2ABX>enaVjq2oC&v^Ha^-dL& z&S>T}N{w)GF4ZyIc`wCB)3VX$Bn#2GzD66ityaw2KXTO1KQ^q<#4xP?R#mmT>Vb%r zNb$e>+P0(RKFY+U8NZ?R64MGbSZ-QFuLHyfP60BfZH86U7|TmR0?}j`XQmhz zv)j9g_OQc*XU9Ndy0bthC$$ll5ENcF_p$6RBXcVTlVaDzW#c=ElJY2XQ9h>8eTJpZXW2bM^V8!GkxQvs+<{REDOYaw7) zOB>HW(ceas%OkQ`XU4bwOe00fI*|DS_1i72G0Y&zI+ZzH7r`U$w7hSEQ=uZv@h{Ic zU7;wF#72(@YN4Xa@9fY9hAlIkt69;98U_afDHn^=w8!XyonH7Qj*2ymNI!H(3b|^& zKWz~yP(L_DGTPc@R711D{S6eW2A|%E4{Qy0}0}!nAk0} zBEAN_h`b+N8?qd$i+=G&mfd{@2t!S`72MymaFuPxgl_%ekquJtb zZ)*oh#?6Vcr?_~7hDtA{SfWnMb#(Wl0wLNj{}8w9o%in$86hkwbJSTlr$qx?IEZ?I z3~YI<@L*7h#RHvV|eccvVF`yQzuI5CtyTkcr{m=ah`yej2D>|hxb(#gxA58LrIHuoIWjT4qEo; zT!~c;CD1y7nLOS2m_?t&gir!mm%P(}5o5PNSjwBUzbIZm)Is(DxA;ieM`3lnl z-;EzgG$rxNkK!x8logb(*IJdkzs|e8-~)8kwOV_jN-`@C#%&|rt#88M=&KfrA96tz zCxAq4W8-cPz&JUvH8USTBQv)MVh|4f*$7mnKhQgI`}Ih?3R!M#6}x=udHcanYADBs z*|F4e&(CL)?n@+#ckildDylNnYctg}xu7!PdJ3AOX-4mzSxx>!|-nlK(5lNOO+cufGZ;~fv&4L(m zeUENpr9v|l;~KDkthCyPH6_0kPQ6hEUqkO#c&UPZf=}Wr48}9Bl8n^XP&hQ+3Dk^L zL%iTFP2(Pk^LpRd=jn512~YfqEN79^l=zW^y341G@BhXz?kZ<@&q`G1{RgrVK!rO? zKv&?-l5enBO|YTr$VO?Oj8waDy-*O3nkEU0u<7_ns9>^LzSa0eKop32AdS~hYFIM zSh{zg+eSxDqcRwLKTbBvFO|3e(>nIM26zA;7TPd*y=MWf{nI9TbpL`uNCLS6)?T;h z|4-I~V`eTxc$9ENNis-x-V{ljc>>&krTYYk#=ewyvV?Tq1h1ZTZm4F*8zJfVj+=S1 zNd9{DAi8z|piU+%_mUIiF|S&wxi#JM1hA)@VBhbg!vel>@D%vIpPKMUk5AP_zdzg= z8HK>$thg^(G5-_yWsDDi?>SOgZ)rhK%>3&Ll}sq zqD~qTK&05DGMijRTu6%^I(-ZvG`aUTA>^Y5;H8<}(dqw5DgJ_o6sf@}Zqec{YDJr# z1bxT3-_i7FA!`AX_s_V^yadf>$!7j_S}#?+fj36A@m_r410OOcfye*DhNCNH-y{<0 z4MDg_--X1;oGe2w9Ol=eB{!GXvSE0zZCN~_H;)Wh@y#dP@U6V}2!tapzr!y6{g zj{AJL{(269&9H3e>BrR{Gj;zL_XXV2UEEVhld=&ATTjlc>PiyVNboK4h*(Je-oBJ&qmU4huW4#-P^)kR9bM<2iLJONb7ZYlv(gn_-) zqF=D`%bheW(kxvGhTlDByQ}9wLH`$L0LkzLX>-im865Kw_JCYdy7A&{D=Z-4SfYZ~ za_f2Iv`*pQu-{!r0H^7t72H?&KP=b3KAfxJz9-*br%i3f;=3;Nakq!>bon9dLAY+i z+2JJfX-TN{-^7cX*o@a3Bb!v~XIoz26cLB$X>hX}Y8mNtu4TEj5QBhuW~hX2BEsEJ zkVA?c15x|Nk((mNXE4g6{XHSq$OpqNmivQ0`wTmoXRq`)!#tEB)ZTn$PHgP)70+(Y zF(^iH-CbGck9*)Gu;X8yv%G25X(3mbBC9*rwIW={L1MO@wfZ@76DMR z>Zjz76#sFC$j4}$piB%{qxmat?6aGLB}662w>TicX??8xO^H`CK7DP&(adCV-Qw#_ z9GI}>J9`=$-ZmTs|Oc+xVeC!nTHDt1Mj*P=`j1XKdj{mIH~p5WPh@?K2sbgbX$kdcV&Dp)IRoY4Ta#rZ|?dd;NMrmgI0YRvnyWhL2FQ`C;a z(O%ZYR5A8s)if5A=a1~d7BWng$ZiDQenV@q(oD#*Ze9;au=BHY@I$A$Wq+$&ID@2f zYkTVtb0He|j7do_0DTr$Ri-8LJMRJ<|F2VB+yfPz|9!Gnx;4h_0kEKFqykm3r@dz9 z^X%rMCY~m;nXlQ^Dnh3z&GuH$J7v!YmF_?ABGCE9?tZr69I!Y0{gW2{zqJ6QSP0;C zPF<=sP`}0~P(QA`Yiiuf*zyXyMBwKrE0o&fXytn?>zu$IK4Q^UVwRN{f`lshTKUMLu>R`nF>v zhS)#I`;T1H4p;J@hTvjg%CUsq5J#ge3s||)2XL&8Z+sRgZUG_flB3yxMtWBO3fwxH zqX-ytEN`cCYP}}LBts$pd#96S3&`8e5Qk^2URVH^o%mgY(19N+SpyiR*t@R>NphTk zZS`GMh;Y#fF;W3d!;PN_TF+7O58i@l+ z8#Z5lY9!hZy35??(eNYN_doE}Z(%rFHo`e75VdWr+F8%fwgO}|jg>%3<(>i1UK1tL zcREkA*1XI#_@}G*Uws#F z2v2!~RPaAeJpIrfzJ>!7OK`aDS{LaofOesYn258AN)$S{hskH3^$M_8m<2B}`>vNe0W_;y{ZU`}x;?6g zx3Cs8Y`!$?bTE=blFy<6-Z-Frni>Dzr=3>MT^9J3>A5hL&9Rsi^A8Jx6aCd&GyuI;?bpXW z=h0AzQ2MxO5lG-P(t!nlx}F%|<32$RUp2np34yOzGc+h^_1hT1Bi^cEqG!c zIDboTTH0`E=n0t7le17DkeHm?qnsOy6ywFlj~oFGVoy%cuAo)gOAmyQ###3fnR4`7 zaG+sn0F?53|8zC_uJ?fSk(c%!;C)!)Zb5fEKw@>|Ay;N8=-sC0RVIx$(D$53ceia6 zhm>wSxyj*xkgrrG-?GPYj$caa9Wk zj41{^=L||+Yr!hto3j2)6znkZ+8;ll?R56fqS%`@yIy#g8S}{%lppU zH1CXCosaa1HXb!>)q(dN|CKqpy@}OL-|Ib`gAKf~1aW9p9_%MMsS0@ta@xO2?lZ*~ z2R!zMBZ)Qnx{cD7g_%pBj&ldNZo+A|mVf-p{pokUM|&QUgWLwrmDBY0_6}l2??A>< zt^+k_t-NK+_%78pb37~U&^iCsYgZR;=tJ$&pQ6AE0)9UV4tnN7JrGAYNDu2k*K>fe z1o2srK%?nt||mK0oXU13J8z3 z>(MR9oY$A4o^*+;lnZI0e&uLMtZ=YQ+gV?n9eK(gH&aZxb}}y=6&6YXlXe2ohzo!~ z-_SQjiA1vGN`XFt%>aT6`>Z(EE)4S0yI|ZlYkKsu5VI?D+!0_FS?EN^Z_VTSGvd#DFjBkR0@>5?flr_TJ4yP!M)uZ`iEW0*RHPsl0f<=~sQTcNg|oiHj^fWd zX9Fa0mFTwhR>n=*DOHvmuJUIfdSxHt#Q4y)--jXNc+gQerLbbcwIJ8<@?hlvCJekb z)PQX$JLRR?F{7p|su6nS_l;mn!{dg&Ak0q0+Ptw&9{g4U1h12>qphqSWR*^HA@qmj zya-L9^Uz-)5*;d93lo$j6^?ldt<6~79|QHqp8>l8mP|35K4{jpmsB#o#~M`&-4BJp zQ;}p!y#?xlDW$@VuA)oq=Rd02>JWz3wAetDh%23KW)sSM*6-**Q^hW1sZBdVoeH6) zEr?Y@6}HILkYnAxy*$-=eY-mY2>&L!sDj6d`axp zd~oQ?30yLlkrhF6!aMNg^D$uJYC(K=Lktt`F|W&Yw=n%cvqx`l&4DavGfCfqI4kf9 zVR_lBxb=akui4^94(I-UBk(xn;=#`*z z#_3cH&Rps#Ke}q&MhtANWhAx;7o-A({=%~>Wen_#t3wCaCkKELrut<3e4YKMUqiB# z7pBErrwiHzXExvB?s2bv3zxAYpF=#eWNcDNb1hPjJC=Bz1Oht+82Fphs7W#@6|n>7 z;G=;X`;Q(l=be6VBzRZ-Az(xbbWS0ymiqeND1}mQt4<0LB_o3jBSx=~Bbo$%2w_r% zIxsJgdAk?TYb`?pI|uc9{d;(9p;9_TVIe0}%}+h5X5o0|Q0|7^BG4c~kq|)1vY@>1 zXZqdLW^?Fe#w6dZhEYZ_c*_DrRUF(hW+ox``?F<_`CG>xRUb#NF}Awu!&i+xzCW6I zAp+{_(=0Uq;`oYEcMv`Q;1_5wW$GSDiva0Xl>m=2^Or0@&TRWYOChAo0qAo^r`jAX z#;Cy&)y(7TrXQ49O3pRD(SrP20^p{`yc#IZ3~}BO2seYLt7bECuuz;cL+*7PGhW#R zdy&#|9PWzQ;^6(2u#oR3DC82-&oL)`96EmI^5GVdDymWFFirVsBbzbXGuS8Appe|; z{GHBnbn#{EPLwmoiwPNw%g{%^r|UlsX@W}p2 ztWyQw*P|A7ItPIllzr)!%QHV!0kFupuk(rgqJULfb}5J22mXY!#M3zZmlAFoNHOr5 z0c;0J3zr=2Bp>`DkeA$gUw$I^Q`jUvjuFBRRC40a^UuR_na)^wH(%A=Y2h1{=SSXE zR4*WJ1i8E4MH55ciT!%2#DQe${gh7l$7a{SBWS1WjNW2YJ$ufJk@v?WO6knYvgq)A z&xw#DM)G>gJ~c+TilFp9a>{!s9peVeYu0zT7?g5afnJ1RNI%Ka@;{o9M;v|N?J81ak z7DF^GLR^3D{10;n;w#){_lEEi4lGh4%Ehs0^>6U{@v8(j61cIjUFT0L1?S^WNYj}C zt!@q)kw*q@AdE!Fs~*qnN-oV82|vlFmK)86gL$UF-WMoMNyc)51D`wSB>OJq>yo+= zpTj$hw12K`O@YdN>FwA?#UH6R8MB{E!eB=U$JcW>Koe37G_>zK<8<8z{;$v9XG|0{ zPE<~6fj?mWH|9qYKeT(hz$VVp&SbnYv5N-disa@XB{0c-$zyX>o!awvgVFp41}A0d z^r;VtK93Pilz|YuRswu>k%49rj$S3%6K!XX`ItmtlJcUVrDcz;kjm}Z?fwNcAGk>$ zV6@HIZoW8E4_ALHyqk#sNZ7EGOaSVpoQSzRq61$#nA|p~h-Oa|4Ydp=i{z@Y6zaFQ zq!|HL48k83gJp1%E&}haaqD@dn z-6PhHJ!i-LTIq;E(oHVJS5xDdEm%GOv(zs0#Y<@F9Q2nd;<@Ct-7`5N=SQe~!Cgmo zReEi*Lf*F5R(6TjroMw*M7I0X!7Feokz#3xKc!nsms&anrJQ-l59WwkqW478b4=IR zi0&mF5h{t*yv_!9fBwd3c`V!)#WDYw&~Tz ze<2|66H6GkhelG`*b$~t%r7^WR}F)rP*&C5@|L3H#MB5ND;e&4=^}gJC(?PL3ootT zzI!Tl3k+MCqeOm)FQqTAqZs)1+Vo%=uA$j6S$_fi@j$%9>?L?S10PWQL!P1eg!G6Spsw8?Y@if`%6a2dJYejb`RT`g z4e>^jo$5ZZH)P5BpOfo!zjt=5j)cpEvlHxP7pXHA%YbAO%xAFxPLKmUbrC5T&b^hz z@zf`>+85vMI}Xb_jz@9>UoH4)8MZ~7#ScB&?&vsvdGFF7iI$xXH*Cbhxb>|%l8=C- z{^Pi063>+ghRfrUue#!`R+5gNw(uxQUj0?HL>8iq>4Uy+x!v>pM#oc(yi0jJH`c-m zX7X5nJD2gSW9*S>5lVt^LrlHlaUI)Tg|%e?%C|7v*lUCMGEhTItVlp*+wLC$Mh|KU?1K)ly* zSyX%wY~hCDT3)PfjiR>&-2&4rMKC8{257dLa(eJ~sOoeYHQV9OCA%j2bqE!u9Y4`7 zS8>f7p665a@9@dJ99~m~o&@9AF=3PIybmni3AAa{GvRaXG0qf{{y$WeErBLOb7uS7M2Y z;xcwvDoouSrOlSAupXMkU0%<2TD+-M_w!C$j0o%MbdPT}wJ#qwDVL_xwL^AAs$C9~!K9 zq8Z@;B}l~-hL_sSvtRuLG?p+kh?P(fH@n$q0qh}cw;FxiV?o4DY2iB9Kv3!9HQuGC zHlQvcL$_nzOeuF&lL5?mSvJqp-b;e6OqDz?XW`|6$yxAovg36qN7nKgaZ^{qZy71# zW4<7-vvF|_=cgS<JuRH#;(WWK^+O^+}ac9(3Q8u<=R z>ydgCglz;pd7mBp@`#pd%9NoR#uuDNfC|<(Rr&mUWXEdeuKpBm$$)=B13v=sRczTA z@w#2r0LT$Se^dA4A+FKzq94!x?)Q{(8>0t+8|?4Z2oVch?MRPB%PBs=-Ey!TrAuT? z2=PGw)6R!d>_nsz=t2=+aB(GM)`2A*r*~&LY46N3^KnTm)OJI)+;1xbAjUTphkpk-EeG{X4q>v)DW(`BcJ^%~ z8!d10qvyO$mof)3km}3YKNR5lDui?0kk;(OiOpLro1^ZftbmyT>y(SHS`PL%Co*J- zj_MnWR{qz{gBv!zH*aF&;~SO{FW#sDX=B&%w+%~B+K2xat?>J?y+kxvF#DxH_NOm86Y%a9m zH56-sXK$$btHQ5#eQ(7Jv=C>gcqH{yBNuPKVS>QWfqB+HDFz?g$yrQLPeRlFL@m>j zO+l+#sW!i-K9?p-fmY)X#Ai}*5=bUjegl5~io~5n=>efUOZCb=0%SXkhq8e>VxSDR z%JCtXH>omnw*T6-(wxW2!4!%n>Gkkcn3cOK>KEg^)A(qHIS&4Bfm!B6hx211m195h~dX)qQXuun>2d3s^z9D=DNAEMoxZs2KGZv_B*E@f%lXg%zFZP z#Ogn_OaHmOEJ?S z%d7yyF7);BAFIH${S+<9FiQh?1+)X+V-edT$Nc6p=}unlC@Ij&)2)W%c5PY2YV@Y}VY7a)-4l5v6qZ(vxgH{b$qxBbv#(c@P3aOiE3oxFu8EB^9OE zs*?$5uUI!(3`P$c%^;t^z^4A9?Fr^_5Q`<QiCa(btKrbb;)W=m^;#$Mx!{URfwG5=)~9|CM1s&$m~4d+AEB zRj?u-bjuaNagu&Coi2fXZ&4%{TZ3DG^0Q7nR9z5hlyKvFM;)lyv>D6kTw6LSfs$vk znsvA0BOm0exlktB)p#;kHriaeX)OKMRS4M3V`4kSDiT=y#^w3lw+oC-Q<8iYNfKXe zH++K;sfiV1ncY|WU81_2rxHXNEE_ghn4UEW5e%lfCLUbg!n34Uu+zQwV`2RGMqH*e z|KdJliAcq&&1m!GS6lb>9i~Xy^(NxCqys3U5;U)R`+c3j>>Zy6w~ySjTiMw_1nYIk zVGoKr!^f=qn@;HI;0v?GF>#+(wG9K7DiZBHMMa zlEU#QEl0+{{dOOnSt}o0#l}1ktEym| z-likLqlv&LW*8Q)x>lxU$dxef+e-&9vX$*0UiJ6AO>?dIzndBCB(~GuSb+A_(%G%+ zw^jb=fFoIWKE>X-{0fanb`6PQEt)WW*1iAIC0lD(GFN~AT0bKc{2(ju_-Um7XPLOC zP)hNV|Fo!@7~b4VaWbe`xT(dlirUfZBDc(x5Xz7jwATkDtC#o=D*W&vis&CLE)h9> zOoo*A;3@H|kcHdi+T9K7wVIemUD@aq+fHtL`EzC=yz=Nz3+_mc`?X3wgEH|Tn2xgI z5ia=hI6M=q(Ap-<`I_GA1Nr^bKv21`yS|r1(F2=>8ix2!S{vjwxaclS8gcC=`iG8% z6UB3t)lj$7e12XL^FzH^Dpp(&=73n@o5+&zW7=a}e3&O+par|r+y&c=j3@=&x|jgl zPt3m!-ETmjwdoJsq0%$)Nc8l1%-r-RUEH{$O5z5+qu2;bUeT|H%Bp2Rq+wib?rmH6 zI{*h7@!&G=Lk715@~E1|+oQHjWTXps`eB$~cD$>Wpce+)axwhmG4p693bs|Na7ypm z0S`>@AX{VLLl%b0`5H>@kjLUTN9RW;<+E*gjg+GJ)QQMq?&))B+#F0TV&O6Hs3e2f zrF!x+KR?cB3x6eaz4P%^%gyGASLLEynnr2)3Iu6;5PLT4cbZ&?YwmT)=S3Gz+#))Z z()QYI2*W`o2ElRRt|8b$mqWhtw7@6PEt86d$Z!;g(G_yeqUjb3yG|npzFJMmSxniD z)dUfRu=XPGCO*Mv=|la-JDuj7{7A)xSo8n0f<=*~f{+l5QRV;33YMSX0;i`+vHyV( zzl6MKhw=W1w~InguwAFqGR`&7H@3KMd9N!Alz+L*OO^T^)R`8eT~su zp;nL=La;Ta61D5{m__wngT8r{emQS2D7&7hTA?@3chL5R(UOAHEBCQbS;evKAO@Y1 zl80FHmXF7P2K8?(fbvYjQn&#fet32V*83IcN?|UDQji6d_u4Kjt3K`@qQX=% zy1^=*=oI<_>#8OG*7xA@k7`hxsc7{cU0Ptl(*a@P!Zl9sH`+5x zp-02QFM}QB8msuJiYxesO!FdxwFjlmQqu+plw)4kWZ(^aT|M_xC}GBC^%a8C=! zxiE)}{`9g~F_(9>2jlzw!CzJ0hULmmk*Kx9%+g|#n}_^KPNPf;yB%Y~3$Agcuf*R` zhCk)C6OTEMOhH<41@XG+o>0UkVi+-r=C9aiiLUCOFv(dORx-yEWS-35KRdh@H*ze|FK%t?xU?k>Z zT1pYP`I`UY=DDy*Dr}=?6Kl~>DIp{~BGY5U0PkO8$yhm1?7qWfE4veGi&$}WQOOpm zdRrthu*ffXa|1dnxIEMM@x#s)9`%?=$hFQJOlD{RC0^a(VkmwP`UoO4pr211`X@Aj zmcT!lsAM#}YqTgCO@zpeptN!rxErY`Pc{Se5}{flS^0(v^_qOedCVPbQ4fnzrz+v; z{K!6Ysm`%zN+jqY7B#4++?@BiD^01iGK+RKXyNPJ(S(dE`k1#xl(G7lPjT)d)E^n| zPdpbmrW6QE`9?FB_^C3BOHK)Dr}a>RJh=NTWy1jSu8S)O5iG=X4?oAMC79SWmiQ>X zZ}k+RH;)NyW$E`LFaKaX04XW{~vqr9Ter(tdDLH zl^i7~Q8JQ2a#D#B1tf=&BqCWP4?!hI1<6T80ZBv7%#e{dB00+tB@IKGVF+_y{PuQ# z;djpO)~Wl?tvXdoS=J2idb@k|>h9J3JZm~v*(z^`QC9Xga&E>e3E}yfak%fzHdl9W zb=SxkRqqX|FO?~zFK}C@a6!$M*iT$a)(=rX2$F1$k(r&QD~j3xIVzjaA3Lj5Y6QR3$r1mE`E z>BpVY8?%H!sr`j05mm)&H$gOUo=)$lyMYu{zyw5wwO${|pdY0v&`n4Xru;|#4QWS) zBR17LAmBo-xlMjcjbK7K$TX^A0Js2Owte9MF(-j2<5p;5A(4MX@;ZLePFl-|q*nE1 zyUAAhP4~eACr8tMt4sn#sc$J>TH;!FZr*ng?I%fB`FizL8eD{6_qyg;`m9*S0>vkF z^Nd;ec&7wbu>Ix`yY;Qgox;5My|=qZnNjYs#Ia1hKA|2|db0qO0XtN5oKn6o*G@$5 zyZ$3%ab~(>Bhm9)s+`348IW`~q)3G&#duz@lR7Sgzxonml!nR{*+|oLK#Z>hFyVml zAbl%W%6h*e`uad*%XQHP)^)blBu*ZI44}AEIb;GfsGgoRT-9@WcY$l7Ry|DD6UVT| z{^(~!B$G2SHSPC5s%RHYu!Wzyh7ehtFFfSVk`JmKr)b@)S(iu`n2E>=2|8jM2u7?I zXP)s>$~kI=KL6Ood#V%8fsedEG_ak0tZM^AXHST1Df9}ZK7|4u0`8)LcTBk>@7fOx z^gn*=(tX(R4nCUh^9D(J7ANm_BLaX;dB;yi_YYJ9%GdmcDL2}DO+%}qwp2EBKx^l6 ziWU!q7-)mmBC*>!9dp^>fASUtq;&%70e)qHRpN3bpT$t2)diQBdLomq2bAQVh@U zh!9h@g%UTPh`WMHHqvd=Zyz7p0>q9m;f4yE0J&xSsQn0$a& z#a;O*O$q94Z?b>ec&f@l7nq|Cif#PlsIt5=U#*CH@_C@f!-U-?WfT(G)*JK{#{TZG zlC_-5CH^nifsX_0ipsRNZkk*3ev>$v9xU}8Z=i$Hk-7xu&FrzZ2ezKyg;B{s{E`&rF?7VOvd$^+bE6ot z6ZYVHB^!b05tU2NWNTBX6aWP>tHa>1eVpMnIGF?C3XgDH8z~D5xo0l6ruHL=W}$2zR`&aEsofpylo-B5-*SvD7cQIi$?}ZoYQW+l1r#!Ev_@^-3WQ^E960l%f4Lg2|ThpKu6#CDkGQ zX|D*>VcJjw0RBVqx!jf?%X*TObz2%*0zj?R;8qbuasrsFZwd6Pjr9vPL`6|FZr{)aeWU$!YVQn)0Uaq4@9_Q%qkz(0oK>?Sz4pYw1d5 zC)k-}ERN$ZkO;P2A}6~p;a|$NNoR96d7p3#J6uQMU=1MU`jLftcf(A0_Xoc=reK%9 zk}x}F0atl!osPEH!88%uQ^u(C?#WKZw{FUOzYyvM0&Z+-wqSr6tE|TVK3D*{w9)Bx z&(%l@>7_&%%L%&bUwY8K#_vwy+A-Xe%##^~?8zIf+PsseD$|$Fa2sDa!OV z&MNhhz@>isQsjB7)CH#T2~W|(hhoG0$VO}BSSI5~V9s`Gy~|4CC(ls;jDymk6pJcr zcuy-4t?5d62N?sc$@d&ZIe=!%KGkv+$Ez#M@%-9{o3BB7!0B)f!eJ*ypC$H*%}O%a ziaJ_~sN3?ZN!=@dADHNvy8SBBddrg_0N;E=CDh@$n;La|rSHmTOO8W6)zeQ8-!!Lu z{-avG&B&B%CI%Y)fq6;BM!p-z#_RNjMb(dyxRMCnj$xtwNEUUSqmO(^07=jLI4R&k z3UEI!%bQR|q-raLu6$gr`jiuo@@k|8fd#>+)Fajxp#gUK#u!!tn=RE?M7FXsXQ?Qq zwPQf+bLNjE@p!Yus8`3Fws=E9ky5?tkMQO3X|TNa$%oiuO_k?{g`&b$2R;K$oL!0M zb{~m8isI3_Xn&H|2_J9)>VhxnZME^sp2@iu#1+$mC?#&>QVb-q`Bmn8>Q%YpkZauS z8tT&7`ROvr5Y+>QB@#^p(`~?;ePawsEvLyNi7VyKoz3F~Swlkr4C4#ot)KNmpBcPv%o)+XNR@NoP>N=vk?kNRpZJVzW^(xT) z?iP9adCpg7-V*xyK>OS^Y>PCqe5oe^Z6oUPK=y|gMG&r#&X)2l)8m6WFBULMqtUhwh)CchSlk^x#>XsQZUzb@e|(E0s;IXm=lKNURZ{^Xcx zx*!u0beq%utBdx%7v2X<#+vYVEYn6H*Az|UQyootOJqi|H`~J;pmSGB?0-l+)A-== zw=QG-CrN_YTg?A7NB%#ek9p}zMgIa-lUd>=M7b^wjFX{g-~7+`f!=2VJ*VDDwL6@6 zEu6sY@DS*K32puyocv$DqrZ?LehK7%`HudjS^NJ}IxONyh(|0iJob-SrvLsYHn;T` zd0K=;P|i>P=L?mR7t~-i&bh;=e_7x^toT2EUe*8xrGh$uv0hfF6?TCCG8VgD@oy6} z7hTQDk24#ro}1;3e^J5K32r!@&D8pf{yB07L%O~COzBVp#($9-kkqTfr2iE3?OVGI$5z8PYp4<% z)nh}jX0ZUf$L~qTmu(!aa(Wjoxz=A1v6;Y}TT$w?Zuhqy09E|sPvNW9wYA(r@n)D&m}y$&Z2o`o6c;*qNcb|F7HsG=s-eEYCW5!&By#_v zWce2<-PNSO%xB{)60#;Q6KFnBh*);O1ywaIsi`&oR1}7o2f(U|>OYnJgZJ;B-i7U( zFdoQay~eCWFCc6?#S^-Ce)vtQZW(A>z;kk*E`CPR8F;M)VmS8|9Vb5%hucqfg&InF zYr54qjHON-{$7Ogd#7I}GuR_P6&$WHVfHDGVeWWz(dX^HfF}~2~ zfIP1XI80ZYGk0rV&Cgm5blte&;59NRQ5S&w*xxioXFjZrYdbcBvc=XbZ6%!+Q@Kkk$Du>!ZO#9p4&TtYILJ>6beJ^mUlFlNa8 zFXW9O1SEnlEop<~z2c%VShz><*DGRE*9{u&6{_koJ;S_i67SY}0*`s2=^T#1epy`e zMFa2;UJH}6xWK`)A@7kv1s~ho3PBo{0|0A5R?eLUv1uX%6>L~xHUwEs{5^mFvW)U^ z4i8G4rLJ>A-A5TLf3o-%!PnMam(u_XI7Xe!EgR@;uQKe_W;=Cx%~e+gebbgWzOGP- zT0x;3*3N^HG7k_{u7{97*tQO^t0z>DQ4puC5op?sci1!EOVcZ*6QBpBQvwS!J#}5D zPA-eIr;`qdI?HXlmdPCgbFnW)cAlp$AcK`%ux6BYA#=3kRkW1H!Wf_a7W2}XJz$#_o|_# z(F8!>`?puQL`hG9XG@!X=UH4|9`5Wr^;}gXB&KxPX!7T--W@CdEi0dsscYC^kaLpd zELiD_U#7_prtQ{F7hvxwZ6D8luFeTX#>TMVPh&^&0HgAiIN9~UQoK-hLDY}2wR zo13fWUsZtx@a)w*gEr>;J1{^@l`J~7O%|Ru+MQn=Ioej+eAAfP>85k?| z#H_gP@#a7MVhlUph}ntCEPGDPTtB8}f4(;7GkZr2u24<4Ze#T=+a<~{91?@^hbJsh zgN)}+w9w!4e40>_7ONDm76MNOKSWL*&&m}CeIIPmfWAkcKE&S^tHX_JgZEf=N*_ST zR04*8VOuv3*wZd08Pi1c*~II*h8~=6L%`wBh_er3gH843>(sb4oaSOi!h&ylze+H;YD21RGMj1^hM1QcSx7>9KH}xmAU(4dnfMJRpDD1zSs|ek*g-I zOp+51U#u@HeW{oh2$n_#BR1P})#*U}l^=8<=ljFAn2j+{$Ae*ekPnSIEBP5g`5(M9 ze7>$%d;~qx7FjkuhkHykodkL7uiBb4)t+R@fd2OC7oPP!jBpD&`AN^|Q4Yqcb3zsn zM(Tn`HKtD_SnhcM`sh=XYnh=0<8y*J&+COg3qk#7!2xTRgwza=7L%;hSuf#*QfaUr z#D`4M$a>$aK;sciQe64WTgXGiLxgwx_EFio1Mm55E2M07) zLjckjDXb70V9hgD~;*fsu*Tz<~%uOpW>o!|-*or1ygms=Ii7ofe4KBIO8dJ`E zvrm<)LH8q0|W~Q^%-r4JKstQo*ztYUPK6@ydsoxs6a?e zm-OPI6V3wNwv@V!x!qM(uysLE!o(6M>m@?9#jVt16H@9nxK)mCFq9no%CJ1+!zRn2EBt~ zkOt%d`rxzY^h`+!F>#%LcNHq33I1NdNq0qoFr?v zijkdUIHU!B0tYzl{dMb)0}AX#Qfy}JPEuiznIcDM9Y@OA`2yWMl!y!}-#g}by5APM z#(47?r)NRXTeSdH*!V5d3M5knBRtG(9ll z-OH*3J~O)q4rF`s2PiX+-a9%3BA73n5V1whq zRJ@W{*f|}WXlI`|<)y}}3ml&52v0hSTic+LVTs2DAwfZJvVGPhnd!Q>j|Of|Fi|Rk zZc;q11Ft$&>i5}6E})Wx!ou|AEQ2>KE9DL7hQLzJxM=cFk(r$oX=%VwoVX)F!EdtN z5`2OrV$VoD_Rglhr1(8j7`JrTdeCPZ1S^jwRbNjTTh?pw*sM>fN(co!EV-=K_ z-gOYSGRiCS6=wQ&W^tbTK#N=Rj98_^*Q)n?x}79{^#Zgxo_CDc^g>_}gcSoGeT^$s z3^wUPm5w*WmNboc>pWrqZnv5E@zF2iKPL2L&r^0y^nAp~NJK-LCMZB_^FC?-Ja7Md zXU|=8Q4ps%l|Cd!VdIq=l9WLi6c49VdGJc*T?-niNRrS^^zgE6;%*R&jO0F_fm4?d zR`)u{xHB0N5L^z$_>Uz4=PWT|Q(F(5O>GR-kz3{khzUw8pmL`tpC;_){LM;e#oOgU zGeMX9XG}70c1>K%yDROxygq=OS4e>~*PT=WJ7_$6BD7cJL5@`C>NThwh0=|hsaorq z)MoaQy53M{&dqmeS_=C|d{rLcA07+6HuPTv{17Pj2_ZOXrs^%vGeTJh!)7TVoDToW zeEn^jTQ3sQd=;L|%r#Y5jtMCbcj?-pVg`?b?v+H>+^c<+RiS>$$t0-|-6JapfXwRH zP$hJ81n zKTddQOSx;Jdz}WP8P0{+Y)CSuGZG0ZSU7-e&6M~uQfnURLw!bRliJq@xxM=b1gsD# znmVLa$VQ10$)Kk4i2&HQ-N}Qw8+9N`SJ*-979?})u({m!R`J_PF=Ms07mhDTxVrgu zvf@Tp6~R9B=gqcNK|3`4GI5jcRbxBYwtD2(YDJ6d<1s_M3Ti=KV{6Bot~WLJwW0%( zS7(|{jxgc-P0BW+!f4smX4iNrT+oOLcPGiRc{}C|ltBofxE!gr{A^zmD_CpQ1=meM z#^iQ3AgoXYs@mxEB7CZ8dnLaWKAm7s?~Kce$7ejM39d!Kl0lmopfH=#JgtSQ;eEo) z@q1TaxJx;eA4;-V!gOlT_PEd{{q%{nPojJiB`_FkE{TZ@}Fxaeu&=iRx2=;0?B3C>r)-XxiDyHyT)+ zX}INQD}ANV&c!*zd*iN>#hn{(BlC=dL{ciFbau>}Cny+eVnr|z&V6W&M$GNhT zMPegrBk6D?>P-8{KPc_Gm`Kd_OWdib%-w}vqn;hqPFcXyTFW48X8T^fOx?V^WI&D+ z6#EH--BRipmqyhXO32}Gff7%md%mj%u=sh;=6Pr4H@-@X#WczzoWRTye{GIG4^Kk^ z#+DI8hQfwj=S1fD6y}i!=R#*HOi?p-P%_agH!?YntL2>CI!QjqV4ty9l2O{VxEvk< z+ysmifev7B?WF(!TBcXb9(J zGSp#lonj443+3K-G!(z$vfABZzp8jgJmp=3&lvN2#-^?+s|D??Od9luhNbWDuuB!- zkxRK)Ysg}K!(BuS=PTf=1j0w%%T_- zyqrS~+@b4vZ)CTw+#OEyhRh>@UY7cqCLMU^jM0pjLoloPaicwOUNyhhw9BvQSRF?q z4i?@Kj%CVkf40`T{#`9DcT%9Lc5)$82y2%{*EJCZkrRzx6mmZFKF#?ZQe8w>-*gEX zoAwg{+sIbLX`~I`1v6deyS06~s@s!TB`fVfvrcwTSTZqXy1iYo5MmgJ`fA>L|~03Sbn!N_l5a!Mp@l<|2#uCUFSrUs47 zSiXOBmix&wiJ2!|dpb$M2gky4J2aK(2&mP*6M3hzdRde8K;$f{ysQQMbRuY_Ww<=Hrxxt!uVoenJ6u&(}^G9qpXD#oA|aeKQfk*wA6K6p+rg?{=4OV@-^Zo$Ie?h;5@ zf;o2=EmOu1^pMyqh)r1gRcbVeEYm#aI4jp%9?is^tuz8To7oiW2_LYx;b-=E(>o_` z-Or|U4Rvm>-0UhS2wq()CIiBi#=V0MAi_YSv(pKoUA0e#Y!|U6Es@@>QHt^QZ*nyv zq1%Z`l^l~U6rjA2l$!w#RvsIhRj?he1HX!rO8W@STzh~)+ofdgtS6=&kPnipB6_sb zq6YGUL6dHPbmE~Se{JB>#v_em+L^gjOA6+O4RY&|dhClPrJQM6$Di$fi?wNwTYkI^ zsDzZ`Hp)r6?LCN&f>tauSeB?QZAH28J43OfMu&W^W6B@BT2kUQbCce|&=EuZMk5l> zI{le3JqcCv4hHNJcf7wttw+)bc1+xSKAi;qIwvl7%JMLcXpu1>sSgJHw#p{#IW}=2mKT|BP;o>9lTgDU=3vc+LZbA|lJ95KhGjBgh9A8SI}2>H7sL z%-1qaCv2es1;|YmJSVGZHCUJIVYtauF?)N(zh|dDSc~F(xhR2d&|hlSeTQ;t0Ugb! z8j&zpB05tPGz?(yYSM+x&6}HpEn_kM-oRmsBP@=Acp`xePIDo%)JiK(!NPAEKv}$A zU3hBelyDa)37~-+gec6krIfZ0P(u` zX#tUZceU9%?|Vn6ke4DkRGRz~(OuQ(gR|P-F0)Yg4_Q4_4T?YyEa3MVI0s)iZJ){X zSwVRfPoAgI#4TCdN{|LH@5gPq8+FYec8uixkmA@n=|%Qbb_^0WRpk-*Pb9$J5T*q% zdbtj2D$)ay_>w}kEno{B)+`{e0RcKNKY!#&lJn7laX6unznW=XQ{eo7m)lV1#7Rdd z4^&^FVDbr&uJq}`s?8P}ScYIzFChffHoXWNbISe`i-7LTN>RlmQB=)gGDI@RQyK6C z%v}=#xwuiN;CA_!E6;zPN}IMWwbkRL@AYOXu9cUscrMg$e47C>6Ga(~89T&gTXIg= zHO+t0IPg!*WLf_+K;uhBLSkYl@o4EQD4J))w)Xj%u%H)4R=z!TS?@~Ue7=RSI61Df z!v#n1^yz=EnKN3Y^N=rV^0Ww(ZR|c0-BJwLJhn9dx#iQI&{zs?SxQX0jMy9(3e=sf z2Gkok{b9=ID01xMbJ|kvi#K5Q-GxPgQXq{&Ll`N-K1Ie;F zQ0d-J0fy(li0u203*d${t{{W#j2kRP8Qd^?O$eO%4X&1-Bi66EuBa&Y@hbkL0{JLb z)?%;}h)qQxfZ8Sx)Bv}j$t{Pv%1yrep!1;2nyVzv*fXcq#yW7GCU|bA=CEtxi{d5~ zXdkGJw=Y!&T2Zp~Y_~L0NTYW$JC>2z?+$B-ip?Z_3St*H;2-i~P|!@q(;2p>Zd+qYAklbiFv=7<8sx5|*Y}Dq=bg8HFJ{ zh;0(vm?~l=J|ffV4abfkxXEW@tvBWG9a*b7TQz_kCwYzLZnhkjZOgmE1b#<;8Gn{K zQUL8ez7IY_48Cq;Qt_h~B~^26)Shz_N|)rD#SYr$QJyJZS0X2(1}!AlXf|y|42?9|qsC=#v8|8h28mD$`^X})G9u)!cR?dWQHQO*EkKVy{?LpNR{1Df z0JJ+^K6MXLHsjXELC!4lY;0}N0-fTU8v}f_!@9%L39x-RtrNdgOa}5KdqI)YGec7P z$D4Y#x|2R*4OQ)Uov)`RA5&{P7P%z^9!)ehWWv1dQZd+pi7mo-)PXZ^`k`%XA#dOcld#$asMpeLb9A(S{b(_%LRcE28?1i(i za;vx&yOXJhjwjv$BPL5vD1=y=Hqat(k5#NUQ+<*@4@q`TFu-PM%qRCd=*DS41)%*N z*ELq?YmCNNX;eg@O-(Yy6ufpy9#dBG49dH5gAub1>O>b6U?-%rny#J zz5QzJ!U_7E(JiDT?+(G>E{eXof7M?@@HGe?bKizagnt6kzeoy@Vz7Y^Q1vLq5u!Uw zY>dQh7}?87nZyLm^`i9@+@?tv-{Q`0^JmMl%bt+AabEUFH8j58+_Hqw4Nb1#7r4=5 z&UkvX#d!HD$ybe`0T{AYct1ZlG%CWIFuz(l2|_XgI8=V3w5Fi)T1=lte+2HhBTdBb z@uVO(bnooMW_&kx6PK0kG$!@6BxP<)X!Lj zf(I#`dQKL(Dq<#(VQNA+!=q+yC|b&REbq%Dv`#@BX2qOBqwL+DZ2YQJRy2!MT?aD^mxYFga2(a zKQ_O*B_1gVoj*L|fzEq+PVRL90fdhQ7;Lip&C+@{NQl(D8-GESu z3vQIs0>Z0qpiH(~Io4&|J)rI26$NQx0V0tWT*3lTB&5qzE9$|XXMq(R8ZcyM2H?iQ zMVeQZJh^zMX)Du1P64U`7Krby4B}irjL*ciE(@=GH>m1-N$L}5rFzfw1_Amb(Ts%$wM}q1P zB&d1BsKZ!>rqrL=o@<7SJ3OH&;>ukDR+yps)UPqdvM*e(ukW0%n5KJZGYI=W<+@)M zITc_1QYkz_KH=oN2sxMzSdduf#9)UYOQ8T%VUMMAg2f)OSqnKasdFlew12t-+j=Fu znGb_teX=$#UB1>m)?ndje}v4dn`4*R&`mCU2AoEq-94FBLypZA>D+zyOrdjYUleSC5$xpSFqc_?FJJ~3Q$fRabSgdCsS*b z(n#Q6;ZnOKUjr<%3re1V@EZt?!{c{pNfUscE#}6%*`q0AC@$IVM&yzLr=*P zsM_sfp0#;DFN@FqPF&21gabuA_w?kH)fs5e)00mjuRF>~dnQ`&hPDrigBYs#Amo~P zt@Hw$!gdGY*2|dfj@b%5HbE3%N4^*H{xz=x`xR9_jkeD9hBN1WNzE`QaS$d1LxES(EqLV2Z*bxla? z0r3zJdpK1fBa;(UUml|BD}AO;rPDnXMW1-a%l=3<+yJouQTbSqRnzR&jTH5016p%J8_3y#yh)1ZozE)v22@# zr9R};o)6w*stM{nT~FK&jPvakhFb3_wZ9>i7%DT>kPDsW$YaOt5OP>kI)1~B(6ah>02UKE5;g`c&se{l-A22wSS~R zw#GmjBmkVG0cRmW=sn+#e+euv%1b8%S@md*z|ncM3Dz!w(J64R!r$Qcr{25ZO-4mS zq9HD%9Gw#e5U0)SL8kwrl3z6{c6@~@-EV5!l!-Z?Vnb|8W8dWd&Q4yQKe$+nAU!Q0zB*h_qqV{)`s?Mt9`NQa>{pywgqa@yp$0xX z7~`9ki2$Yi{_JHUTAk|XhurH1>if4n|ke2u;`^WVM=%Xx93Vca!Qdll#jBQ)lP z)?OJF`u)CtzSrFZ3e4>J_1HS*q7J-dVu=-;9up4i`O||hX!r*t+-@& z{KxMI!S{dF`tuseceQQ8H#vFl0Lfl|d+pa<^~IWB2K|<78z9+Jn*RUtb>J*gCjYNq z$0L>RzkD6U$hFP9BZ3FI`l^6nRlmZ2i~la}w_w)+!D4VJwO(ZZ2Nkaq;yEziQU1TL z1lKcoG<64AHY(4+RcX~BOktNi)Ezb)ayuu7=${;@yg zj==JmAFpTrD~o~wp!C>6`AgqD-%+`jB(v@P@mFbzFo0W2e1X8+aX0~Dk(l>gsvWAy zhn^@+Oq1ndy%%Au4oqXh~y3P%+QrC<8P03hZt0I68^6yO~&vz;h>j>vRv z7?F(SQ&p4f&!Gq7mnpedkgo@{= zgbrd5rrl}^?o`&ml?9LuET)*d5>`3D$>4rNp6vi@N5A;Ev2W`N@LP#GBFI6u6OKon z#B_or0K})R?{JLO^&x;L>j`Yt0op-is2VIhnAEnlKoJ|d|Mg`*o0aWbH z=>oC?fIpwzE>vS-yR_i;S1&-|oh<-QmK5d(kPC#xNlX{ch!_;;05@3OlI^hbL&)xr z*W}^%USAC(q~@|)!~M9XXtLk4&CYVHjv4ZSj%>@ZtHqN5fbA~6$NeDYc$tkt;b)_T zW7y}dm#*PszZG84idX}LE>pLD${fDhBUe0iuV4Qd(|#drcl4Z3gx~&Z*l?a&1e5pV zEje@poYc*KB(pN0w{A27umMF2pE)mUfDXW-RCdO>N}FEJ*F8@fiTtWWUrc(5W5(slP0jK_mGrPN=&RKy;_gIoon@K#l>R>n%6C&5&NG z_ZK#^n2iKG9Kr3Ed;umSA{+pwUum0}o(q79CtZ*Wl{DIUKff!X$I&g1bK5mke?$ z_sEO+Q*NX=K!K!#nOQieWd;!UoO7Nuw`0=b;3pC~k4Fr)1>F2J>$fWVD~}s7nRA^L2y@~wC)Vaj6tE6teN~>!5 z;Vl|$!=#e4EI3E-7lO(0n^p5@ zWl4!NZ(1UW+=5SLjAQ6Z(wlz9R+RyCSp_ye(U5GB_^r?QGA}~_S8f2DJ=UY@V9q}h zIvy4W0`Sx8T+sk4!2;<8wtUs&ZRUV2N^bz)yBvJH7m^lXJJ0|S4j{&|-f?`Vi0yXl zZp&5(^v9rKxNPaSQiT@;dmV3XDK&brN0CK3va7}xPW9*pV@J}}m7O@Ch&SbCL_PKZ z9}V27DfWgKSJOE_&607LyBQ4FAF@`1R|w21$nC(tkPfskdX)Xp^d1L1;l38n2hNvI z0Z3kW=cHL&2;Nn8XV77haTxA&O>yy>BIX`f7 zI9`wU8*-7XgINkr<0V_^BCy1*`;WullYC?-CQpY26qRbDk}K}*zFU6l9X6lSh(q`V z0UQ?g`y#M(h;J9)_^AD~jL-vhg3@NsQ;D*;afUAAAnFIo?Cx-6miDiZf1LTH*(>@n zr*d10w-S?&Uu!vc(Wn~_9E<@?PFBr;F zI45m1ZIf`j5sJ=Jwc5233bz?4W>*K&la5O#1B3#m`b8JSYrL#4m$eLkpHI%91duLs z02MfhT*8+!F^%I5Wj!8om6mDw$E(g1INnPl4MuEj%*fbXa9fn?pwr|bz! zY=PKsg&YWILiN~DHSpwR@gQyBcQNt7q|W_x;{OMKLpD{GUfc9V>XOo8TRC!xnDjj?Apdx z>86YunY)aY=E~oK2CeXcdixZgGvR+|PD{rBXd@lxpTN{X*+$%k1*oLlBwr$ckit$1 zKs(PA?3E2ZKP?B)GoO59jJI-gVZnZj5Usdjn?V!mly|6#N$Cx#~ za5QSaaxatrVfRyLa_Jo*tEN6FLGxQyo6wCxI=fF*q9Lr85$tSk&kV7nJ6V6|hDrhb z-W~%gcI9gf#UcRN=R4=mfl-^Zcv$;XLH$ zWHw^6kwy0CE?%}OU__@;ywDWkXU5>J8*ObWVCU*!rjpgz(m{(w=CJ3uvt54mNyWbwCjIW{J5rh70^mEccS6*C`gO%`AaEIpWeLYA36^jL<9$tI8(DAqR`#@bhRxlaze^@E>+>EJ z?Xwaxmbh|KC_OCC9v?V~Ht9Msb#~mjb+@yA&C>pKkJ;T#swVcsDZTxyXpoaQ)dQ0c zCXg-R;`4O0FN;SZ;VXUB6u(T7EH{F^W+nVyf=4;vz22$YG(-2KCGnneaIKh&@k`~< zbPw>>Xu#@39jLVOLhqbnn3M)SPhY+yi_OM+J~)&fTWaHXc6xHRyvb-YN*Wlj9tg&t zNS~o6AJ~@al`94pi+W?M_~6M%lx(Zi`$w#{?OeZ7GpC5V0{3D=Sv$LL2jth?hSu6- zF48S&zLm46=8GBKJF)=CUKPU@tX_>q4i8r6R9^xn8o8Ma0IZd1QrUNXtN@Fh>ya}< z6a-T^DGt7tdo#Ep8k4bAgt)zPg_zb(A2#?iu1%r66BSueh0RxLIy< z>5sJ`x2X{=#R_XwPZU$@b=Yg)t2iqs_{5XID1YQwMr|Hy&BX5BeYSHFZz;p@$j=_2 z<1U)=C-H2nuNQj9EglVjY^PRu46V@5mAaao`dH~u8vkJq=>o4r?P9xbYwnT>Si<=Q zI2T}z=9JFHcd&WYIky)VVAG~5LiXrLjy|tEgu$?G*Ug|g@;FraS~mmi+9RR6%b%y2 z&_>RT@CGU7duhJZP|>BNh3rubiRz<%M8*JhCyB4l@sgARivc2_nU5p*2P8jDz?PbY zF*xm*Bsgt>DX}Xi0BUrDyX%B6W)4**!65l#*07&UEdR?3<=7-t*TRF^HaCYc&uK2! z!=??{`3JD5BSYY3PY2lHU}Ydzzu4j0iiz}ABwzhWKTtbd>bUXxR&vl&j_({F3R9}| zf9-4035>{R3n8>#&1VIQ16A-6RG6^=J_?}kPAw!R-naAeWFnF1bD}Zl!=qelbzjaA z`60X~{M=z6?d@T1qVnJ!4f?kbulAzujYz3itfR9V?bYtaJSparYgvhqvRjgj?#{rL$SM2|v)}CBDBJr=?!c z8vR`R$Vh`UBI}G>q;F(i!y;nIW7b_t7_q3?`?OO(lr;-F>Pt&NB6qFxYSy(*9%XHO z{#cj3`A;b%_KX6r9!h0K3eE}*Ogh{k>q^-voqGN;uG^T3Qy3c)VCYeKM*ly$R zo+F-$sy$~JxYf*u&rkW@58_ciA{)m36&8F{l{5h(=+A01wO{dt8fJ^7tECCK zZ~Wv#h>;0hqFg0D|CD?V?C5t(`78se1DPdeI-fVy>A%#mHFB>GT)$%@_?j2?MXG_< zy_M(@{>o@kEECk=upxhw;!eBO;<>f$uM2^fHA=wAmJ$Rf95?X$oDMKyOh>zf%MZ}f z;nz>Kf1WZj1b30M(Z>oaQ@M+k7C5&)DI0U_xkCj)2~}vvE1qMtq;Byxzr{p{yxb1W zj$_J7OwObMRjfpEJ}gRGmUdnmrBI?rzZ=U}#Drja3B)mi69=X|Q>FI@NxMIRNve-ovTg@S{K2wuR z`(ki8v6)%|P!iGI&QVw0*WM zZ51=R%k+~D9eq~|%+u9~8GJA!CgR4bPF>8V+TmsTB1AX4vkP457tK4YyGp$(h*hq< zgc3kdKU!-ST0@7^GkM_C@5WQVUNUpc{rRNlfq<)hg)Cs<^_SKlY*4CAB?|{RkyXs= zZv%3jXCmAb4(g=1@;=%50n9!rt3Z0p-mT+kFM`M7; z6EOo6l+vRvXC;t8Y)nf>C$5)s^AX#Y87YnOrKq)uw9|JGZ)yv2ji zx*j%AbRX}YSEY$a<%q)jFDwK($6JbwoH=eDUV>No!=3e%HFcg>*+=vOBn3nIRKg;{ zBC&+6><`!AU7l(d84@njMHP`4BJ+6~#<0$7m!kF`Ywc%0@_rZ48O8fVxM2Be7QNB8 ze83xYUs1K0&ZUcBS_?b>f7<)Xu&COuZAuA2x9r z5a|Y~TVSXmr9`AuQW%gyN?-`#-FTPxoyz_rlP!<3+Nc@A zj5s=@tOxX_%^93)C3Z1`AhQ*a6&%IuG%VM*uf2WO^>X4xsRVJMBIXJW7DtPt6{V<= zz47`>@a&|rb(r_9J~~Zbw<1RtE+J*_6>oDUWSA##wC%0ELPBNC2lVENn7_D*w`*b~ zqVWKC7gTV#1M1L|ee6c9r0WmrJ6f6f5(s&G0gDx1q#L}KS@bD0?Q_(iOVE`=<C2R|2%>h~YS`2H*vEf*v^R3u|Qm5#}Mh9EDU%)?2l^mJuO5q!JJ$1b^=yNSY z;j78FM9YZnwmOMAh@d8g^bUzV+OG=9>ADO-OhWFxNECY?4b3dlecRA|CXc~VlGZNC zO6`A#?O7RM7$&l9Hcufpx061)p4>PQ!4XC+pZ9E{e1>g~e+(~0N1DBwQZ{+H)p@Bo zP`&0-e6aZC8dedQCWOl3`Hfoe0I))39Aa`&Z>1HHti@_M&DHnL0lmkrvO?egI@~Wj z^|0yI%q2QwQYNuhT>VBl*$Ve-Q!Hj3F$dvOm)=*)>FJ6G5z5mF(xUuOUPgCh^fa)} zlE8x{%Le<)-IF|U#hamyW3gJDY;Ko%cU+lhw!PKVthH9e=U*(VTfEtXX@rR<+p1`U zpqb0;^eoF5Vb>Q0OI}i)vVt))K$sEI;f>6Ai=qu|@WRIzczya(oV34o;&w32 zYfn!I9wOynpooObC>oN3uvAqP&t@2BfB+NpnLM>1aVa^=(-lx&2a~JF;g~%yW#i+; z^zce4^4YjA8EF=3h-XbVwmV>KS9&}Td7+rxF|M(i09M|0%bUvGXHe977^=lrCrHRHSnvI}4i&ubtR2Vg~J1T&rWH5VR-ijUm>GD)NT z^i`_)3%v}sk3wA^yJ?P6c3tRw&h?q&^)pkZs2&5pZIKPm+hv>KR_F6c8{d#qtoZA_ zy2NJ{ws1m*=;hKZdu&7seRJqKCBW_&@oFvADp$m3(0fTbfFbtMLv~a zcLYxRtauwr<>iGaNXbZ1>GxF5)S_fX8_`cY4l)nnGhYfG&(DAi5l=g8ENcwYKmu`)d7 zI*MF}hxc)05GUhf%>$AH%>=kYR%x~ZrKCX#smXA?3C>s$+_+*8{|k0l4zjaFuK=8d zOKY5H!^-mL5gE}HoNKDA?K&Wdf%+r8WxsnKBL=h7JrEwf=;H9;2(ffEyGB;HQ1ABr z2_BtPEDM}6mj*lTT=`JgY1WdkP<-0EU5=vC>h}IZ9B+RVb+7S@f|&&Lz$TBD`HPA) z@fW4&f%uTo=pA=k4~>yfVL?W|V8? zW-}{O;-g?V#>2+@UYW~=_K9(`Ap~T@gQ{&s{BC1MwNc@dSE7HyY#&KW1ylB?VOv@U6uZKxxIS8)j5f@4ks4Ov3df$}f*}{K%-$$ElRr~yuD5qi35&yZz ztIGzY&^vd1P7jfuAmofnen-=A%GLfkwYLv4fFxhpPkUHzJ3Y}rqk^l>jG;E3u$L_< z-9ZV)kwA=lF&I|??^#uL*QlEP_{mMu;&yXOp_0Z24HG1tqiUdsq~wtmrD(Ne)=qAD z5@inMD+Pf&cY@aF!f^%6!91V~Q8B!eEiSXsmKUDe{3!%*{%yBVX`m)Y-o zMe=9s`k_*Ew>JThS|Qt_7{gK%N@vZeMbexIRGKR1` z(i@-+d^)A$dcjW3kkd6Y^>U41{CL!MkiKTIlLm?7Li(VsQ-p4r8eIejLT$`f&tupp zM-&9F6>8*iq8drd#@%%jWZ{ZviaF|+%w9Ce{v#)Y>NB=E=b zUA_{_u@tl&N7o68rM4D|;=?~a1ecwcE*e+`t-sWM5>E>5U{RD}$q=u%;-XgP-Sy_N zN1%tR#e^fy9{nkcon~pCVL& z1XBlg&qk{+r9pv^-DS$DOoDhql7Bg}uhV*2dW7PNb;a~SJ&;skD_s;j=Gi`OhslSE zEk!8e93|vr0KT{%LwWr1y_yShC4bZmCBAAcks0xfPI|6Jo)q^H=vX^7EtbJa>+{Ke?B)9eO$_Wt)2jZl2@dfx@ zc_cKmEzAQL8g~g2r6Zj%Hw;@&1?oU$H~VJ;9=evt;Y4{Ub89V<+E;0R&d)|ieh4=V zWXhm62JC5nl@*8+a!N^m7JCMKp-G+_i>6Km}qXx^+}mAzy-IEI~<+_(KKmxYA2A?{Jyim}B7_TqTv?0w~eNE$(U z7!{5#oj(DbczKi_AIi?=lJzCp;a(Zy5IEXuX{|uHkfC-*Vp8cNM!li6XcO~f3vz}h z#OY0w^w{>On%qFEGYfUWVpx!CBzH}5plzbFy=2abhveai>TTq~RhhZ#l3H{_7xpYX z!G3Kv-3AsT2>mT#ZcdY`xGDJyA&|N4+QSh5S)4ff@rk8 z7s_YdBV$F&1@c`Xc-eZ&4O|cESxJ_w=71PP+$$7MsUEUkfYKlMO&5!I&agjJ$Iw}c zMIPUGm6glvOWa-*MsQ8`fd9lmuI6#=eh7MCz!dV%{0nP)VXLROXs8gYwER;_u9&_Vn0OTi7zjZsx=r!>vwg=}0H z$FPV^AdfOybQE^OI?fPUaTJ%1DcmaEHGA$>P_DfK5|YUH@UHgt@xUGsvv9M7?KqTV z8h^4JkiKsKyH~d(Qc+x9zf%vC7LQDNf+*M6Cj=(O{B_rwHwM#i@YCq=Noqgk_?3^9 zq9H4TfXDGb4wlj$gL}0k7aeCd!##g{4G&trT5{_fCP(%K4xpTCnf6MgZS^InYYmb? zG^+AAJB}hw_nfR?HFamEpaxo1OSF>-Cs@v5#UJPxvFu|HrUe(F&#WuTr~MuO!~%fn zt5GdkJKnD4904V1r!Vrz+pbR&SkKAP*zQ@@n7@h(QI}`$$FkRcb>gZ@Si*@a5X`8e z(`(U6L(h;;_Py&yD5}`mU8Nf9gQ%BnZ4rk@B#I#PVQ;ey57Xm?BSBEdLxPRUNYF0F zrw<1}adUUlTIuSxI3$m3vJA`@z7%1-t>Nu~7Dw!+Mr{keQG<}ZBz1W;Gh=w@159$Z z!4d=&(25?07WZx9fb-&u-xrUpdCw}X;>2iX?NKa<&9;TO8E%)Ag9&!qS zz#D|LA{o|!IPF}0C%+8Gn|&Bu4DkT;w&pVcHj~|C%S9WVYEVGcY3f11)HBwPW`^US zxAN3|W9$j>52VgVvl{G`#;GV=rNmF1; z+{vhjtjdfk7mF6ntX3etdA*ue>bNC;B2qT7trr-c!=Op%fTmFIXU3{FDu%2$%p}yM zJ*X^4KVAT4U1ueYX_iUc;fP#Fb*`4<+Q8pV+KpdUD}mxbYK8hEEgxTrrS>&>x*QQ0 zD{DL`t>p`k=g1%<(uD%0)~mQoVQEfdF!dmzN#I7x;CR=|>Z!@~%1cq6+4E#0*>7>Y zb@KE8)Itu*vi50|b-J;4qeDd1&>%kdXds>nH6mthH}VZbbSR z!8DD(nk}uUuS~3~Yfn9LwAC+9E2HS81C^{tOqAcYdVJjjuRnvGc8QwRV!X6AC`Y67 zn(FFQL%EUWG2hB&Xuf5GcroRHO7FrY*7xJ3Qc5|8o=$f0LF*h*N#AVy_*~ssUH%zb#h!Rr`D;EC_yATRTIzU6wikQ%jmN`^_1c%& zYS8HV+N2>gM~kgJ=|x{Ga!%HctdC2z4L9~-%OA8ND7txW+Z%I+Sl}$xPN={%eVdXu zMlvgPtVaxbuC(#h@53}Xx_A8_oFAiQSM8AO@8-% zLS5&9zdOFysFA+_~X2pb5$774% zcIu}nlEPn-oC9XscMN*-1E>hFW#s7_PqjRj@%rCz3Ap2gP^g5ZC;O_{6XxbL^j zz&)J{;pSJ=-~YVeW73G8-92bQr|;L67-*Fwqtgc#;_HWJ)wyZ9TOw6dG5Da#Ri!Eh zk1x344U4tiVvnp7wiQa*Y6icjmi)lq=6F(AFJyZIz++fVSEZL;W-s_Mo|zUzk?6%~ zn6e@V8IQ;7a$iV+s2DIxf^!^SRw1gULy~I6-~Ns>%@xP0M;M?C{5h)&q9Pn{w(xa} zN~QV_a%=J6-0u?S50($=ZAHj~sNtr9H9#~(m=Wr>_~v$D!dzpEbHtERr%;C<1o^sj zrr~75lHB!k9^T`*=(E6$w1d|^0~eh>uend?fJ07wti#qnxB^8xz3vnve4j%%@&vP% z+M|Tc{2qs!1NT!ls*HdMe{i13cHCR5?9(hV!?a;7xOijiOmF~j#m`7#oo6$n99$~V zVtlChni2%D669VaEQu$r!9ICn1ee$(hEw_V{*ucUa%`TKLNbKa0v zT|iMs4`mt|(T>dRT8EXP$@Z8lRVCBW1aF~{r)+1#ZKRxXBt_tX7tfN}l@*QR zsZxDO>s9OcVy%aRABD@bn1lTIe0nh_Bl%TU-++ z)O*j|J**9_F?@C1LV9y;-wEiHTVly_TnQ#}Sr@|I~gD zG>qL*!Jb`o5Hq%5@0_xD=*mSxW^X;&A0XPwImjKx@Ia+?lKb|~%W29yRmEY+iHa8E zjX``d-WKxknDCvJk!m({*lzo=tN=ijp4~e?{3aE%ixyr{?!jPXzYLPYOI>tfRu**t zN&)&>DUQN(Tle*BGF*q%FcfWP?>8f61SUmp6yulA5SDz6tE_?$-EtO)n%y} z*omLntdBa$k>VHZnvjd1PD)wtQc?}#icNjN#;(i~PjDa;i)gWAcXq0KtsMA~3de>H z#HE|3_g4Hx9AG$lkOxrBgV)2r{#VAiXt*%A$$-xt?TO{vJ(pH$S@g+G^FYxhy8X`c zfh$B%Dw)J@aQ9@qaWceMGW8Tc+7T_5xQ3|g_GrJ_UU*d{!+9#?q$9R-lGzpiUHVws z6cSCYpQEZ8x7G|eR9MO!Axs&jzMrFUOvTD<=C;sm_GF&(+!XZBF8HHw_8@a>7USk5 zQ4A=alkV|zbFgN_HDD79kds?R6*(-fPrWW7eZed!Ru7nl`@Mz8y9irtaNgD>+ORK4J9ROoNBQf?5$R}wsjzU% z_oCYuU^heL7V%sZQJSYrtW%M)z@@Y0nzfG)oeT?KZCC zNSTC{uv-84T+O*L0{htj9Eah+M@l=2rm%`d^>N>%iG0SPRD^SHn5qTnl&yvU#vS6q z!}yxyz&yMyv`y8CL@wEl0?AG`ct@3hpJr)JD2#k}UJ@;erYb35e*lvnR>~tfy*$}v zFyH3AT^~21`-E+d1>PrSj&k9%xEcNI96wsdk)3?=iP0ixZUI0(TAq-SFPcV>ZeJ6I zhu_Qs0L5imITlgqPwa-w?K{mK&NSPJ6zfZV$gCw%(rkUkkF6bToQ3VCeJA_*kgM?c z)ZybHH$=|Iv|6l8e^y1IP4?}2z;b*^xUCJV_*QUFE34+;lJq_>*XX3WfpHM!7guGY zv&Xo(EEDLVjYY=A0))5His#|&ULtzH8Xeg6v>n|&+U`1!n{LMAYqf(7aROjIWl+~} z{GJuzq4?F&)qNKYy#|wM_f`Hy0p%M(plL4}Q5t!geye1%)saSJ`AdNq2<*&90D|YY zYcf0Gfp{}Ca$@6&F3MK-$6Xo(ArAFGv^#vOgyu78Z+_D)c6n?LI7y9wNuRP+4t=yM zA*uyEU==DH9@u+mhoeklgT73t$7a~+O-r0f&4R5Hmy}5p2nJFxk|jdY=zh^`7A-|AMi;GMs{m3`w)I;qkV0DCBVNGF8hJz}@!=lGDQXM?&8-Hm!`Q<*TFgr#r& zm2(X{wyyTvUMS+@^$)&D2T9_Yy3L7b4zZx!Q-aHku>AZE16m>@jGC^Zpe$`rI33NL$ z#p{yv9oH&DY0m7|#T?u00U9qR=NYZu(s)pHa^B!P6HS=scgyFO#Io#JKxX7X6Ol^YE)`}{#eA)Py_){CV zCdW>6tTV4KLbYLyu*bM}&fk z2B*el(KgODVR%7!s#l!^%W6drnv>vh^{Rz8%jikX?~y%if5|5Mn;B%SHa$LiM#}-! zsJJ2lf*$)k&%;VPwt`cu>+J9O9-MyTAUR?YSgar6f5};woH!bJ@wv>i6#eojVMAM5 z9m+U4n{tkyoRvKn4fnZde&-3jPQ@q-zSfMYb-LhGgpdDqd0N-wF?b*xAJu;jOqpQB z91zmEA5G?JeGC9GO!qfI3DpmV#gSI{pu|tL--uD|GaR|h4IbVH$TIPXqun=~^aqvb z8PV+Lp1+fjTA!Ssiwf%iygfc9jvtvxtw8{*smhD+YaV%Qz5jOV@fS^iceU2LKJ@#~ zORxh5{xbBOJP1ql3i(zK{e!D@{@z~*fcI`)6#NpUB{p%?ZMlE-FN&9}^|{>pD4>+k zSGOX5FDCs(Ujiy4Rm{1}6QC;!ZL1-c6;LS~XWhS8jrv!CMPoJozgske1o}$Q=LaEu zC$B#)kE8=taa&}Z6N1Jd?R`|S*C51ZsxX2ma+50SI`j;ery!9Iy0w83q+F@Z^ zY#!W(ZA{yGt#@7Otfjo^t2OdFQ1Lr*+P_?WxsMf$8(O{YWdPq_)qA-S_t<>x!6ba2 zOJDovw+EVPnf2vjnx_mYf7vnhaIdl)xI8D~r(;-+D5zo-EJgiIH2Y6d+F$fGpYZm+ zXV_hrmqHIlrzcSM63Vbv=lysA-;P=l^{yBHN#y)aYyP@yjclKa3+@W3QSb%ld~z=Y z`s?+s@DUM@+?^r4z3~grz&;lv4ow#9@>{NQp32g*ij^FMR_| zoU8Yxq@w9=RExTo+=l}1ahKM4VFi~O=hX3Us`wI^-di!SUaH`TqRqKT5Pt)vOf|<^ED=qjz%s!CovG^uvMs1`%P^KO0{Dn<53e zYnts6g@(F5zgV??|GM8-;N}!wDDC|KEAuy^$bYuL`AhQucpF(H*_8W9pFopvgYOWt zCjjFQD;hFnSm86B{f!Ng{VQLx07sdao}%HMm*JXg>)NAOawwADl(cZza6E%%hWbC* zi9bp}B{vLT>aulYUkywEH_{SUns~lE+aXi91*Za8IJ&e{>wki7#hD^z9_zfrQ zTQ;hGW%);cYO`d#PH$w@E6cYF@0jk8Bv+wSK6@Db!V_?j&|GH@$r7cfz34uhH;K}c z0L~$I-KhD9Jx5GXx!=;azxLn1tJ4ui;5|_~stG3PV$03^=iLUyGE$!QUQV-Pv3?jH zzvH6*{mwuC7tcK5wL&L)FYMv<$XIIVGa2`uJb?h$a)s|2jQ;P!l!$FFHplKl#Dsa8 z0O!r0$rslnn>N?)zM&3PQvZO_uyiY5^bd~ezi7c99M#b(JVGWZW>j^5xPZXtgPFAo z?eHq8kOG~+EFr*1)oNfiyumH_`ZtmGe|er&CY;dA@?MNrE*@N17Ll%%1}1TyBzF&V z>De}6H3YOo&KCUayUKEs(}-J{P2EVQyCPMcu$c!iGl3kbLMI3vD6Za#y71~~SZ_z` zPvQNScM8LcW)$ag61OqT;S$LV;PumjX^M(EZ|}IAp*R5B{fgw^*}V&tM$u$L z!ox89$T8=hA5bCJCfeRx=uHXXs0`Y8fjv$CQ~S4?g%he=?|yPo0ejIA>ikt6rb+C& zAGBi4BWzLWyZVGQiBOy%4Z}&+In(jBvJ{5quFjexKAX)a+w?#%Q?0J=>oL}!$ z+f7GSJ9Q6K^WG=U_$qpQ22yx`W8&wY6BAz%tFI*ZdAo$yd_sT)hxzH*RtI+1>Oyem zQ+UfWM{>5@xu_~lfgO@xFnY3B1pvxMo%Fnd_tQy)0ovYz(H6Q@~zJJ2M{?#9^!^H^%fx~@l|7Gz%dFhz=iRai| zfdBk+bpf7Z;Jn}|yz>L|{kxs=_vQbI`8#*{e`5Y_ul=7ke`kAs#c0T@sVre&T$EB( zl+*ufqyE1%AfubzlawF@!={HyhSgR|C!0H{wy2cufVywF?RQWAZvfUVz~j>>nV%twZ%=trqxn|ck6fup)GjGtTZ~yCMJzg}eAlG^B(!ko#R_D(GAHG{B zrIFJ%4zB!at?!%jmwf-Wk^lU$)=-Qx^WGRbvh#Rv@{7Hud%kv573zb}8LE3BqUlN5 zuf)BEo;W1tjdYT{gpA{!qM;iv7T3QVzUowX!hgr}c`xW!!v7C=%+2@V9i!FT#T|a} z8aY7gtgQMezq`jDUG@rpYWt6{F!zH$_;i{3EkIsIy#iUTgFNe5XZt@0Q~%cJ2=&gL z4f~mYl+EFgrCkIf-zyjQpYb64Y?eWt>vgnI@8{}A;Qq6zKfNEyRy{i!XxxSFzGtu4V zvlg$2Jm*tiy!xrw$coj?z5+2*1HHS=;X*9MpXuVqr@xfsFb8h*5)rQxF;2O9P5`sF z{>---oFsp2i1l|E@aJPClFNE2J=&<70HEk)rjqxJNt} +``` +It might be a good idea to add a corresponding line to your `.bashrc` or `.profile`. + +### Download & Install Hadoop Utils for Windows + +If you are trying to run the application on Windows, you also need the *Hadoop Winutils*, which is a set of +DLLs required for the Hadoop libraries to be working. You can get a copy at https://github.com/kontext-tech/winutils . +Once you downloaded the appropriate version, you need to place the DLLs into a directory `$HADOOP_HOME/bin`, where +`HADOOP_HOME` refers to some arbitrary location of your choice on your Windows PC. You also need to set the following +environment variables: +* `HADOOP_HOME` should point to the parent directory of the `bin` directory +* `PATH` should also contain `$HADOOP_HOME/bin` + +The documentation contains a [dedicated section for Windows users](cookbook/windows.md) + + +## 2. Downloading Flowman + +Since version 0.14.1, prebuilt releases are provided on the [Flowman Homepage](https://flowman.io) or on [GitHub](https://github.com/dimajix/flowman/releases). This probably is the simplest way to grab a working Flowman package. Note that for each release, there are different packages being provided, for different Spark and Hadoop versions. The naming is very simple: @@ -35,15 +69,14 @@ https://github.com/dimajix/flowman/releases/download/0.20.1/flowman-dist-0.20.1- ``` - -## Building Flowman +### Building Flowman As an alternative to downloading a pre-built distribution of Flowman, you might also want to [build Flowman](building.md) yourself in order to match your environment. A task which is not difficult for someone who has basic experience with Maven. -## Local Installation +## 3. Local Installation Flowman is distributed as a `tar.gz` file, which simply needs to be extracted at some location on your computer or server. This can be done via @@ -57,8 +90,6 @@ tar xvzf flowman-dist-X.Y.Z-bin.tar.gz ├── bin ├── conf ├── examples -│   ├── plugin-example -│   │   └── job │   ├── sftp-upload │   │   ├── config │   │   ├── data @@ -89,7 +120,7 @@ tar xvzf flowman-dist-X.Y.Z-bin.tar.gz * The `examples` directory contains some example projects -## Configuration +## 4. Configuration (optional) You probably need to perform some basic global configuration of Flowman. The relevant files are stored in the `conf` directory. @@ -187,7 +218,7 @@ plugins: ### `default-namespace.yml` On top of the very global settings, Flowman also supports so called *namespaces*. Each project is executed within the -context of one namespace, if nothing else is specified the *defautlt namespace*. Each namespace contains some +context of one namespace, if nothing else is specified the *default namespace*. Each namespace contains some configuration, such that different namespaces might represent different tenants or different staging environments. #### Example @@ -229,9 +260,27 @@ store: ``` -## Running in a Kerberized Environment +## 5. Running Flowman + +Now when you have installed Spark and Flowman, you can easily start Flowman via +```shell +cd +export SPARK_HOME= + +bin/flowshell -f examples/weather +``` + + +## 6. Related Topics + +### Running Flowman on Windows +Please have a look at [Running Flowman on Windows](cookbook/windows.md) for detailed information. + + +### Running in a Kerberized Environment Please have a look at [Kerberos](cookbook/kerberos.md) for detailed information. -## Deploying with Docker -It is also possible to run Flowman inside Docker. This simply requires a Docker image with a working Spark and -Hadoop installation such that Flowman can be installed inside the image just as it is installed locally. + +### Running in Docker +It is also possible to run Flowman inside Docker. We now also provide some images at +[Docker Hub](https://hub.docker.com/repository/docker/dimajix/flowman) diff --git a/docs/quickstart.md b/docs/quickstart.md index 0a429f9d9..9603d53bc 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -14,7 +14,7 @@ Fortunately, Spark is rather simple to install locally on your machine: ### Download & Install Spark -As of this writing, the latest release of Flowman is 0.18.0 and is available prebuilt for Spark 3.1.2 on the Spark +As of this writing, the latest release of Flowman is 0.22.0 and is available prebuilt for Spark 3.2.1 on the Spark homepage. So we download the appropriate Spark distribution from the Apache archive and unpack it. ```shell @@ -22,8 +22,8 @@ homepage. So we download the appropriate Spark distribution from the Apache arch mkdir playground cd playground# Download and unpack Spark & Hadoop -curl -L https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | tar xvzf -# Create a nice link -ln -snf spark-3.1.2-bin-hadoop3.2 spark +curl -L https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz | tar xvzf -# Create a nice link +ln -snf spark-3.2.1-bin-hadoop3.2 spark ``` The Spark package already contains Hadoop, so with this single download you already have both installed and integrated with each other. @@ -31,12 +31,12 @@ The Spark package already contains Hadoop, so with this single download you alre ## 2. Install Flowman You find prebuilt Flowman packages on the corresponding release page on GitHub. For this quickstart, we chose -`flowman-dist-0.17.0-oss-spark3.0-hadoop3.2-bin.tar.gz` which nicely fits to the Spark package we just downloaded before. +`flowman-dist-0.22.0-oss-spark3.2-hadoop3.3-bin.tar.gz` which nicely fits to the Spark package we just downloaded before. ```shell # Download and unpack Flowman -curl -L https://github.com/dimajix/flowman/releases/download/0.17.0/flowman-dist-0.18.0-oss-spark3.1-hadoop3.2-bin.tar.gz | tar xvzf -# Create a nice link -ln -snf flowman-0.18.0 flowman +curl -L https://github.com/dimajix/flowman/releases/download/0.22.0/flowman-dist-0.22.0-oss-spark3.2-hadoop3.3-bin.tar.gz | tar xvzf -# Create a nice link +ln -snf flowman-0.22.0 flowman ``` ### Flowman Configuration @@ -68,13 +68,9 @@ That’s all we need to run the Flowman example. ## 3. Flowman Shell -The example data is stored in a S3 bucket provided by myself. In order to access the data, you should provide valid -AWS credentials in your environment (not needed any more, since the example uses anonymous authentication): - -```shell -$ export AWS_ACCESS_KEY_ID= -$ export AWS_SECRET_ACCESS_KEY= -``` +The example data is stored in a S3 bucket provided by myself. Since the data is publicly available and the project is +configured to use anonymous AWS authentication, you do not need to provide your AWS credentials (you even do not +even need to have an account on AWS) ### Start interactive Flowman shell diff --git a/docs/spec/index.md b/docs/spec/index.md index 602af6927..4ef18e91f 100644 --- a/docs/spec/index.md +++ b/docs/spec/index.md @@ -1,4 +1,4 @@ -# Project Specification +# **Reference Documentation** Flowman uses so called *flow specifications* which contain all details of data transformations and additional processing steps. Flow specifications are provided by the user as (potentially From 62d8f9773387da3c677e6df9055e98a2dde11e56 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Thu, 24 Feb 2022 19:32:17 +0100 Subject: [PATCH 82/95] Minor documentation update --- docs/conf.py | 4 ++-- docs/cookbook/data-qualioty.md | 4 ++-- docs/cookbook/impala.md | 2 +- docs/cookbook/validation.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c79f24d29..1c1898c26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = '0.20' +version = '0.22' # The full version, including alpha/beta/rc tags. -release = '0.20.0' +release = '0.22.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/cookbook/data-qualioty.md b/docs/cookbook/data-qualioty.md index bf4559f9e..873f9085b 100644 --- a/docs/cookbook/data-qualioty.md +++ b/docs/cookbook/data-qualioty.md @@ -1,4 +1,4 @@ -# Data Quality +# Data Quality Checks Data quality is an important topic, which is also addressed in Flowman in multiple, complementary ways. @@ -36,7 +36,7 @@ targets: ``` -## Data Quality Tests as Documentation +## Data Quality Checks as Documentation With the new [documentation framework](../documenting/index.md), Flowman adds the possibility not only to document mappings and relations, but also to add test cases. These will be executed as part of the documentation (which is diff --git a/docs/cookbook/impala.md b/docs/cookbook/impala.md index 5b55d508f..a2f710ee5 100644 --- a/docs/cookbook/impala.md +++ b/docs/cookbook/impala.md @@ -1,4 +1,4 @@ -# Impala +# Updating Impala Metadata Impala is another "SQL on Hadoop" execution engine mainly developed and backed up by Cloudera. Impala allows you to access data stored in Hadoop and registered in the Hive metastore, just like Hive itself, but often at a significantly diff --git a/docs/cookbook/validation.md b/docs/cookbook/validation.md index 1b9d69895..f449e9cde 100644 --- a/docs/cookbook/validation.md +++ b/docs/cookbook/validation.md @@ -1,4 +1,4 @@ -# Validations +# Pre-build Validations In many cases, you'd like to perform some validation of input data before you start processing. For example when joining data, you often assume some uniqueness constraint on the join key in some tables or mappings. If that From 2a35269eab62e53d0f8b4ebf9c0b057a2e922164 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 25 Feb 2022 09:59:34 +0100 Subject: [PATCH 83/95] Refactor migration infrastructure --- .../dimajix/flowman/catalog/TableChange.scala | 27 ++-- .../{jdbc => catalog}/TableDefinition.scala | 22 +++- .../flowman/execution/RootContext.scala | 6 +- .../dimajix/flowman/jdbc/BaseDialect.scala | 3 +- .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 13 ++ .../dimajix/flowman/jdbc/SqlStatements.scala | 3 +- .../dimajix/flowman/types/SchemaUtils.scala | 5 +- .../flowman/catalog/TableChangeTest.scala | 121 +++++++++--------- .../flowman/jdbc/BaseDialectTest.scala | 4 +- .../dimajix/flowman/jdbc/DerbyJdbcTest.scala | 1 + .../com/dimajix/flowman/jdbc/H2JdbcTest.scala | 7 +- .../dimajix/flowman/jdbc/JdbcUtilsTest.scala | 8 +- .../spec/relation/DeltaFileRelation.scala | 6 +- .../flowman/spec/relation/DeltaRelation.scala | 22 ++-- .../spec/relation/DeltaTableRelation.scala | 5 +- .../spec/relation/SqlServerRelation.scala | 5 +- .../spec/relation/HiveTableRelation.scala | 27 ++-- .../flowman/spec/relation/JdbcRelation.scala | 37 +++--- .../spec/relation/JdbcRelationTest.scala | 2 +- 19 files changed, 198 insertions(+), 126 deletions(-) rename flowman-core/src/main/scala/com/dimajix/flowman/{jdbc => catalog}/TableDefinition.scala (56%) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala index 72a6f2f0a..4997b4c2e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,9 @@ import com.dimajix.flowman.types.SchemaUtils.coerce import com.dimajix.flowman.types.StructType -abstract sealed class TableChange +abstract sealed class TableChange extends Product with Serializable abstract sealed class ColumnChange extends TableChange +abstract sealed class IndexChange extends TableChange object TableChange { case class ReplaceTable(schema:StructType) extends TableChange @@ -38,6 +39,11 @@ object TableChange { case class UpdateColumnType(column:String, dataType:FieldType) extends ColumnChange case class UpdateColumnComment(column:String, comment:Option[String]) extends ColumnChange + case class AddPrimaryKey(columns:Seq[String]) extends IndexChange + case class DropPrimaryKey() extends IndexChange + case class AddIndex(name:String, columns:Seq[String]) extends IndexChange + case class DropIndex(name:String) extends IndexChange + /** * Creates a Sequence of [[TableChange]] objects, which will transform a source schema into a target schema. * The specified [[MigrationPolicy]] is used to decide on a per-column basis, if a migration is required. @@ -46,10 +52,10 @@ object TableChange { * @param migrationPolicy * @return */ - def migrate(sourceSchema:StructType, targetSchema:StructType, migrationPolicy:MigrationPolicy) : Seq[TableChange] = { - val targetFields = targetSchema.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)) + def migrate(sourceTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy) : Seq[TableChange] = { + val targetFields = targetTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)) val targetFieldsByName = targetFields.toMap - val sourceFieldsByName = sourceSchema.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap + val sourceFieldsByName = sourceTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap val dropFields = (sourceFieldsByName.keySet -- targetFieldsByName.keySet).toSeq.flatMap { fieldName => if (migrationPolicy == MigrationPolicy.STRICT) @@ -89,19 +95,18 @@ object TableChange { dropFields ++ changeFields } - def requiresMigration(sourceSchema:StructType, targetSchema:StructType, migrationPolicy:MigrationPolicy) : Boolean = { + def requiresMigration(sourceTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy) : Boolean = { // Ensure that current real Hive schema is compatible with specified schema migrationPolicy match { case MigrationPolicy.RELAXED => - val sourceFields = sourceSchema.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap - targetSchema.fields.exists { tgt => + val sourceFields = sourceTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap + targetTable.fields.exists { tgt => !sourceFields.get(tgt.name.toLowerCase(Locale.ROOT)) .exists(src => SchemaUtils.isCompatible(tgt, src)) } case MigrationPolicy.STRICT => - SchemaUtils.normalize(sourceSchema) - val sourceFields = SchemaUtils.normalize(sourceSchema).fields.sortBy(_.name) - val targetFields = SchemaUtils.normalize(targetSchema).fields.sortBy(_.name) + val sourceFields = SchemaUtils.normalize(sourceTable.fields).sortBy(_.name) + val targetFields = SchemaUtils.normalize(targetTable.fields).sortBy(_.name) sourceFields != targetFields } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/TableDefinition.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala similarity index 56% rename from flowman-core/src/main/scala/com/dimajix/flowman/jdbc/TableDefinition.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala index 4c69f6871..8c411cb6e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/TableDefinition.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,33 @@ * limitations under the License. */ -package com.dimajix.flowman.jdbc +package com.dimajix.flowman.catalog import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.CatalogTable import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.StructType +object TableDefinition { + def ofTable(table:CatalogTable) : TableDefinition = { + val sourceSchema = com.dimajix.flowman.types.StructType.of(table.dataSchema) + TableDefinition(table.identifier, sourceSchema.fields) + } +} case class TableDefinition( identifier: TableIdentifier, fields: Seq[Field], comment: Option[String] = None, - primaryKey: Seq[String] = Seq() + primaryKey: Seq[String] = Seq(), + indexes: Seq[TableIndex] = Seq() ) { + def schema : StructType = StructType(fields) } + + +case class TableIndex( + name: String, + columns: Seq[String] +) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala index 7f4b5c435..84023c833 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/execution/RootContext.scala @@ -321,7 +321,7 @@ final class RootContext private[execution]( } // We need to instantiate the projects job within its context, so we create a very temporary context - def getJob(name:String) : Job = { + def getImportJob(name:String) : Job = { try { val projectContext = ProjectContext.builder(this, project) .withEnvironment(project.environment, SettingLevel.PROJECT_SETTING) @@ -337,10 +337,10 @@ final class RootContext private[execution]( _imports.get(project.name).foreach { case(context,imprt) => val job = context.evaluate(imprt.job) match { case Some(name) => - Some(getJob(name)) + Some(getImportJob(name)) case None => if (project.jobs.contains("main")) - Some(getJob("main")) + Some(getImportJob("main")) else None } job.foreach { job => diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala index 241e22490..d5d6f611c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,7 @@ import com.dimajix.flowman.catalog.TableChange.DropColumn import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.MergeClause diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index b46134aa3..e522897d7 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -49,6 +49,7 @@ import com.dimajix.flowman.catalog.TableChange.DropColumn import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.StructType @@ -156,6 +157,18 @@ object JdbcUtils { } } + /** + * Returns the table definition of a table + * @param conn + * @param table + * @param options + * @return + */ + def getTable(conn: Connection, table:TableIdentifier, options: JDBCOptions) : TableDefinition = { + val currentSchema = JdbcUtils.getSchema(conn, table, options) + TableDefinition(table, currentSchema.fields) + } + /** * Returns the schema if the table already exists in the JDBC database. */ diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala index 178b91878..92b31eb0b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.MergeClause diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala index ddb96ea1a..76c75219d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/types/SchemaUtils.scala @@ -30,7 +30,10 @@ object SchemaUtils { * @return */ def normalize(schema:StructType) : StructType = { - com.dimajix.flowman.types.StructType(schema.fields.map(normalize)) + com.dimajix.flowman.types.StructType(normalize(schema.fields)) + } + def normalize(fields:Seq[Field]) : Seq[Field] = { + fields.map(normalize) } private def normalize(field:Field) : Field = { Field(field.name.toLowerCase(Locale.ROOT), normalize(field.ftype), field.nullable, description=field.description) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala index e94ee5e52..0fe454118 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.catalog +import org.apache.spark.sql.catalyst.TableIdentifier import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -35,184 +36,186 @@ import com.dimajix.flowman.types.VarcharType class TableChangeTest extends AnyFlatSpec with Matchers { "TableChange.requiresMigration" should "accept same schemas in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (false) } it should "not accept dropped columns in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), - StructType(Seq(Field("f1", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType))), MigrationPolicy.STRICT ) should be (true) } it should "not accept added columns in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType))), - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (true) } it should "not accept changed data types in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", IntegerType), Field("f2", StringType))), - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", IntegerType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (true) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), - StructType(Seq(Field("f1", IntegerType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", IntegerType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (true) TableChange.requiresMigration( - StructType(Seq(Field("f1", LongType), Field("f2", StringType))), - StructType(Seq(Field("f1", IntegerType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", IntegerType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (true) TableChange.requiresMigration( - StructType(Seq(Field("f1", VarcharType(10)), Field("f2", StringType))), - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", VarcharType(10)), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), MigrationPolicy.STRICT ) should be (true) } it should "not accept changed nullability in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, true))), - StructType(Seq(Field("f1", StringType, false))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, true))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, false))), MigrationPolicy.STRICT ) should be (true) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, false))), - StructType(Seq(Field("f1", StringType, true))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, false))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, true))), MigrationPolicy.STRICT ) should be (true) } it should "accept changed comments in strict mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = Some("lala")))), - StructType(Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = Some("lala")))), - StructType(Seq(Field("f1", StringType, description = Some("lolo")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lolo")))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = None))), - StructType(Seq(Field("f1", StringType, description = Some("lolo")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = None))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lolo")))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = Some("lala")))), - StructType(Seq(Field("f1", StringType, description = None))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = None))), MigrationPolicy.RELAXED ) should be (false) } it should "accept same schemas in relaxed mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), - StructType(Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (false) } it should "handle data type in relaxed mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", LongType), Field("f2", StringType))), - StructType(Seq(Field("f1", IntegerType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", IntegerType), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", IntegerType), Field("f2", StringType))), - StructType(Seq(Field("f1", LongType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", IntegerType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (true) } it should "accept removed columns in relaxed mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", LongType), Field("f2", StringType))), - StructType(Seq(Field("f1", LongType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType))), MigrationPolicy.RELAXED ) should be (false) } it should "not accept added columns in relaxed mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", LongType))), - StructType(Seq(Field("f1", LongType), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", LongType), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (true) } it should "accept changed comments in relaxed mode" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = Some("lala")))), - StructType(Seq(Field("F1", StringType, description = Some("lolo")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType, description = Some("lolo")))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = None))), - StructType(Seq(Field("F1", StringType, description = Some("lolo")))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = None))), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType, description = Some("lolo")))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, description = Some("lala")))), - StructType(Seq(Field("F1", StringType, description = None))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, description = Some("lala")))), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType, description = None))), MigrationPolicy.RELAXED ) should be (false) } it should "handle changed nullability" in { TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, true), Field("f2", StringType))), - StructType(Seq(Field("F1", StringType, false), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, true), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType, false), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (false) TableChange.requiresMigration( - StructType(Seq(Field("f1", StringType, false), Field("f2", StringType))), - StructType(Seq(Field("F1", StringType, true), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType, false), Field("f2", StringType))), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType, true), Field("f2", StringType))), MigrationPolicy.RELAXED ) should be (true) } "TableChange.migrate" should "work in strict mode" in { - val changes = TableChange.migrate( - StructType(Seq( + val oldTable = TableDefinition(TableIdentifier(""), + Seq( Field("f1", StringType, true), Field("f2", LongType), Field("f3", StringType), Field("f4", StringType), Field("f6", StringType, false) - )), - StructType(Seq( + ) + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( Field("F1", StringType, false), Field("F2", StringType), Field("F3", LongType), Field("F5", StringType), Field("F6", StringType, true) - )), - MigrationPolicy.STRICT + ) ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.STRICT) changes should be (Seq( DropColumn("f4"), @@ -225,23 +228,25 @@ class TableChangeTest extends AnyFlatSpec with Matchers { } it should "work in relaxed mode" in { - val changes = TableChange.migrate( - StructType(Seq( + val oldTable = TableDefinition(TableIdentifier(""), + Seq( Field("f1", StringType, true), Field("f2", LongType), Field("f3", StringType), Field("f4", StringType), Field("f6", StringType, false) - )), - StructType(Seq( + ) + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( Field("F1", StringType, false), Field("F2", StringType), Field("F3", LongType), Field("F5", StringType), Field("F6", StringType, true) - )), - MigrationPolicy.RELAXED + ) ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) changes should be (Seq( UpdateColumnType("f2", StringType), diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala index dccc92aef..b8def97e7 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.types.StructType import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.PartitionSpec +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.UpdateClause @@ -79,7 +81,7 @@ class BaseDialectTest extends AnyFlatSpec with Matchers { it should "provide CREATE statements with PK" in { val dialect = NoopDialect val table = TableIdentifier("table_1", Some("my_db")) - val tableDefinition = TableDefinition( + val tableDefinition = catalog.TableDefinition( table, Seq( Field("id", com.dimajix.flowman.types.IntegerType, nullable = false), diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala index ab4a6ecef..c359dcacc 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.StringType diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala index 183e13790..8dad118b0 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala @@ -29,6 +29,9 @@ import org.apache.spark.sql.types.StructField import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.catalog +import com.dimajix.flowman.catalog +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.UpdateClause @@ -74,7 +77,7 @@ class H2JdbcTest extends AnyFlatSpec with Matchers with LocalSparkSession { "JdbcUtils.mergeTable()" should "work with complex clauses" in { val options = new JDBCOptions(url, "table_002", Map(JDBCOptions.JDBC_DRIVER_CLASS -> driver)) val conn = JdbcUtils.createConnection(options) - val table = TableDefinition( + val table = catalog.TableDefinition( TableIdentifier("table_001"), Seq( Field("id", IntegerType), @@ -171,7 +174,7 @@ class H2JdbcTest extends AnyFlatSpec with Matchers with LocalSparkSession { it should "work with trivial clauses" in { val options = new JDBCOptions(url, "table_002", Map(JDBCOptions.JDBC_DRIVER_CLASS -> driver)) val conn = JdbcUtils.createConnection(options) - val table = TableDefinition( + val table = catalog.TableDefinition( TableIdentifier("table_001"), Seq( Field("id", IntegerType), diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala index d046d1863..ec2047f96 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala @@ -23,7 +23,9 @@ import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableChange +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.types.BooleanType import com.dimajix.flowman.types.Field @@ -73,7 +75,7 @@ class JdbcUtilsTest extends AnyFlatSpec with Matchers with LocalTempDir { "JdbcUtils.alterTable()" should "work" in { val options = new JDBCOptions(url, "table_002", Map(JDBCOptions.JDBC_DRIVER_CLASS -> driver)) val conn = JdbcUtils.createConnection(options) - val table = TableDefinition( + val table = catalog.TableDefinition( TableIdentifier("table_001"), Seq( Field("str_field", VarcharType(20)), @@ -91,7 +93,9 @@ class JdbcUtilsTest extends AnyFlatSpec with Matchers with LocalTempDir { Field("BOOL_FIELD", BooleanType) )) - val migrations = TableChange.migrate(curSchema, newSchema, MigrationPolicy.STRICT) + val curTable = TableDefinition(TableIdentifier(""), curSchema.fields) + val newTable = TableDefinition(TableIdentifier(""), newSchema.fields) + val migrations = TableChange.migrate(curTable, newTable, MigrationPolicy.STRICT) JdbcUtils.alterTable(conn, table.identifier, migrations, options) JdbcUtils.getSchema(conn, table.identifier, options) should be (newSchema) diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala index ab6af4da7..26b28acac 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala @@ -24,6 +24,7 @@ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.functions.col import org.apache.spark.sql.streaming.StreamingQuery @@ -37,6 +38,7 @@ import com.dimajix.common.Trilean import com.dimajix.common.Yes import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MergeClause @@ -223,7 +225,9 @@ case class DeltaFileRelation( val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) - !TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) + val sourceTable = TableDefinition(TableIdentifier(""), sourceSchema.fields) + val targetTable = TableDefinition(TableIdentifier(""), targetSchema.fields) + !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } else { true diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala index f80c834cf..f6cfa135d 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala @@ -22,6 +22,7 @@ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand import org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand @@ -39,6 +40,7 @@ import com.dimajix.flowman.catalog.TableChange.DropColumn import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.execution.MigrationFailedException @@ -124,30 +126,32 @@ abstract class DeltaRelation(options: Map[String,String], mergeKey: Seq[String]) val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) + val sourceTable = TableDefinition(TableIdentifier(""), sourceSchema.fields) + val targetTable = TableDefinition(TableIdentifier(""), targetSchema.fields) - val requiresMigration = TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) + val requiresMigration = TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) if (requiresMigration) { - doMigration(execution, table, sourceSchema, targetSchema, migrationPolicy, migrationStrategy) + doMigration(execution, table, sourceTable, targetTable, migrationPolicy, migrationStrategy) provides.foreach(execution.refreshResource) } } - private def doMigration(execution: Execution, table:DeltaTableV2, currentSchema:com.dimajix.flowman.types.StructType, targetSchema:com.dimajix.flowman.types.StructType, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { + private def doMigration(execution: Execution, table:DeltaTableV2, currentTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for Delta relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.warn(s"Migration required for Delta relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate Delta relation '$identifier', since migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate Delta relation '$identifier', since migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER => - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.exists(m => !supported(m))) { - logger.error(s"Cannot migrate Delta relation '$identifier', since that would require unsupported changes.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate Delta relation '$identifier', since that would require unsupported changes.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) } alter(migrations) case MigrationStrategy.ALTER_REPLACE => - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.forall(m => supported(m))) { alter(migrations) } @@ -159,7 +163,7 @@ abstract class DeltaRelation(options: Map[String,String], mergeKey: Seq[String]) } def alter(migrations:Seq[TableChange]) : Unit = { - logger.info(s"Migrating Delta relation '$identifier'. New schema:\n${targetSchema.treeString}") + logger.info(s"Migrating Delta relation '$identifier'. New schema:\n${targetTable.schema.treeString}") try { val spark = execution.spark diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala index 47bfa6f7d..ac0606ecf 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala @@ -36,6 +36,7 @@ import com.dimajix.common.Trilean import com.dimajix.common.Yes import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MergeClause @@ -229,7 +230,9 @@ case class DeltaTableRelation( val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) - !TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) + val sourceTable = TableDefinition(tableIdentifier, sourceSchema.fields) + val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) + !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } else { true diff --git a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala index 5edf2db12..c8c7aa985 100644 --- a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala +++ b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala @@ -24,11 +24,12 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions +import com.dimajix.flowman.catalog +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.jdbc.JdbcUtils import com.dimajix.flowman.jdbc.SqlDialects -import com.dimajix.flowman.jdbc.TableDefinition import com.dimajix.flowman.model.Connection import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Reference @@ -108,7 +109,7 @@ case class SqlServerRelation( } // Create temp table with specified schema, but without any primary key or indices - val table = TableDefinition( + val table = catalog.TableDefinition( tempTableIdentifier, schema.fields ) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala index c955d65c2..02cb6ce8e 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala @@ -49,6 +49,7 @@ import com.dimajix.flowman.catalog.TableChange.DropColumn import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MigrationFailedException @@ -352,7 +353,7 @@ case class HiveTableRelation( false } else { - val sourceSchema = com.dimajix.flowman.types.StructType.of(table.dataSchema) + val sourceTable = TableDefinition.ofTable(table) val targetSchema = { val dataSchema = com.dimajix.flowman.types.StructType(schema.get.fields) if (hiveVarcharSupported) @@ -360,8 +361,9 @@ case class HiveTableRelation( else SchemaUtils.replaceCharVarchar(dataSchema) } + val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) - !TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) + !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } } else { @@ -533,7 +535,7 @@ case class HiveTableRelation( } } else { - val sourceSchema = com.dimajix.flowman.types.StructType.of(table.dataSchema) + val sourceTable = TableDefinition.ofTable(table) val targetSchema = { val dataSchema = com.dimajix.flowman.types.StructType(schema.get.fields) if (hiveVarcharSupported) @@ -541,32 +543,33 @@ case class HiveTableRelation( else SchemaUtils.replaceCharVarchar(dataSchema) } + val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) - val requiresMigration = TableChange.requiresMigration(sourceSchema, targetSchema, migrationPolicy) + val requiresMigration = TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) if (requiresMigration) { - doMigration(execution, sourceSchema, targetSchema, migrationPolicy, migrationStrategy) + doMigration(execution, sourceTable, targetTable, migrationPolicy, migrationStrategy) provides.foreach(execution.refreshResource) } } } } - private def doMigration(execution: Execution, currentSchema:com.dimajix.flowman.types.StructType, targetSchema:com.dimajix.flowman.types.StructType, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { + private def doMigration(execution: Execution, currentTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveTable relation '$identifier' of Hive table $tableIdentifier, but migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.warn(s"Migration required for HiveTable relation '$identifier' of Hive table $tableIdentifier, but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER => - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.exists(m => !supported(m))) { - logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since that would require unsupported changes.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since that would require unsupported changes.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) } alter(migrations) case MigrationStrategy.ALTER_REPLACE => - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.forall(m => supported(m))) { alter(migrations) } @@ -578,7 +581,7 @@ case class HiveTableRelation( } def alter(migrations:Seq[TableChange]) : Unit = { - logger.info(s"Migrating HiveTable relation '$identifier', this will alter the Hive table $tableIdentifier. New schema:\n${targetSchema.treeString}") + logger.info(s"Migrating HiveTable relation '$identifier', this will alter the Hive table $tableIdentifier. New schema:\n${targetTable.schema.treeString}") if (migrations.isEmpty) { logger.warn("Empty list of migrations - nothing to do") } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 97e707d42..882ea191b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -40,7 +40,9 @@ import org.slf4j.LoggerFactory import com.dimajix.common.SetIgnoreCase import com.dimajix.common.Trilean +import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableChange +import com.dimajix.flowman.catalog.TableDefinition import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.Execution @@ -55,7 +57,6 @@ import com.dimajix.flowman.execution.UpdateClause import com.dimajix.flowman.jdbc.JdbcUtils import com.dimajix.flowman.jdbc.SqlDialect import com.dimajix.flowman.jdbc.SqlDialects -import com.dimajix.flowman.jdbc.TableDefinition import com.dimajix.flowman.model.BaseRelation import com.dimajix.flowman.model.Connection import com.dimajix.flowman.model.PartitionField @@ -369,9 +370,10 @@ class JdbcRelationBase( withConnection { (con, options) => if (JdbcUtils.tableExists(con, tableIdentifier, options)) { if (schema.nonEmpty) { - val targetSchema = fullSchema.get - val currentSchema = JdbcUtils.getSchema(con, tableIdentifier, options) - !TableChange.requiresMigration(currentSchema, targetSchema, migrationPolicy) + val currentTable = JdbcUtils.getTable(con, tableIdentifier, options) + val targetTable = TableDefinition(tableIdentifier, fullSchema.get.fields) + + !TableChange.requiresMigration(currentTable, targetTable, migrationPolicy) } else { true @@ -431,7 +433,7 @@ class JdbcRelationBase( throw new UnspecifiedSchemaException(identifier) } val schema = this.schema.get - val table = TableDefinition( + val table = catalog.TableDefinition( tableIdentifier, schema.fields ++ partitions.map(_.field), schema.description, @@ -467,10 +469,11 @@ class JdbcRelationBase( if (schema.isDefined) { withConnection { (con, options) => if (JdbcUtils.tableExists(con, tableIdentifier, options)) { - val targetSchema = fullSchema.get - val currentSchema = JdbcUtils.getSchema(con, tableIdentifier, options) - if (TableChange.requiresMigration(currentSchema, targetSchema, migrationPolicy)) { - doMigration(currentSchema, targetSchema, migrationPolicy, migrationStrategy) + val currentTable = JdbcUtils.getTable(con, tableIdentifier, options) + val targetTable = TableDefinition(tableIdentifier, fullSchema.get.fields) + + if (TableChange.requiresMigration(currentTable, targetTable, migrationPolicy)) { + doMigration(currentTable, targetTable, migrationPolicy, migrationStrategy) provides.foreach(execution.refreshResource) } } @@ -478,25 +481,25 @@ class JdbcRelationBase( } } - private def doMigration(currentSchema:FlowmanStructType, targetSchema:FlowmanStructType, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { + private def doMigration(currentTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { withConnection { (con, options) => migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.warn(s"Migration required for relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate relation '$identifier', but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER => val dialect = SqlDialects.get(options.url) - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.exists(m => !dialect.supportsChange(tableIdentifier, m))) { - logger.error(s"Cannot migrate relation JDBC relation '$identifier' of table $tableIdentifier, since that would require unsupported changes.\nCurrent schema:\n${currentSchema.treeString}New schema:\n${targetSchema.treeString}") + logger.error(s"Cannot migrate relation JDBC relation '$identifier' of table $tableIdentifier, since that would require unsupported changes.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) } alter(migrations, con, options) case MigrationStrategy.ALTER_REPLACE => val dialect = SqlDialects.get(options.url) - val migrations = TableChange.migrate(currentSchema, targetSchema, migrationPolicy) + val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.forall(m => dialect.supportsChange(tableIdentifier, m))) { try { alter(migrations, con, options) @@ -516,7 +519,7 @@ class JdbcRelationBase( } def alter(migrations:Seq[TableChange], con:java.sql.Connection, options:JDBCOptions) : Unit = { - logger.info(s"Migrating JDBC relation '$identifier', this will alter JDBC table $tableIdentifier. New schema:\n${targetSchema.treeString}") + logger.info(s"Migrating JDBC relation '$identifier', this will alter JDBC table $tableIdentifier. New schema:\n${targetTable.schema.treeString}") if (migrations.isEmpty) logger.warn("Empty list of migrations - nothing to do") @@ -530,7 +533,7 @@ class JdbcRelationBase( def recreate(con:java.sql.Connection, options:JDBCOptions) : Unit = { try { - logger.info(s"Migrating JDBC relation '$identifier', this will recreate JDBC table $tableIdentifier. New schema:\n${targetSchema.treeString}") + logger.info(s"Migrating JDBC relation '$identifier', this will recreate JDBC table $tableIdentifier. New schema:\n${targetTable.schema.treeString}") JdbcUtils.dropTable(con, tableIdentifier, options) doCreate(con, options) } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala index ebb507999..3dc227aac 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From a13b5413892f992b62de9102f5d09cea5b4965fa Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Fri, 25 Feb 2022 15:19:00 +0100 Subject: [PATCH 84/95] Add new migrations for table indexes --- .../dimajix/flowman/catalog/TableChange.scala | 83 ++++++- .../flowman/catalog/TableDefinition.scala | 28 ++- .../dimajix/flowman/jdbc/BaseDialect.scala | 2 +- .../flowman/catalog/TableChangeTest.scala | 225 +++++++++++++++++- 4 files changed, 319 insertions(+), 19 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala index 4997b4c2e..9ffe793b6 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala @@ -53,10 +53,32 @@ object TableChange { * @return */ def migrate(sourceTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy) : Seq[TableChange] = { - val targetFields = targetTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)) + val normalizedSource = sourceTable.normalize() + val normalizedTarget = targetTable.normalize() + + // Check which Indexes need to be dropped + val dropIndexes = sourceTable.indexes.flatMap { src => + targetTable.indexes.find(_.name.toLowerCase(Locale.ROOT) == src.name.toLowerCase(Locale.ROOT)) match { + case None => + Some(DropIndex(src.name)) + case Some(tgt) => + if (src.normalize() != tgt.normalize()) + Some(DropIndex(src.name)) + else None + } + } + + // Check if primary key needs to be dropped + val dropPk = if(normalizedSource.primaryKey.nonEmpty && normalizedSource.primaryKey != normalizedTarget.primaryKey) + Some(DropPrimaryKey()) + else + None + + val targetFields = targetTable.columns.map(f => (f.name.toLowerCase(Locale.ROOT), f)) val targetFieldsByName = targetFields.toMap - val sourceFieldsByName = sourceTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap + val sourceFieldsByName = sourceTable.columns.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap + // Check which fields need to be dropped val dropFields = (sourceFieldsByName.keySet -- targetFieldsByName.keySet).toSeq.flatMap { fieldName => if (migrationPolicy == MigrationPolicy.STRICT) Some(DropColumn(sourceFieldsByName(fieldName).name)) @@ -64,6 +86,7 @@ object TableChange { None } + // Infer column changes val changeFields = targetFields.flatMap { case(tgtName,tgtField) => sourceFieldsByName.get(tgtName) match { case None => Seq(AddColumn(tgtField)) @@ -92,22 +115,64 @@ object TableChange { } } - dropFields ++ changeFields + // Create new PK + val createPk = if (normalizedTarget.primaryKey.nonEmpty && normalizedTarget.primaryKey != normalizedSource.primaryKey) + Some(AddPrimaryKey(targetTable.primaryKey)) + else + None + + // Create new indexes + val addIndexes = targetTable.indexes.flatMap { tgt => + sourceTable.indexes.find(_.name.toLowerCase(Locale.ROOT) == tgt.name.toLowerCase(Locale.ROOT)) match { + case None => + Some(AddIndex(tgt.name, tgt.columns)) + case Some(src) => + if (src.normalize() != tgt.normalize()) + Some(AddIndex(tgt.name, tgt.columns)) + else + None + } + } + + dropIndexes ++ dropPk ++ dropFields ++ changeFields ++ createPk ++ addIndexes } + /** + * Performs a check if a migration is required + * @param sourceTable + * @param targetTable + * @param migrationPolicy + * @return + */ def requiresMigration(sourceTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy) : Boolean = { - // Ensure that current real Hive schema is compatible with specified schema - migrationPolicy match { + val normalizedSource = sourceTable.normalize() + val normalizedTarget = targetTable.normalize() + + // Check if PK needs change + val pkChanges = normalizedSource.primaryKey != normalizedTarget.primaryKey + + // Check if indices need change + val dropIndexes = !normalizedSource.indexes.forall(src => + normalizedTarget.indexes.contains(src) + ) + val addIndexes = !normalizedTarget.indexes.forall(tgt => + normalizedSource.indexes.contains(tgt) + ) + + // Ensure that current real schema is compatible with specified schema + val columnChanges = migrationPolicy match { case MigrationPolicy.RELAXED => - val sourceFields = sourceTable.fields.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap - targetTable.fields.exists { tgt => + val sourceFields = sourceTable.columns.map(f => (f.name.toLowerCase(Locale.ROOT), f)).toMap + targetTable.columns.exists { tgt => !sourceFields.get(tgt.name.toLowerCase(Locale.ROOT)) .exists(src => SchemaUtils.isCompatible(tgt, src)) } case MigrationPolicy.STRICT => - val sourceFields = SchemaUtils.normalize(sourceTable.fields).sortBy(_.name) - val targetFields = SchemaUtils.normalize(targetTable.fields).sortBy(_.name) + val sourceFields = SchemaUtils.normalize(sourceTable.columns).sortBy(_.name) + val targetFields = SchemaUtils.normalize(targetTable.columns).sortBy(_.name) sourceFields != targetFields } + + pkChanges || dropIndexes || addIndexes || columnChanges } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala index 8c411cb6e..19850a475 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala @@ -16,6 +16,8 @@ package com.dimajix.flowman.catalog +import java.util.Locale + import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable @@ -29,18 +31,30 @@ object TableDefinition { TableDefinition(table.identifier, sourceSchema.fields) } } -case class TableDefinition( +final case class TableDefinition( identifier: TableIdentifier, - fields: Seq[Field], + columns: Seq[Field] = Seq.empty, comment: Option[String] = None, - primaryKey: Seq[String] = Seq(), - indexes: Seq[TableIndex] = Seq() + primaryKey: Seq[String] = Seq.empty, + indexes: Seq[TableIndex] = Seq.empty ) { - def schema : StructType = StructType(fields) + def schema : StructType = StructType(columns) + + def normalize() : TableDefinition = copy( + columns = columns.map(f => f.copy(name = f.name.toLowerCase(Locale.ROOT))), + primaryKey = primaryKey.map(_.toLowerCase(Locale.ROOT)).sorted, + indexes = indexes.map(_.normalize()) + ) + } -case class TableIndex( +final case class TableIndex( name: String, columns: Seq[String] -) +) { + def normalize() : TableIndex = copy( + name = name.toLowerCase(Locale.ROOT), + columns = columns.map(_.toLowerCase(Locale.ROOT)).sorted + ) +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala index d5d6f611c..45d67f20b 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala @@ -229,7 +229,7 @@ class BaseStatements(dialect: SqlDialect) extends SqlStatements { override def createTable(table: TableDefinition): String = { // Column definitions - val columns = table.fields.map { field => + val columns = table.columns.map { field => val name = dialect.quoteIdentifier(field.name) val typ = dialect.getJdbcType(field.ftype).databaseTypeDefinition val nullable = if (field.nullable) "" diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala index 0fe454118..8079868dd 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,11 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog.TableChange.AddColumn +import com.dimajix.flowman.catalog.TableChange.AddIndex +import com.dimajix.flowman.catalog.TableChange.AddPrimaryKey import com.dimajix.flowman.catalog.TableChange.DropColumn +import com.dimajix.flowman.catalog.TableChange.DropIndex +import com.dimajix.flowman.catalog.TableChange.DropPrimaryKey import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.execution.MigrationPolicy @@ -29,7 +33,6 @@ import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.LongType import com.dimajix.flowman.types.StringType -import com.dimajix.flowman.types.StructType import com.dimajix.flowman.types.VarcharType @@ -196,6 +199,77 @@ class TableChangeTest extends AnyFlatSpec with Matchers { ) should be (true) } + it should "handle changed primary key" in { + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + MigrationPolicy.RELAXED + ) should be (false) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType), Field("f2", StringType)), primaryKey=Seq("f2", "f1")), + MigrationPolicy.RELAXED + ) should be (false) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType)), primaryKey=Seq("f1")), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType), Field("f2", StringType)), primaryKey=Seq()), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(Field("f1", StringType), Field("f2", StringType)), primaryKey=Seq()), + TableDefinition(TableIdentifier(""), Seq(Field("F1", StringType), Field("f2", StringType)), primaryKey=Seq("f1", "f2")), + MigrationPolicy.RELAXED + ) should be (true) + } + + it should "handle changed index" in { + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + MigrationPolicy.RELAXED + ) should be (false) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("C1")))), + MigrationPolicy.RELAXED + ) should be (false) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("NAME", Seq("C1")))), + MigrationPolicy.RELAXED + ) should be (false) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq()), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("NAME", Seq("C1")))), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq()), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("other", Seq("C1")))), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("C1","c2")))), + MigrationPolicy.RELAXED + ) should be (true) + TableChange.requiresMigration( + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("c2","c1")))), + TableDefinition(TableIdentifier(""), Seq(), indexes=Seq(TableIndex("name", Seq("C1","c2")))), + MigrationPolicy.RELAXED + ) should be (false) + } + "TableChange.migrate" should "work in strict mode" in { val oldTable = TableDefinition(TableIdentifier(""), Seq( @@ -254,4 +328,151 @@ class TableChangeTest extends AnyFlatSpec with Matchers { UpdateColumnNullability("f6", true) )) } + + it should "do nothing on unchanged PK" in { + val oldTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq("f1", "f2") + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( + Field("F1", StringType), + Field("F2", LongType), + Field("F3", StringType) + ), + primaryKey = Seq("F2", "f1") + ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq()) + } + + it should "add PK" in { + val oldTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq() + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( + Field("F1", StringType), + Field("F2", LongType), + Field("F3", StringType) + ), + primaryKey = Seq("f1", "f2") + ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq( + AddPrimaryKey(Seq("f1", "f2")) + )) + } + + it should "drop PK" in { + val oldTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq("f1", "f2") + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq() + ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq( + DropPrimaryKey() + )) + } + + it should "drop/add PK" in { + val oldTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq("f1", "f2") + ) + val newTable = TableDefinition(TableIdentifier(""), + Seq( + Field("f1", StringType), + Field("f2", LongType), + Field("f3", StringType) + ), + primaryKey = Seq("f2") + ) + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq( + DropPrimaryKey(), + AddPrimaryKey(Seq("f2")) + )) + } + + it should "do nothing on an unchanged index" in { + val oldTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("name", Seq("col1", "col2"))) + ) + val newTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("NAME", Seq("col2", "COL1"))) + ) + + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq.empty) + } + + it should "add an index" in { + val oldTable = TableDefinition(TableIdentifier(""), + indexes = Seq() + ) + val newTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("NAME", Seq("col2", "COL1"))) + ) + + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq(AddIndex("NAME", Seq("col2", "COL1")))) + } + + it should "drop an index" in { + val oldTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("name", Seq("col1", "col2"))) + ) + val newTable = TableDefinition(TableIdentifier(""), + indexes = Seq() + ) + + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq(DropIndex("name"))) + } + + it should "drop/add an index" in { + val oldTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("name", Seq("col1", "col3"))) + ) + val newTable = TableDefinition(TableIdentifier(""), + indexes = Seq(TableIndex("NAME", Seq("col2", "COL1"))) + ) + + val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) + + changes should be (Seq(DropIndex("name"), AddIndex("NAME", Seq("col2", "COL1")))) + } } From b189414126287e93c071b8f24d5311c47092fcf0 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sun, 27 Feb 2022 11:48:04 +0100 Subject: [PATCH 85/95] Improve support for indexes in JDBC relation --- .../dimajix/flowman/catalog/TableChange.scala | 10 +- .../flowman/catalog/TableDefinition.scala | 3 +- .../dimajix/flowman/jdbc/BaseDialect.scala | 17 ++ .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 123 +++++++++++++- .../flowman/jdbc/MsSqlServerDialect.scala | 4 + .../dimajix/flowman/jdbc/MySQLDialect.scala | 4 + .../dimajix/flowman/jdbc/SqlStatements.scala | 7 + .../flowman/model/ResourceIdentifier.scala | 11 +- .../flowman/catalog/TableChangeTest.scala | 12 +- .../flowman/execution/RunnerJobTest.scala | 2 +- .../dimajix/flowman/jdbc/DerbyJdbcTest.scala | 40 +++-- .../com/dimajix/flowman/jdbc/H2JdbcTest.scala | 28 +++- .../flowman/dsl/relation/HiveTable.scala | 6 +- .../flowman/dsl/relation/HiveUnionTable.scala | 9 +- .../flowman/dsl/relation/HiveView.scala | 7 +- .../spec/relation/DeltaTableRelation.scala | 90 +++++----- .../spec/target/DeltaVacuumTarget.scala | 6 +- .../relation/DeltaTableRelationTest.scala | 60 +++---- .../spec/target/DeltaVacuumTargetTest.scala | 10 +- .../spec/relation/SqlServerRelation.scala | 31 ++-- .../flowman/spec/relation/HiveRelation.scala | 12 +- .../spec/relation/HiveTableRelation.scala | 100 ++++++------ .../relation/HiveUnionTableRelation.scala | 49 +++--- .../spec/relation/HiveViewRelation.scala | 57 ++++--- .../spec/relation/IndexedRelationSpec.scala | 42 +++++ .../flowman/spec/relation/JdbcRelation.scala | 115 +++++++------ .../flowman/spec/mapping/ReadHiveTest.scala | 7 +- .../spec/relation/HiveTableRelationTest.scala | 50 ++---- .../relation/HiveUnionTableRelationTest.scala | 10 +- .../spec/relation/HiveViewRelationTest.scala | 31 ++-- .../spec/relation/JdbcRelationTest.scala | 154 +++++++++++++++++- 31 files changed, 723 insertions(+), 384 deletions(-) create mode 100644 flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/IndexedRelationSpec.scala diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala index 9ffe793b6..e7fc65f79 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableChange.scala @@ -39,9 +39,9 @@ object TableChange { case class UpdateColumnType(column:String, dataType:FieldType) extends ColumnChange case class UpdateColumnComment(column:String, comment:Option[String]) extends ColumnChange - case class AddPrimaryKey(columns:Seq[String]) extends IndexChange + case class CreatePrimaryKey(columns:Seq[String]) extends IndexChange case class DropPrimaryKey() extends IndexChange - case class AddIndex(name:String, columns:Seq[String]) extends IndexChange + case class CreateIndex(name:String, columns:Seq[String], unique:Boolean) extends IndexChange case class DropIndex(name:String) extends IndexChange /** @@ -117,7 +117,7 @@ object TableChange { // Create new PK val createPk = if (normalizedTarget.primaryKey.nonEmpty && normalizedTarget.primaryKey != normalizedSource.primaryKey) - Some(AddPrimaryKey(targetTable.primaryKey)) + Some(CreatePrimaryKey(targetTable.primaryKey)) else None @@ -125,10 +125,10 @@ object TableChange { val addIndexes = targetTable.indexes.flatMap { tgt => sourceTable.indexes.find(_.name.toLowerCase(Locale.ROOT) == tgt.name.toLowerCase(Locale.ROOT)) match { case None => - Some(AddIndex(tgt.name, tgt.columns)) + Some(CreateIndex(tgt.name, tgt.columns, tgt.unique)) case Some(src) => if (src.normalize() != tgt.normalize()) - Some(AddIndex(tgt.name, tgt.columns)) + Some(CreateIndex(tgt.name, tgt.columns, tgt.unique)) else None } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala index 19850a475..0220d7686 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala @@ -51,7 +51,8 @@ final case class TableDefinition( final case class TableIndex( name: String, - columns: Seq[String] + columns: Seq[String], + unique:Boolean = false ) { def normalize() : TableIndex = copy( name = name.toLowerCase(Locale.ROOT), diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala index 45d67f20b..69404c4f5 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala @@ -39,6 +39,7 @@ import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.MergeClause @@ -337,6 +338,22 @@ class BaseStatements(dialect: SqlDialect) extends SqlStatements { newExpr.sql } + + override def dropPrimaryKey(table: TableIdentifier): String = ??? + + override def addPrimaryKey(table: TableIdentifier, columns: Seq[String]): String = ??? + + override def dropIndex(table: TableIdentifier, indexName: String): String = { + s"DROP INDEX ${dialect.quoteIdentifier(indexName)}" + } + + override def createIndex(table: TableIdentifier, index: TableIndex): String = { + // Column definitions + val columns = index.columns.map(dialect.quoteIdentifier) + val unique = if (index.unique) "UNIQUE" else "" + + s"CREATE $unique INDEX ${dialect.quoteIdentifier(index.name)} ON ${dialect.quote(table)} (${columns.mkString(",")})" + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index e522897d7..05f0a0781 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -17,6 +17,7 @@ package com.dimajix.flowman.jdbc import java.sql.Connection +import java.sql.DatabaseMetaData import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.ResultSetMetaData @@ -45,11 +46,16 @@ import slick.jdbc.SQLiteProfile import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableChange.AddColumn +import com.dimajix.flowman.catalog.TableChange.CreateIndex +import com.dimajix.flowman.catalog.TableChange.CreatePrimaryKey import com.dimajix.flowman.catalog.TableChange.DropColumn +import com.dimajix.flowman.catalog.TableChange.DropIndex +import com.dimajix.flowman.catalog.TableChange.DropPrimaryKey import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.StructType @@ -165,8 +171,65 @@ object JdbcUtils { * @return */ def getTable(conn: Connection, table:TableIdentifier, options: JDBCOptions) : TableDefinition = { - val currentSchema = JdbcUtils.getSchema(conn, table, options) - TableDefinition(table, currentSchema.fields) + val meta = conn.getMetaData + val realTable = resolveTable(meta, table) + + val currentSchema = getSchema(conn, table, options) + val pk = getPrimaryKey(meta, realTable) + val idxs = getIndexes(meta, realTable) + // Remove primary key + .filter { idx => + idx.normalize().columns != pk.map(_.toLowerCase(Locale.ROOT)).sorted + } + + TableDefinition(table, currentSchema.fields, primaryKey=pk, indexes=idxs) + } + + private def getPrimaryKey(meta: DatabaseMetaData, table:TableIdentifier) : Seq[String] = { + val pkrs = meta.getPrimaryKeys(null, table.database.orNull, table.table) + val pk = mutable.ListBuffer[String]() + while(pkrs.next()) { + val col = pkrs.getString(4) + pk.append(col) + } + pkrs.close() + pk + } + + private def getIndexes(meta: DatabaseMetaData, table:TableIdentifier) : Seq[TableIndex] = { + val idxrs = meta.getIndexInfo(null, table.database.orNull, table.table, false, true) + val idxcols = mutable.ListBuffer[(String, String, Boolean)]() + while(idxrs.next()) { + val unique = !idxrs.getBoolean(4) + val name = idxrs.getString(6) + val col = idxrs.getString(9) + idxcols.append((name, col, unique)) + } + idxrs.close() + + idxcols.groupBy(_._1).map { case(name,cols) => + TableIndex(name, cols.map(_._2), cols.foldLeft(false)(_ || _._3)) + }.toSeq + } + + /** + * Resolves the table name, even if upper/lower case does not match + * @param conn + * @param table + * @return + */ + private def resolveTable(meta: DatabaseMetaData, table:TableIdentifier) : TableIdentifier = { + val tblrs = meta.getTables(null, table.database.orNull, null, Array("TABLE")) + var name = table.table + val db = table.database + while(tblrs.next()) { + val thisName = tblrs.getString(3) + if (name.toLowerCase(Locale.ROOT) == thisName.toLowerCase(Locale.ROOT)) + name = thisName + } + tblrs.close() + + TableIdentifier(name, db) } /** @@ -205,7 +268,13 @@ object JdbcUtils { val dialect = SqlDialects.get(options.url) withStatement(conn, dialect.statement.schema(table), options) { statement => - getJdbcSchemaImpl(statement.executeQuery()) + val rs = statement.executeQuery() + try { + getJdbcSchemaImpl(rs) + } + finally { + rs.close() + } } } @@ -252,9 +321,11 @@ object JdbcUtils { */ def createTable(conn:Connection, table:TableDefinition, options: JDBCOptions) : Unit = { val dialect = SqlDialects.get(options.url) - val sql = dialect.statement.createTable(table) + val tableSql = dialect.statement.createTable(table) + val indexSql = table.indexes.map(idx => dialect.statement.createIndex(table.identifier, idx)) withStatement(conn, options) { statement => - statement.executeUpdate(sql) + statement.executeUpdate(tableSql) + indexSql.foreach(statement.executeUpdate) } } @@ -267,6 +338,7 @@ object JdbcUtils { def dropTable(conn:Connection, table:TableIdentifier, options: JDBCOptions) : Unit = { val dialect = SqlDialects.get(options.url) withStatement(conn, options) { statement => + // TODO: Drop indices(?) statement.executeUpdate(s"DROP TABLE ${dialect.quote(table)}") } } @@ -285,6 +357,35 @@ object JdbcUtils { } } + /** + * Adds an index to an existing table + * @param conn + * @param table + * @param index + * @param options + */ + def createIndex(conn:Connection, table:TableIdentifier, index:TableIndex, options: JDBCOptions) : Unit = { + val dialect = SqlDialects.get(options.url) + val indexSql = dialect.statement.createIndex(table, index) + withStatement(conn, options) { statement => + statement.executeUpdate(indexSql) + } + } + + /** + * Drops an index from an existing table + * @param conn + * @param indexName + * @param options + */ + def dropIndex(conn:Connection, table:TableIdentifier, indexName:String, options: JDBCOptions) : Unit = { + val dialect = SqlDialects.get(options.url) + val indexSql = dialect.statement.dropIndex(table, indexName) + withStatement(conn, options) { statement => + statement.executeUpdate(indexSql) + } + } + /** * Applies a list of [[TableChange]] to an existing table. Will throw an exception if one of the operations * is not supported or if the table does not exist. @@ -325,6 +426,18 @@ object JdbcUtils { case u:UpdateColumnComment => logger.info(s"Updating comment of column ${u.column} in JDBC table $table") None + case idx:CreateIndex => + logger.info(s"Adding index ${idx.name} to JDBC table $table on columns ${idx.columns.mkString(",")}") + Some(statements.createIndex(table, TableIndex(idx.name, idx.columns, idx.unique))) + case idx:DropIndex => + logger.info(s"Dropping index ${idx.name} from JDBC table $table") + Some(statements.dropIndex(table, idx.name)) + case pk:CreatePrimaryKey => + logger.info(s"Creating primary key for JDBC table $table on columns ${pk.columns.mkString(",")}") + Some(statements.addPrimaryKey(table, pk.columns)) + case pk:DropPrimaryKey => + logger.info(s"Removing primary key from JDBC table $table}") + Some(statements.dropPrimaryKey(table)) case chg:TableChange => throw new SQLException(s"Unsupported table change $chg for JDBC table $table") } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala index 9b63d07cd..36ade3d23 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala @@ -105,4 +105,8 @@ class MsSqlServerStatements(dialect: BaseDialect) extends BaseStatements(dialect val nullable = if (isNullable) "NULL" else "NOT NULL" s"ALTER TABLE ${dialect.quote(table)} ALTER COLUMN ${dialect.quoteIdentifier(columnName)} $dataType $nullable" } + + override def dropIndex(table: TableIdentifier, indexName: String): String = { + s"DROP INDEX ${dialect.quote(table)}.${dialect.quoteIdentifier(indexName)}" + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala index 0e0cf414c..495c8ed97 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala @@ -81,4 +81,8 @@ class MySQLStatements(dialect: BaseDialect) extends BaseStatements(dialect) { override def updateColumnNullability(table: TableIdentifier, columnName: String, dataType:String, isNullable: Boolean): String = { throw new SQLFeatureNotSupportedException(s"UpdateColumnNullability is not supported") } + + override def dropIndex(table: TableIdentifier, indexName: String): String = { + s"DROP INDEX ${dialect.quoteIdentifier(indexName)} ON ${dialect.quote(table)}" + } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala index 92b31eb0b..567cb1285 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.MergeClause @@ -58,5 +59,11 @@ abstract class SqlStatements { def updateColumnType(table: TableIdentifier, columnName: String, newDataType: String): String def updateColumnNullability(table: TableIdentifier, columnName: String, dataType: String, isNullable: Boolean): String + def dropPrimaryKey(table: TableIdentifier) : String + def addPrimaryKey(table: TableIdentifier, columns:Seq[String]) : String + + def dropIndex(table: TableIdentifier, indexName: String) : String + def createIndex(table: TableIdentifier, index:TableIndex) : String + def merge(table: TableIdentifier, targetAlias:String, targetSchema:Option[StructType], sourceAlias:String, sourceSchema:StructType, condition:Column, clauses:Seq[MergeClause]) : String } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala index 45357eff7..6696f9650 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala @@ -23,6 +23,7 @@ import java.util.regex.Pattern import scala.annotation.tailrec import org.apache.hadoop.fs.Path +import org.apache.spark.sql.catalyst.TableIdentifier import com.dimajix.flowman.hadoop.GlobPattern @@ -36,26 +37,34 @@ object ResourceIdentifier { GlobbingResourceIdentifier("local", new Path(file.toURI.getPath).toString) def ofHiveDatabase(database:String): RegexResourceIdentifier = RegexResourceIdentifier("hiveDatabase", database) + def ofHiveTable(table:TableIdentifier): RegexResourceIdentifier = + ofHiveTable(table.table, table.database) def ofHiveTable(table:String): RegexResourceIdentifier = RegexResourceIdentifier("hiveTable", table) def ofHiveTable(table:String, database:Option[String]): RegexResourceIdentifier = RegexResourceIdentifier("hiveTable", fqTable(table, database)) + def ofHivePartition(table:TableIdentifier, partition:Map[String,Any]): RegexResourceIdentifier = + ofHivePartition(table.table, table.database, partition) def ofHivePartition(table:String, partition:Map[String,Any]): RegexResourceIdentifier = RegexResourceIdentifier("hiveTablePartition", table, partition.map { case(k,v) => k -> v.toString }) def ofHivePartition(table:String, database:Option[String], partition:Map[String,Any]): RegexResourceIdentifier = RegexResourceIdentifier("hiveTablePartition", fqTable(table, database), partition.map { case(k,v) => k -> v.toString }) def ofJdbcDatabase(database:String): RegexResourceIdentifier = RegexResourceIdentifier("jdbcDatabase", database) + def ofJdbcTable(table:TableIdentifier): RegexResourceIdentifier = + ofJdbcTable(table.table, table.database) def ofJdbcTable(table:String, database:Option[String]): RegexResourceIdentifier = RegexResourceIdentifier("jdbcTable", fqTable(table, database)) def ofJdbcQuery(query:String): SimpleResourceIdentifier = SimpleResourceIdentifier("jdbcQuery", "") + def ofJdbcTablePartition(table:TableIdentifier, partition:Map[String,Any]): RegexResourceIdentifier = + ofJdbcTablePartition(table.table, table.database, partition) def ofJdbcTablePartition(table:String, database:Option[String], partition:Map[String,Any]): RegexResourceIdentifier = RegexResourceIdentifier("jdbcTablePartition", fqTable(table, database), partition.map { case(k,v) => k -> v.toString }) def ofURL(url:URL): RegexResourceIdentifier = RegexResourceIdentifier("url", url.toString) - private def fqTable(table:String, database:Option[String]) : String = database.map(_ + ".").getOrElse("") + table + private def fqTable(table:String, database:Option[String]) : String = database.filter(_.nonEmpty).map(_ + ".").getOrElse("") + table } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala index 8079868dd..294040b72 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala @@ -21,8 +21,8 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog.TableChange.AddColumn -import com.dimajix.flowman.catalog.TableChange.AddIndex -import com.dimajix.flowman.catalog.TableChange.AddPrimaryKey +import com.dimajix.flowman.catalog.TableChange.CreateIndex +import com.dimajix.flowman.catalog.TableChange.CreatePrimaryKey import com.dimajix.flowman.catalog.TableChange.DropColumn import com.dimajix.flowman.catalog.TableChange.DropIndex import com.dimajix.flowman.catalog.TableChange.DropPrimaryKey @@ -371,7 +371,7 @@ class TableChangeTest extends AnyFlatSpec with Matchers { val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) changes should be (Seq( - AddPrimaryKey(Seq("f1", "f2")) + CreatePrimaryKey(Seq("f1", "f2")) )) } @@ -420,7 +420,7 @@ class TableChangeTest extends AnyFlatSpec with Matchers { changes should be (Seq( DropPrimaryKey(), - AddPrimaryKey(Seq("f2")) + CreatePrimaryKey(Seq("f2")) )) } @@ -447,7 +447,7 @@ class TableChangeTest extends AnyFlatSpec with Matchers { val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) - changes should be (Seq(AddIndex("NAME", Seq("col2", "COL1")))) + changes should be (Seq(CreateIndex("NAME", Seq("col2", "COL1"), false))) } it should "drop an index" in { @@ -473,6 +473,6 @@ class TableChangeTest extends AnyFlatSpec with Matchers { val changes = TableChange.migrate(oldTable, newTable, MigrationPolicy.RELAXED) - changes should be (Seq(DropIndex("name"), AddIndex("NAME", Seq("col2", "COL1")))) + changes should be (Seq(DropIndex("name"), CreateIndex("NAME", Seq("col2", "COL1"), false))) } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerJobTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerJobTest.scala index 307f0d6cd..5683858ec 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerJobTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/execution/RunnerJobTest.scala @@ -403,7 +403,7 @@ class RunnerJobTest extends AnyFlatSpec with MockFactory with Matchers with Loca name = "default", targets = Map( "t0" -> genTarget("t0", true, Yes, produces=Set(ResourceIdentifier.ofHivePartition("some_table", Map("p1" -> "123")))), - "t1" -> genTarget("t1", true, No, requires=Set(ResourceIdentifier.ofHivePartition("some_table", Map()))), + "t1" -> genTarget("t1", true, No, requires=Set(ResourceIdentifier.ofHivePartition("some_table", Map.empty[String,Any]))), "t2" -> genTarget("t2", false, No) ) ) diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala index c359dcacc..ab1c454f0 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,11 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.StringType +import com.dimajix.flowman.types.VarcharType import com.dimajix.spark.testing.LocalTempDir @@ -44,21 +46,39 @@ class DerbyJdbcTest extends AnyFlatSpec with Matchers with LocalTempDir { "A Derby Table" should "be creatable" in { val options = new JDBCOptions(url, "table_001", Map(JDBCOptions.JDBC_DRIVER_CLASS -> driver)) val conn = JdbcUtils.createConnection(options) - val table = TableDefinition( + val table1 = TableDefinition( TableIdentifier("table_001"), Seq( Field("Id", IntegerType, nullable=false), - Field("str_field", StringType), + Field("str_field", VarcharType(32)), Field("int_field", IntegerType) ), - None, - Seq("Id") + primaryKey = Seq("Id"), + indexes = Seq( + TableIndex("table_001_idx1", Seq("str_field", "int_field")) + ) ) - JdbcUtils.tableExists(conn, table.identifier, options) should be (false) - JdbcUtils.createTable(conn, table, options) - JdbcUtils.tableExists(conn, table.identifier, options) should be (true) - JdbcUtils.dropTable(conn, table.identifier, options) - JdbcUtils.tableExists(conn, table.identifier, options) should be (false) + + //==== CREATE ================================================================================================ + JdbcUtils.tableExists(conn, table1.identifier, options) should be (false) + JdbcUtils.createTable(conn, table1, options) + JdbcUtils.tableExists(conn, table1.identifier, options) should be (true) + + JdbcUtils.getTable(conn, table1.identifier, options) should be (table1) + + //==== DROP INDEX ============================================================================================ + val table2 = table1.copy(indexes = Seq.empty) + JdbcUtils.dropIndex(conn, table1.identifier, "table_001_idx1", options) + JdbcUtils.getTable(conn, table1.identifier, options) should be (table2) + + //==== CREATE INDEX ============================================================================================ + val table3 = table2.copy(indexes = Seq(TableIndex("table_001_idx1", Seq("str_field", "Id")))) + JdbcUtils.createIndex(conn, table3.identifier, table3.indexes.head, options) + JdbcUtils.getTable(conn, table3.identifier, options) should be (table3) + + //==== DROP ================================================================================================== + JdbcUtils.dropTable(conn, table1.identifier, options) + JdbcUtils.tableExists(conn, table1.identifier, options) should be (false) conn.close() } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala index 8dad118b0..99a3bc10e 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,12 +32,14 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.UpdateClause import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.StringType +import com.dimajix.flowman.types.VarcharType import com.dimajix.spark.sql.DataFrameBuilder import com.dimajix.spark.testing.LocalSparkSession @@ -60,15 +62,33 @@ class H2JdbcTest extends AnyFlatSpec with Matchers with LocalSparkSession { TableIdentifier("table_001"), Seq( Field("Id", IntegerType, nullable=false), - Field("str_field", StringType), + Field("str_field", VarcharType(32)), Field("int_field", IntegerType) ), - None, - Seq("iD") + primaryKey = Seq("iD"), + indexes = Seq( + TableIndex("table_001_idx1", Seq("str_field", "int_field")) + ) ) + + //==== CREATE ================================================================================================ JdbcUtils.tableExists(conn, table.identifier, options) should be (false) JdbcUtils.createTable(conn, table, options) JdbcUtils.tableExists(conn, table.identifier, options) should be (true) + + JdbcUtils.getTable(conn, table.identifier, options).normalize() should be (table.normalize()) + + //==== DROP INDEX ============================================================================================ + val table2 = table.copy(indexes = Seq.empty) + JdbcUtils.dropIndex(conn, table.identifier, "table_001_idx1", options) + JdbcUtils.getTable(conn, table.identifier, options).normalize() should be (table2.normalize()) + + //==== CREATE INDEX ============================================================================================ + val table3 = table2.copy(indexes = Seq(TableIndex("table_001_idx1", Seq("str_field", "Id")))) + JdbcUtils.createIndex(conn, table3.identifier, table3.indexes.head, options) + JdbcUtils.getTable(conn, table3.identifier, options).normalize() should be (table3.normalize()) + + //==== DROP ================================================================================================== JdbcUtils.dropTable(conn, table.identifier, options) JdbcUtils.tableExists(conn, table.identifier, options) should be (false) conn.close() diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala index 848214c49..d676f2a16 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.dimajix.flowman.dsl.relation import org.apache.hadoop.fs.Path +import org.apache.spark.sql.catalyst.TableIdentifier import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.PartitionField @@ -45,10 +46,9 @@ case class HiveTable( override def apply(props:Relation.Properties) : HiveTableRelation = { HiveTableRelation( props, - database = database, schema = schema.map(s => s.instantiate(props.context)), partitions = partitions, - table = table, + table = TableIdentifier(table, database), external = external, location = location, format = format, diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala index 515de80f4..2d708c648 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.dimajix.flowman.dsl.relation import org.apache.hadoop.fs.Path +import org.apache.spark.sql.catalyst.TableIdentifier import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.PartitionField @@ -49,11 +50,9 @@ case class HiveUnionTable( props, schema.map(_.instantiate(context)), partitions, - tableDatabase, - tablePrefix, + TableIdentifier(tablePrefix, tableDatabase), locationPrefix, - viewDatabase, - view, + TableIdentifier(view, viewDatabase), external, format, options, diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala index e5b0376cd..906f0c8f4 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2020 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package com.dimajix.flowman.dsl.relation +import org.apache.spark.sql.catalyst.TableIdentifier + import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.PartitionField @@ -33,8 +35,7 @@ case class HiveView( override def apply(props: Relation.Properties): HiveViewRelation = { HiveViewRelation( props, - database, - view, + TableIdentifier(view, database), partitions, sql, mapping diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala index ac0606ecf..06df5a95f 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala @@ -60,8 +60,7 @@ case class DeltaTableRelation( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, override val partitions: Seq[PartitionField] = Seq(), - database: String, - table: String, + table: TableIdentifier, location: Option[Path] = None, options: Map[String,String] = Map(), properties: Map[String, String] = Map(), @@ -69,17 +68,13 @@ case class DeltaTableRelation( ) extends DeltaRelation(options, mergeKey) { private val logger = LoggerFactory.getLogger(classOf[DeltaTableRelation]) - private lazy val tableIdentifier: TableIdentifier = { - TableIdentifier(table, Some(database)) - } - /** * Returns the list of all resources which will be created by this relation. * * @return */ override def provides: Set[ResourceIdentifier] = { - Set(ResourceIdentifier.ofHiveTable(table, Some(database))) + Set(ResourceIdentifier.ofHiveTable(table)) } /** @@ -88,7 +83,7 @@ case class DeltaTableRelation( * @return */ override def requires: Set[ResourceIdentifier] = { - Set(ResourceIdentifier.ofHiveDatabase(database)) + table.database.map(ResourceIdentifier.ofHiveDatabase).toSet } /** @@ -105,7 +100,7 @@ case class DeltaTableRelation( requireValidPartitionKeys(partition) val allPartitions = PartitionSchema(this.partitions).interpolate(partition) - allPartitions.map(p =>ResourceIdentifier.ofHivePartition(table, Some(database), p.toMap)).toSet + allPartitions.map(p =>ResourceIdentifier.ofHivePartition(table, p.toMap)).toSet } /** @@ -117,11 +112,11 @@ case class DeltaTableRelation( * @return */ override def read(execution: Execution, partitions: Map[String, FieldValue]): DataFrame = { - logger.info(s"Reading Delta relation '$identifier' from table $tableIdentifier using partition values $partitions") + logger.info(s"Reading Delta relation '$identifier' from table $table using partition values $partitions") val tableDf = execution.spark.read .options(options) - .table(tableIdentifier.quotedString) + .table(table.quotedString) val filteredDf = filterPartition(tableDf, partitions) applyInputSchema(execution, filteredDf) @@ -139,7 +134,7 @@ case class DeltaTableRelation( val partitionSpec = PartitionSchema(partitions).spec(partition) - logger.info(s"Writing Delta relation '$identifier' to table $tableIdentifier partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode'") + logger.info(s"Writing Delta relation '$identifier' to table $table partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode'") val extDf = applyOutputSchema(execution, addPartition(df, partition)) mode match { @@ -148,7 +143,7 @@ case class DeltaTableRelation( case _ => doWrite(extDf, partitionSpec, mode) } - execution.catalog.refreshTable(tableIdentifier) + execution.catalog.refreshTable(table) } private def doWrite(df: DataFrame, partitionSpec: PartitionSpec, mode: OutputMode) : Unit = { val writer = @@ -164,12 +159,12 @@ case class DeltaTableRelation( .format("delta") .options(options) .mode(mode.batchMode) - .insertInto(tableIdentifier.quotedString) + .insertInto(table.quotedString) } private def doUpdate(df: DataFrame, partitionSpec: PartitionSpec) : Unit = { val withinPartitionKeyColumns = if (mergeKey.nonEmpty) mergeKey else schema.map(_.primaryKey).getOrElse(Seq()) val keyColumns = SetIgnoreCase(partitions.map(_.name)) -- partitionSpec.keys ++ withinPartitionKeyColumns - val table = DeltaTable.forName(df.sparkSession, tableIdentifier.quotedString) + val table = DeltaTable.forName(df.sparkSession, this.table.quotedString) DeltaUtils.upsert(table, df, keyColumns, partitionSpec) } @@ -181,8 +176,8 @@ case class DeltaTableRelation( * @return */ override def readStream(execution: Execution): DataFrame = { - logger.info(s"Streaming from Delta table relation '$identifier' at $tableIdentifier") - val location = DeltaUtils.getLocation(execution, tableIdentifier) + logger.info(s"Streaming from Delta table relation '$identifier' at $table") + val location = DeltaUtils.getLocation(execution, table) readStreamFrom(execution, location) } @@ -194,8 +189,8 @@ case class DeltaTableRelation( * @return */ override def writeStream(execution: Execution, df: DataFrame, mode: OutputMode, trigger: Trigger, checkpointLocation: Path): StreamingQuery = { - logger.info(s"Streaming to Delta table relation '$identifier' $tableIdentifier") - val location = DeltaUtils.getLocation(execution, tableIdentifier) + logger.info(s"Streaming to Delta table relation '$identifier' $table") + val location = DeltaUtils.getLocation(execution, table) writeStreamTo(execution, df, location, mode, trigger, checkpointLocation) } @@ -208,7 +203,7 @@ case class DeltaTableRelation( * @return */ override def exists(execution: Execution): Trilean = { - execution.catalog.tableExists(tableIdentifier) + execution.catalog.tableExists(table) } @@ -221,8 +216,8 @@ case class DeltaTableRelation( */ override def conforms(execution: Execution, migrationPolicy: MigrationPolicy): Trilean = { val catalog = execution.catalog - if (catalog.tableExists(tableIdentifier)) { - val table = catalog.getTable(tableIdentifier) + if (catalog.tableExists(table)) { + val table = catalog.getTable(this.table) if (table.tableType == CatalogTableType.VIEW) { false } @@ -230,8 +225,8 @@ case class DeltaTableRelation( val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) - val sourceTable = TableDefinition(tableIdentifier, sourceSchema.fields) - val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) + val sourceTable = TableDefinition(this.table, sourceSchema.fields) + val targetTable = TableDefinition(this.table, targetSchema.fields) !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } else { @@ -259,15 +254,15 @@ case class DeltaTableRelation( requireValidPartitionKeys(partition) val catalog = execution.catalog - if (!catalog.tableExists(tableIdentifier)) { + if (!catalog.tableExists(table)) { false } else if (partitions.nonEmpty) { val partitionSpec = PartitionSchema(partitions).spec(partition) - DeltaUtils.isLoaded(execution, tableIdentifier, partitionSpec) + DeltaUtils.isLoaded(execution, table, partitionSpec) } else { - val location = catalog.getTableLocation(tableIdentifier) + val location = catalog.getTableLocation(table) DeltaUtils.isLoaded(execution, location) } } @@ -282,17 +277,17 @@ case class DeltaTableRelation( val tableExists = exists(execution) == Yes if (!ifNotExists || !tableExists) { val sparkSchema = HiveTableRelation.cleanupSchema(StructType(fields.map(_.catalogField))) - logger.info(s"Creating Delta table relation '$identifier' with table $tableIdentifier and schema\n${sparkSchema.treeString}") + logger.info(s"Creating Delta table relation '$identifier' with table $table and schema\n${sparkSchema.treeString}") if (schema.isEmpty) { throw new UnspecifiedSchemaException(identifier) } if (tableExists) - throw new TableAlreadyExistsException(database, table) + throw new TableAlreadyExistsException(table.database.getOrElse(""), table.table) DeltaUtils.createTable( execution, - Some(tableIdentifier), + Some(table), location, sparkSchema, partitions, @@ -314,15 +309,15 @@ case class DeltaTableRelation( requireValidPartitionKeys(partitions) if (partitions.nonEmpty) { - val deltaTable = DeltaTable.forName(execution.spark, tableIdentifier.quotedString) + val deltaTable = DeltaTable.forName(execution.spark, table.quotedString) PartitionSchema(this.partitions).interpolate(partitions).foreach { p => deltaTable.delete(p.predicate) } deltaTable.vacuum() } else { - logger.info(s"Truncating Delta table relation '$identifier' by truncating table $tableIdentifier") - val deltaTable = DeltaTable.forName(execution.spark, tableIdentifier.quotedString) + logger.info(s"Truncating Delta table relation '$identifier' by truncating table $table") + val deltaTable = DeltaTable.forName(execution.spark, table.quotedString) deltaTable.delete() deltaTable.vacuum() } @@ -338,9 +333,9 @@ case class DeltaTableRelation( require(execution != null) val catalog = execution.catalog - if (!ifExists || catalog.tableExists(tableIdentifier)) { - logger.info(s"Destroying Delta table relation '$identifier' by dropping table $tableIdentifier") - catalog.dropTable(tableIdentifier) + if (!ifExists || catalog.tableExists(table)) { + logger.info(s"Destroying Delta table relation '$identifier' by dropping table $table") + catalog.dropTable(table) provides.foreach(execution.refreshResource) } } @@ -354,18 +349,18 @@ case class DeltaTableRelation( require(execution != null) val catalog = execution.catalog - if (catalog.tableExists(tableIdentifier)) { - val table = catalog.getTable(tableIdentifier) + if (catalog.tableExists(table)) { + val table = catalog.getTable(this.table) if (table.tableType == CatalogTableType.VIEW) { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveTable relation '$identifier' from VIEW to a TABLE $tableIdentifier, but migrations are disabled.") + logger.warn(s"Migration required for HiveTable relation '$identifier' from VIEW to a TABLE $this.table, but migrations are disabled.") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveTable '$identifier' from VIEW to a TABLE $tableIdentifier, since migrations are disabled.") + logger.error(s"Cannot migrate relation HiveTable '$identifier' from VIEW to a TABLE $this.table, since migrations are disabled.") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER|MigrationStrategy.ALTER_REPLACE|MigrationStrategy.REPLACE => - logger.warn(s"TABLE target $tableIdentifier is currently a VIEW, dropping...") - catalog.dropView(tableIdentifier, false) + logger.warn(s"TABLE target $this.table is currently a VIEW, dropping...") + catalog.dropView(this.table, false) create(execution, false) } } @@ -376,18 +371,18 @@ case class DeltaTableRelation( } override protected def deltaTable(execution: Execution) : DeltaTable = { - DeltaTable.forName(execution.spark, tableIdentifier.quotedString) + DeltaTable.forName(execution.spark, table.quotedString) } override protected def deltaCatalogTable(execution: Execution): DeltaTableV2 = { val catalog = execution.catalog - val table = catalog.getTable(tableIdentifier) + val table = catalog.getTable(this.table) DeltaTableV2( execution.spark, new Path(table.location), catalogTable = Some(table), - tableIdentifier = Some(tableIdentifier.toString()) + tableIdentifier = Some(table.toString()) ) } } @@ -395,7 +390,7 @@ case class DeltaTableRelation( @RelationType(kind="deltaTable") class DeltaTableRelationSpec extends RelationSpec with SchemaRelationSpec with PartitionedRelationSpec { - @JsonProperty(value = "database", required = false) private var database: String = "default" + @JsonProperty(value = "database", required = false) private var database: Option[String] = Some("default") @JsonProperty(value = "table", required = true) private var table: String = "" @JsonProperty(value = "location", required = false) private var location: Option[String] = None @JsonProperty(value = "options", required=false) private var options:Map[String,String] = Map() @@ -407,8 +402,7 @@ class DeltaTableRelationSpec extends RelationSpec with SchemaRelationSpec with P instanceProperties(context), schema.map(_.instantiate(context)), partitions.map(_.instantiate(context)), - context.evaluate(database), - context.evaluate(table), + TableIdentifier(context.evaluate(table), context.evaluate(database)), context.evaluate(location).map(p => new Path(p)), context.evaluate(options), context.evaluate(properties), diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala index fb7409572..272eb05fe 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala @@ -99,7 +99,7 @@ case class DeltaVacuumTarget( */ override protected def build(execution: Execution): Unit = { val deltaTable = relation.value match { - case table:DeltaTableRelation => DeltaTable.forName(execution.spark, TableIdentifier(table.table, Some(table.database)).toString()) + case table:DeltaTableRelation => DeltaTable.forName(execution.spark, table.table.toString()) case files:DeltaFileRelation => DeltaTable.forPath(execution.spark, files.location.toString) case rel:Relation => throw new IllegalArgumentException(s"DeltaVacuumTarget only supports relations of type deltaTable and deltaFiles, but it was given relation '${rel.identifier}' of kind '${rel.kind}'") } @@ -127,7 +127,7 @@ case class DeltaVacuumTarget( private def compact(deltaTable:DeltaTable) : Unit = { val spark = deltaTable.toDF.sparkSession val deltaLog = relation.value match { - case table:DeltaTableRelation => DeltaLog.forTable(spark, TableIdentifier(table.table, Some(table.database))) + case table:DeltaTableRelation => DeltaLog.forTable(spark, table.table) case files:DeltaFileRelation => DeltaLog.forTable(spark, files.location.toString) case rel:Relation => throw new IllegalArgumentException(s"DeltaVacuumTarget only supports relations of type deltaTable and deltaFiles, but it was given relation '${rel.identifier}' of kind '${rel.kind}'") } @@ -149,7 +149,7 @@ case class DeltaVacuumTarget( filter.map(writer.option("replaceWhere", _)) relation.value match { - case table:DeltaTableRelation => writer.insertInto(TableIdentifier(table.table, Some(table.database)).toString()) + case table:DeltaTableRelation => writer.insertInto(table.table.toString()) case files:DeltaFileRelation => writer.save(files.location.toString) case rel:Relation => throw new IllegalArgumentException(s"DeltaVacuumTarget only supports relations of type deltaTable and deltaFiles, but it was given relation '${rel.identifier}' of kind '${rel.kind}'") } diff --git a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala index 443bbaeb6..9a45bbdf8 100644 --- a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala +++ b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala @@ -86,8 +86,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = relationSpec.instantiate(session.context).asInstanceOf[DeltaTableRelation] relation.description should be (Some("Some Delta Table")) relation.partitions should be (Seq()) - relation.database should be ("some_db") - relation.table should be ("some_table") + relation.table should be (TableIdentifier("some_table", Some("some_db"))) relation.location should be (Some(new Path("hdfs://ns/some/path"))) relation.options should be (Map()) relation.properties should be (Map()) @@ -107,8 +106,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe Field("int_col", ftypes.IntegerType) )) ), - database = "default", - table = "delta_table" + table = TableIdentifier("delta_table", Some("default")) ) relation.fields should be (Seq( @@ -216,8 +214,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val location = new File(tempDir, "delta/default/lala2") val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -373,8 +370,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -479,8 +475,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val location = new File(tempDir, "delta/default/lala2") val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -575,8 +570,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -632,8 +626,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -737,8 +730,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation0 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -767,8 +759,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe // == Check ================================================================================================= val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), partitions = Seq( PartitionField("part", ftypes.StringType) ) @@ -892,8 +883,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -967,8 +957,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table = TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1157,8 +1146,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val location = new File(tempDir, "delta/default/lala3") val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table2", + table =TableIdentifier("delta_table2", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1240,8 +1228,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val rel_1 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table =TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1252,8 +1239,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe ) val rel_2 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1354,8 +1340,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val rel_1 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1366,8 +1351,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe ) val rel_2 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1447,8 +1431,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val rel_1 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1459,8 +1442,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe ) val rel_2 = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1549,8 +1531,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "streaming_test", + table = TableIdentifier("streaming_test", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( @@ -1610,8 +1591,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe val relation = DeltaTableRelation( Relation.Properties(context, "delta_relation"), - database = "default", - table = "streaming_test", + table = TableIdentifier("streaming_test", Some("default")), schema = Some(EmbeddedSchema( Schema.Properties(context, "delta_schema"), fields = Seq( diff --git a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala index b1963db40..03d792877 100644 --- a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala +++ b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala @@ -22,6 +22,7 @@ import java.time.Duration import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -149,8 +150,7 @@ class DeltaVacuumTargetTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("int_col", IntegerType) ) )), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), location = Some(new Path(location.toURI)) ) @@ -193,8 +193,7 @@ class DeltaVacuumTargetTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("int_col", IntegerType) ) )), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), location = Some(new Path(location.toURI)) ) @@ -252,8 +251,7 @@ class DeltaVacuumTargetTest extends AnyFlatSpec with Matchers with LocalSparkSes partitions = Seq( PartitionField("part", StringType) ), - database = "default", - table = "delta_table", + table = TableIdentifier("delta_table", Some("default")), location = Some(new Path(location.toURI)) ) diff --git a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala index c8c7aa985..c66fece31 100644 --- a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala +++ b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.jdbc.JdbcUtils @@ -44,15 +45,15 @@ import com.dimajix.flowman.types.StructType case class SqlServerRelation( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, - override val partitions: Seq[PartitionField] = Seq(), + override val partitions: Seq[PartitionField] = Seq.empty, connection: Reference[Connection], - properties: Map[String,String] = Map(), - database: Option[String] = None, - table: Option[String] = None, + properties: Map[String,String] = Map.empty, + table: Option[TableIdentifier] = None, query: Option[String] = None, - mergeKey: Seq[String] = Seq(), - primaryKey: Seq[String] = Seq() -) extends JdbcRelationBase(instanceProperties, schema, partitions, connection, properties, database, table, query, mergeKey, primaryKey) { + mergeKey: Seq[String] = Seq.empty, + primaryKey: Seq[String] = Seq.empty, + indexes: Seq[TableIndex] = Seq.empty +) extends JdbcRelationBase(instanceProperties, schema, partitions, connection, properties, table, query, mergeKey, primaryKey, indexes) { private val tempTableIdentifier = TableIdentifier(s"##${tableIdentifier.table}_temp_staging") override protected def doOverwriteAll(execution: Execution, df:DataFrame) : Unit = { @@ -134,14 +135,14 @@ case class SqlServerRelation( @RelationType(kind="sqlserver") -class SqlServerRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec { +class SqlServerRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec with IndexedRelationSpec { @JsonProperty(value = "connection", required = true) private var connection: ConnectionReferenceSpec = _ - @JsonProperty(value = "properties", required = false) private var properties: Map[String, String] = Map() + @JsonProperty(value = "properties", required = false) private var properties: Map[String, String] = Map.empty @JsonProperty(value = "database", required = false) private var database: Option[String] = None @JsonProperty(value = "table", required = false) private var table: Option[String] = None @JsonProperty(value = "query", required = false) private var query: Option[String] = None - @JsonProperty(value = "mergeKey", required = false) private var mergeKey: Seq[String] = Seq() - @JsonProperty(value = "primaryKey", required = false) private var primaryKey: Seq[String] = Seq() + @JsonProperty(value = "mergeKey", required = false) private var mergeKey: Seq[String] = Seq.empty + @JsonProperty(value = "primaryKey", required = false) private var primaryKey: Seq[String] = Seq.empty override def instantiate(context: Context): SqlServerRelation = { new SqlServerRelation( @@ -150,11 +151,11 @@ class SqlServerRelationSpec extends RelationSpec with PartitionedRelationSpec wi partitions.map(_.instantiate(context)), connection.instantiate(context), context.evaluate(properties), - database.map(context.evaluate).filter(_.nonEmpty), - table.map(context.evaluate).filter(_.nonEmpty), - query.map(context.evaluate).filter(_.nonEmpty), + context.evaluate(table).map(t => TableIdentifier(t, context.evaluate(database))), + context.evaluate(query), mergeKey.map(context.evaluate), - primaryKey.map(context.evaluate) + primaryKey.map(context.evaluate), + indexes.map(_.instantiate(context)) ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala index 45c7bd33b..dfc93174f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,9 +30,7 @@ import com.dimajix.flowman.types.FieldValue abstract class HiveRelation extends BaseRelation with PartitionedRelation { protected val logger:Logger - def database: Option[String] - def table: String - def tableIdentifier: TableIdentifier = new TableIdentifier(table, database) + def table: TableIdentifier /** * Reads data from the relation, possibly from specific partitions @@ -46,10 +44,10 @@ abstract class HiveRelation extends BaseRelation with PartitionedRelation { require(execution != null) require(partitions != null) - logger.info(s"Reading Hive relation '$identifier' from table $tableIdentifier using partition values $partitions") + logger.info(s"Reading Hive relation '$identifier' from table $table using partition values $partitions") val reader = execution.spark.read - val tableDf = reader.table(tableIdentifier.unquotedString) + val tableDf = reader.table(table.unquotedString) val filteredDf = filterPartition(tableDf, partitions) applyInputSchema(execution, filteredDf) @@ -64,6 +62,6 @@ abstract class HiveRelation extends BaseRelation with PartitionedRelation { require(execution != null) val catalog = execution.catalog - catalog.tableExists(tableIdentifier) + catalog.tableExists(table) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala index 02cb6ce8e..364b6de53 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala @@ -96,8 +96,7 @@ case class HiveTableRelation( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, override val partitions: Seq[PartitionField] = Seq(), - override val database: Option[String] = None, - override val table: String, + override val table: TableIdentifier, external: Boolean = false, location: Option[Path] = None, format: Option[String] = None, @@ -125,7 +124,7 @@ case class HiveTableRelation( // Only return Hive table partitions! val allPartitions = PartitionSchema(this.partitions).interpolate(partition) - allPartitions.map(p => ResourceIdentifier.ofHivePartition(table, database, p.toMap)).toSet + allPartitions.map(p => ResourceIdentifier.ofHivePartition(table, p.toMap)).toSet } /** @@ -134,7 +133,7 @@ case class HiveTableRelation( * @return */ override def provides : Set[ResourceIdentifier] = Set( - ResourceIdentifier.ofHiveTable(table, database) + ResourceIdentifier.ofHiveTable(table) ) /** @@ -143,7 +142,7 @@ case class HiveTableRelation( * @return */ override def requires : Set[ResourceIdentifier] = { - database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ super.requires + table.database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ super.requires } /** @@ -187,7 +186,7 @@ case class HiveTableRelation( require(partitionSpec != null) require(mode != null) - logger.info(s"Writing Hive relation '$identifier' to table $tableIdentifier partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode' using Hive insert") + logger.info(s"Writing Hive relation '$identifier' to table $table partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode' using Hive insert") // Apply output schema before writing to Hive val outputDf = { @@ -200,10 +199,10 @@ case class HiveTableRelation( def loaded() : Boolean = { val catalog = execution.catalog if (partitionSpec.nonEmpty) { - catalog.partitionExists(tableIdentifier, partitionSpec) + catalog.partitionExists(table, partitionSpec) } else { - val location = catalog.getTableLocation(tableIdentifier) + val location = catalog.getTableLocation(table) val fs = location.getFileSystem(execution.hadoopConf) FileUtils.isValidHiveData(fs, location) } @@ -219,9 +218,9 @@ case class HiveTableRelation( case OutputMode.ERROR_IF_EXISTS => if (loaded()) { if (partitionSpec.nonEmpty) - throw new PartitionAlreadyExistsException(database.getOrElse(""), table, partitionSpec.mapValues(_.toString).toMap) + throw new PartitionAlreadyExistsException(table.database.getOrElse(""), table.table, partitionSpec.mapValues(_.toString).toMap) else - throw new TableAlreadyExistsException(database.getOrElse(""), table) + throw new TableAlreadyExistsException(table.database.getOrElse(""), table.table) } writeHiveTable(execution, outputDf, partitionSpec, mode) case _ => @@ -234,7 +233,7 @@ case class HiveTableRelation( val catalog = execution.catalog if (partitionSpec.nonEmpty) { - val hiveTable = catalog.getTable(TableIdentifier(table, database)) + val hiveTable = catalog.getTable(table) val query = df.queryExecution.logical val overwrite = mode == OutputMode.OVERWRITE || mode == OutputMode.OVERWRITE_DYNAMIC @@ -250,20 +249,20 @@ case class HiveTableRelation( SparkShim.withNewExecutionId(spark, qe)(qe.toRdd) // Finally refresh Hive partition - catalog.refreshPartition(tableIdentifier, partitionSpec) + catalog.refreshPartition(table, partitionSpec) } else { // If OVERWRITE is specified, remove all partitions for partitioned tables if (partitions.nonEmpty && mode == OutputMode.OVERWRITE) { - catalog.truncateTable(tableIdentifier) + catalog.truncateTable(table) } val writer = df.write .mode(mode.batchMode) .options(options) format.foreach(writer.format) - writer.insertInto(tableIdentifier.unquotedString) + writer.insertInto(table.unquotedString) - execution.catalog.refreshTable(tableIdentifier) + execution.catalog.refreshTable(table) } } @@ -282,7 +281,7 @@ case class HiveTableRelation( require(partitionSpec != null) require(mode != null) - logger.info(s"Writing Hive relation '$identifier' to table $tableIdentifier partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode' using direct write") + logger.info(s"Writing Hive relation '$identifier' to table $table partition ${HiveDialect.expr.partition(partitionSpec)} with mode '$mode' using direct write") if (location.isEmpty) throw new IllegalArgumentException("Hive table relation requires 'location' for direct write mode") @@ -303,10 +302,10 @@ case class HiveTableRelation( // Finally add Hive partition val catalog = execution.catalog if (partitionSpec.nonEmpty) { - catalog.addOrReplacePartition(tableIdentifier, partitionSpec, outputPath) + catalog.addOrReplacePartition(table, partitionSpec, outputPath) } else { - catalog.refreshTable(tableIdentifier) + catalog.refreshTable(table) } } @@ -327,13 +326,13 @@ case class HiveTableRelation( if (partitions.nonEmpty) { val partitionSchema = PartitionSchema(this.partitions) partitionSchema.interpolate(partitions).foreach { spec => - logger.info(s"Truncating Hive relation '$identifier' by truncating table $tableIdentifier partition ${HiveDialect.expr.partition(spec)}") - catalog.dropPartition(tableIdentifier, spec) + logger.info(s"Truncating Hive relation '$identifier' by truncating table $table partition ${HiveDialect.expr.partition(spec)}") + catalog.dropPartition(table, spec) } } else { - logger.info(s"Truncating Hive relation '$identifier' by truncating table $tableIdentifier") - catalog.truncateTable(tableIdentifier) + logger.info(s"Truncating Hive relation '$identifier' by truncating table $table") + catalog.truncateTable(table) } } @@ -346,9 +345,9 @@ case class HiveTableRelation( */ override def conforms(execution: Execution, migrationPolicy: MigrationPolicy): Trilean = { val catalog = execution.catalog - if (catalog.tableExists(tableIdentifier)) { + if (catalog.tableExists(table)) { if (schema.nonEmpty) { - val table = catalog.getTable(tableIdentifier) + val table = catalog.getTable(this.table) if (table.tableType == CatalogTableType.VIEW) { false } @@ -361,7 +360,7 @@ case class HiveTableRelation( else SchemaUtils.replaceCharVarchar(dataSchema) } - val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) + val targetTable = TableDefinition(this.table, targetSchema.fields) !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } @@ -394,12 +393,12 @@ case class HiveTableRelation( if (partitions.nonEmpty) { val schema = PartitionSchema(partitions) val partitionSpec = schema.spec(partition) - catalog.tableExists(tableIdentifier) && - catalog.partitionExists(tableIdentifier, partitionSpec) + catalog.tableExists(table) && + catalog.partitionExists(table, partitionSpec) } else { - if (catalog.tableExists(tableIdentifier)) { - val location = catalog.getTableLocation(tableIdentifier) + if (catalog.tableExists(table)) { + val location = catalog.getTableLocation(table) val fs = location.getFileSystem(execution.hadoopConf) FileUtils.isValidHiveData(fs, location) } @@ -419,7 +418,7 @@ case class HiveTableRelation( if (!ifNotExists || exists(execution) == No) { val catalogSchema = HiveTableRelation.cleanupSchema(StructType(fields.map(_.catalogField))) - logger.info(s"Creating Hive table relation '$identifier' with table $tableIdentifier and schema\n${catalogSchema.treeString}") + logger.info(s"Creating Hive table relation '$identifier' with table $table and schema\n${catalogSchema.treeString}") if (schema.isEmpty) { throw new UnspecifiedSchemaException(identifier) } @@ -443,7 +442,7 @@ case class HiveTableRelation( outputFormat = s.outputFormat, serde = s.serde) case None => - throw new IllegalArgumentException(s"File format '$format' not supported in Hive relation '$identifier' while creating hive table $tableIdentifier") + throw new IllegalArgumentException(s"File format '$format' not supported in Hive relation '$identifier' while creating hive table $table") } } else { @@ -466,7 +465,7 @@ case class HiveTableRelation( // Configure catalog table by assembling all options val catalogTable = CatalogTable( - identifier = tableIdentifier, + identifier = table, tableType = if (external) CatalogTableType.EXTERNAL @@ -503,9 +502,9 @@ case class HiveTableRelation( require(execution != null) val catalog = execution.catalog - if (!ifExists || catalog.tableExists(tableIdentifier)) { - logger.info(s"Destroying Hive table relation '$identifier' by dropping table $tableIdentifier") - catalog.dropTable(tableIdentifier) + if (!ifExists || catalog.tableExists(table)) { + logger.info(s"Destroying Hive table relation '$identifier' by dropping table $table") + catalog.dropTable(table) provides.foreach(execution.refreshResource) } } @@ -518,18 +517,18 @@ case class HiveTableRelation( require(execution != null) val catalog = execution.catalog - if (schema.nonEmpty && catalog.tableExists(tableIdentifier)) { - val table = catalog.getTable(tableIdentifier) + if (schema.nonEmpty && catalog.tableExists(table)) { + val table = catalog.getTable(this.table) if (table.tableType == CatalogTableType.VIEW) { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveTable relation '$identifier' from VIEW to a TABLE $tableIdentifier, but migrations are disabled.") + logger.warn(s"Migration required for HiveTable relation '$identifier' from VIEW to a TABLE ${this.table}, but migrations are disabled.") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveTable '$identifier' from VIEW to a TABLE $tableIdentifier, since migrations are disabled.") + logger.error(s"Cannot migrate relation HiveTable '$identifier' from VIEW to a TABLE ${this.table}, since migrations are disabled.") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER|MigrationStrategy.ALTER_REPLACE|MigrationStrategy.REPLACE => - logger.warn(s"TABLE target $tableIdentifier is currently a VIEW, dropping...") - catalog.dropView(tableIdentifier, false) + logger.warn(s"TABLE target ${this.table} is currently a VIEW, dropping...") + catalog.dropView(this.table, false) create(execution, false) provides.foreach(execution.refreshResource) } @@ -543,7 +542,7 @@ case class HiveTableRelation( else SchemaUtils.replaceCharVarchar(dataSchema) } - val targetTable = TableDefinition(tableIdentifier, targetSchema.fields) + val targetTable = TableDefinition(this.table, targetSchema.fields) val requiresMigration = TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) if (requiresMigration) { @@ -557,14 +556,14 @@ case class HiveTableRelation( private def doMigration(execution: Execution, currentTable:TableDefinition, targetTable:TableDefinition, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveTable relation '$identifier' of Hive table $tableIdentifier, but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") + logger.warn(s"Migration required for HiveTable relation '$identifier' of Hive table $table, but migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") + logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $table, since migrations are disabled.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER => val migrations = TableChange.migrate(currentTable, targetTable, migrationPolicy) if (migrations.exists(m => !supported(m))) { - logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $tableIdentifier, since that would require unsupported changes.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") + logger.error(s"Cannot migrate relation HiveTable '$identifier' of Hive table $table, since that would require unsupported changes.\nCurrent schema:\n${currentTable.schema.treeString}New schema:\n${targetTable.schema.treeString}") throw new MigrationFailedException(identifier) } alter(migrations) @@ -581,13 +580,13 @@ case class HiveTableRelation( } def alter(migrations:Seq[TableChange]) : Unit = { - logger.info(s"Migrating HiveTable relation '$identifier', this will alter the Hive table $tableIdentifier. New schema:\n${targetTable.schema.treeString}") + logger.info(s"Migrating HiveTable relation '$identifier', this will alter the Hive table $table. New schema:\n${targetTable.schema.treeString}") if (migrations.isEmpty) { logger.warn("Empty list of migrations - nothing to do") } try { - execution.catalog.alterTable(tableIdentifier, migrations) + execution.catalog.alterTable(table, migrations) } catch { case NonFatal(ex) => throw new MigrationFailedException(identifier, ex) @@ -595,7 +594,7 @@ case class HiveTableRelation( } def recreate() : Unit = { - logger.info(s"Migrating HiveTable relation '$identifier', this will drop/create the Hive table $tableIdentifier.") + logger.info(s"Migrating HiveTable relation '$identifier', this will drop/create the Hive table $table.") try { destroy(execution, true) create(execution, true) @@ -619,7 +618,7 @@ case class HiveTableRelation( override protected def outputSchema(execution:Execution) : Option[StructType] = { // We specifically use the existing physical Hive schema - val currentSchema = execution.catalog.getTable(tableIdentifier).dataSchema + val currentSchema = execution.catalog.getTable(table).dataSchema // If a schema is explicitly specified, we use that one to back-merge VarChar(n) and Char(n). This // is mainly required for Spark < 3.1, which cannot correctly handle VARCHAR and CHAR types in Hive @@ -698,8 +697,7 @@ class HiveTableRelationSpec extends RelationSpec with SchemaRelationSpec with Pa instanceProperties(context), schema.map(_.instantiate(context)), partitions.map(_.instantiate(context)), - context.evaluate(database), - context.evaluate(table), + TableIdentifier(context.evaluate(table), context.evaluate(database)), context.evaluate(external).toBoolean, context.evaluate(location).map(p => new Path(p)), context.evaluate(format), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala index c16cd7ee9..79d439aea 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala @@ -75,11 +75,9 @@ case class HiveUnionTableRelation( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, override val partitions: Seq[PartitionField] = Seq(), - tableDatabase: Option[String] = None, - tablePrefix: String, + tablePrefix: TableIdentifier, locationPrefix: Option[Path] = None, - viewDatabase: Option[String] = None, - view: String, + view: TableIdentifier, external: Boolean = false, format: Option[String] = None, options: Map[String,String] = Map(), @@ -91,15 +89,22 @@ case class HiveUnionTableRelation( ) extends BaseRelation with SchemaRelation with PartitionedRelation { private val logger = LoggerFactory.getLogger(classOf[HiveUnionTableRelation]) - def viewIdentifier: TableIdentifier = TableIdentifier(view, viewDatabase) - def tableIdentifier(version:Int) : TableIdentifier = { - TableIdentifier(tablePrefix + "_" + version.toString, tableDatabase) + private lazy val tableRegex : TableIdentifier = { + TableIdentifier(tablePrefix.table + "_[0-9]+", tablePrefix.database.orElse(view.database)) } + private lazy val viewIdentifier : TableIdentifier = { + TableIdentifier(view.table, view.database.orElse(tablePrefix.database)) + } + private def tableIdentifier(version:Int) : TableIdentifier = { + TableIdentifier(tablePrefix.table + "_" + version.toString, tablePrefix.database.orElse(view.database)) + } + + private def resolve(execution: Execution, table:TableIdentifier) : TableIdentifier = TableIdentifier(table.table, table.database.orElse(view.database).orElse(Some(execution.catalog.currentDatabase))) private def listTables(executor: Execution) : Seq[TableIdentifier] = { val catalog = executor.catalog - val regex = (TableIdentifier(tablePrefix, tableDatabase.orElse(Some(catalog.currentDatabase))).unquotedString + "_[0-9]+").r - catalog.listTables(tableDatabase.getOrElse(catalog.currentDatabase), tablePrefix + "_*") + val regex = resolve(executor, tableRegex).unquotedString.r + catalog.listTables(tablePrefix.database.getOrElse(catalog.currentDatabase), tablePrefix.table + "_*") .filter { table => table.unquotedString match { case regex() => true @@ -110,7 +115,7 @@ case class HiveUnionTableRelation( private def tableRelation(version:Int) : HiveTableRelation = tableRelation( - TableIdentifier(tablePrefix + "_" + version.toString, tableDatabase), + TableIdentifier(tablePrefix.table + "_" + version.toString, tablePrefix.database.orElse(view.database)), locationPrefix.map(p => new Path(p.toString + "_" + version.toString)) ) @@ -118,8 +123,7 @@ case class HiveUnionTableRelation( instanceProperties, schema, partitions, - tableIdentifier.database, - tableIdentifier.table, + tableIdentifier, external, location, format, @@ -135,7 +139,6 @@ case class HiveUnionTableRelation( private def viewRelationFromSql(sql:String) : HiveViewRelation = { HiveViewRelation( instanceProperties, - viewDatabase, view, partitions, Some(sql), @@ -158,8 +161,8 @@ case class HiveUnionTableRelation( * @return */ override def provides: Set[ResourceIdentifier] = Set( - ResourceIdentifier.ofHiveTable(tablePrefix + "_[0-9]+", tableDatabase), - ResourceIdentifier.ofHiveTable(view, viewDatabase.orElse(tableDatabase)) + ResourceIdentifier.ofHiveTable(tableRegex), + ResourceIdentifier.ofHiveTable(viewIdentifier) ) /** @@ -168,8 +171,8 @@ case class HiveUnionTableRelation( * @return */ override def requires: Set[ResourceIdentifier] = { - tableDatabase.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ - viewDatabase.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ + tablePrefix.database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ + viewIdentifier.database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet ++ super.requires } @@ -188,7 +191,7 @@ case class HiveUnionTableRelation( // Only return Hive table partitions! val allPartitions = PartitionSchema(this.partitions).interpolate(partition) - allPartitions.map(p => ResourceIdentifier.ofHivePartition(tablePrefix + "_[0-9]+", tableDatabase, p.toMap)).toSet + allPartitions.map(p => ResourceIdentifier.ofHivePartition(tableRegex, p.toMap)).toSet } @@ -416,7 +419,7 @@ case class HiveUnionTableRelation( // Create initial view val spark = execution.spark - val df = spark.read.table(hiveTableRelation.tableIdentifier.unquotedString) + val df = spark.read.table(hiveTableRelation.table.unquotedString) val sql = new SqlBuilder(df).toSQL val hiveViewRelation = viewRelationFromSql(sql) hiveViewRelation.create(execution, ifNotExists) @@ -541,7 +544,7 @@ case class HiveUnionTableRelation( class HiveUnionTableRelationSpec extends RelationSpec with SchemaRelationSpec with PartitionedRelationSpec { @JsonProperty(value = "tableDatabase", required = false) private var tableDatabase: Option[String] = None - @JsonProperty(value = "tablePrefix", required = true) private var tablePrefix: String = "" + @JsonProperty(value = "tablePrefix", required = true) private var tablePrefix: String = "zz" @JsonProperty(value = "locationPrefix", required = false) private var locationPrefix: Option[String] = None @JsonProperty(value = "viewDatabase", required = false) private var viewDatabase: Option[String] = None @JsonProperty(value = "view", required = true) private var view: String = "" @@ -564,11 +567,9 @@ class HiveUnionTableRelationSpec extends RelationSpec with SchemaRelationSpec wi instanceProperties(context), schema.map(_.instantiate(context)), partitions.map(_.instantiate(context)), - context.evaluate(tableDatabase), - context.evaluate(tablePrefix), + TableIdentifier(context.evaluate(tablePrefix), context.evaluate(tableDatabase)), context.evaluate(locationPrefix).map(p => new Path(context.evaluate(p))), - context.evaluate(viewDatabase), - context.evaluate(view), + TableIdentifier(context.evaluate(view), context.evaluate(viewDatabase)), context.evaluate(external).toBoolean, context.evaluate(format), context.evaluate(options), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala index b023815b5..420decc8d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala @@ -18,6 +18,7 @@ package com.dimajix.flowman.spec.relation import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.slf4j.LoggerFactory @@ -44,8 +45,7 @@ import com.dimajix.spark.sql.catalyst.SqlBuilder case class HiveViewRelation( override val instanceProperties:Relation.Properties, - override val database: Option[String], - override val table: String, + override val table: TableIdentifier, override val partitions: Seq[PartitionField] = Seq(), sql: Option[String] = None, mapping: Option[MappingOutputIdentifier] = None @@ -63,7 +63,7 @@ case class HiveViewRelation( mapping.map(m => MappingUtils.requires(context, m.mapping)) .orElse( // Only return Hive Table Partitions! - sql.map(s => SqlParser.resolveDependencies(s).map(t => ResourceIdentifier.ofHivePartition(t, Map()).asInstanceOf[ResourceIdentifier])) + sql.map(s => SqlParser.resolveDependencies(s).map(t => ResourceIdentifier.ofHivePartition(t, Map.empty[String,Any]).asInstanceOf[ResourceIdentifier])) ) .getOrElse(Set()) } @@ -74,7 +74,7 @@ case class HiveViewRelation( * @return */ override def provides : Set[ResourceIdentifier] = Set( - ResourceIdentifier.ofHiveTable(table, database) + ResourceIdentifier.ofHiveTable(table) ) /** @@ -83,7 +83,7 @@ case class HiveViewRelation( * @return */ override def requires : Set[ResourceIdentifier] = { - val db = database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet + val db = table.database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet val other = mapping.map {m => MappingUtils.requires(context, m.mapping) // Replace all Hive partitions with Hive tables @@ -120,13 +120,13 @@ case class HiveViewRelation( */ override def conforms(execution: Execution, migrationPolicy: MigrationPolicy): Trilean = { val catalog = execution.catalog - if (catalog.tableExists(tableIdentifier)) { + if (catalog.tableExists(table)) { val newSelect = getSelect(execution) - val curTable = catalog.getTable(tableIdentifier) + val curTable = catalog.getTable(table) // Check if current table is a VIEW or a table if (curTable.tableType == CatalogTableType.VIEW) { // Check that both SQL and schema are correct - val curTable = catalog.getTable(tableIdentifier) + val curTable = catalog.getTable(table) val curSchema = SchemaUtils.normalize(curTable.schema) val newSchema = SchemaUtils.normalize(catalog.spark.sql(newSelect).schema) curTable.viewText.get == newSelect && curSchema == newSchema @@ -161,9 +161,9 @@ case class HiveViewRelation( override def create(execution:Execution, ifNotExists:Boolean=false) : Unit = { val select = getSelect(execution) val catalog = execution.catalog - if (!ifNotExists || !catalog.tableExists(tableIdentifier)) { - logger.info(s"Creating Hive view relation '$identifier' with VIEW $tableIdentifier") - catalog.createView(tableIdentifier, select, ifNotExists) + if (!ifNotExists || !catalog.tableExists(table)) { + logger.info(s"Creating Hive view relation '$identifier' with VIEW $table") + catalog.createView(table, select, ifNotExists) provides.foreach(execution.refreshResource) } } @@ -175,9 +175,9 @@ case class HiveViewRelation( */ override def migrate(execution:Execution, migrationPolicy:MigrationPolicy, migrationStrategy:MigrationStrategy) : Unit = { val catalog = execution.catalog - if (catalog.tableExists(tableIdentifier)) { + if (catalog.tableExists(table)) { val newSelect = getSelect(execution) - val curTable = catalog.getTable(tableIdentifier) + val curTable = catalog.getTable(table) // Check if current table is a VIEW or a table if (curTable.tableType == CatalogTableType.VIEW) { migrateFromView(catalog, newSelect, migrationStrategy) @@ -190,19 +190,19 @@ case class HiveViewRelation( } private def migrateFromView(catalog:HiveCatalog, newSelect:String, migrationStrategy:MigrationStrategy) : Unit = { - val curTable = catalog.getTable(tableIdentifier) + val curTable = catalog.getTable(table) val curSchema = SchemaUtils.normalize(curTable.schema) val newSchema = SchemaUtils.normalize(catalog.spark.sql(newSelect).schema) if (curTable.viewText.get != newSelect || curSchema != newSchema) { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveView relation '$identifier' of Hive view $tableIdentifier, but migrations are disabled.") + logger.warn(s"Migration required for HiveView relation '$identifier' of Hive view $table, but migrations are disabled.") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveView '$identifier' of Hive view $tableIdentifier, since migrations are disabled.") + logger.error(s"Cannot migrate relation HiveView '$identifier' of Hive view $table, since migrations are disabled.") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER|MigrationStrategy.ALTER_REPLACE|MigrationStrategy.REPLACE => - logger.info(s"Migrating HiveView relation '$identifier' with VIEW $tableIdentifier") - catalog.alterView(tableIdentifier, newSelect) + logger.info(s"Migrating HiveView relation '$identifier' with VIEW $table") + catalog.alterView(table, newSelect) } } } @@ -210,14 +210,14 @@ case class HiveViewRelation( private def migrateFromTable(catalog:HiveCatalog, newSelect:String, migrationStrategy:MigrationStrategy) : Unit = { migrationStrategy match { case MigrationStrategy.NEVER => - logger.warn(s"Migration required for HiveView relation '$identifier' from TABLE to a VIEW $tableIdentifier, but migrations are disabled.") + logger.warn(s"Migration required for HiveView relation '$identifier' from TABLE to a VIEW $table, but migrations are disabled.") case MigrationStrategy.FAIL => - logger.error(s"Cannot migrate relation HiveView '$identifier' from TABLE to a VIEW $tableIdentifier, since migrations are disabled.") + logger.error(s"Cannot migrate relation HiveView '$identifier' from TABLE to a VIEW $table, since migrations are disabled.") throw new MigrationFailedException(identifier) case MigrationStrategy.ALTER|MigrationStrategy.ALTER_REPLACE|MigrationStrategy.REPLACE => - logger.info(s"Migrating HiveView relation '$identifier' from TABLE to a VIEW $tableIdentifier") - catalog.dropTable(tableIdentifier, false) - catalog.createView(tableIdentifier, newSelect, false) + logger.info(s"Migrating HiveView relation '$identifier' from TABLE to a VIEW $table") + catalog.dropTable(table, false) + catalog.createView(table, newSelect, false) } } @@ -227,9 +227,9 @@ case class HiveViewRelation( */ override def destroy(execution:Execution, ifExists:Boolean=false) : Unit = { val catalog = execution.catalog - if (!ifExists || catalog.tableExists(tableIdentifier)) { - logger.info(s"Destroying Hive view relation '$identifier' with VIEW $tableIdentifier") - catalog.dropView(tableIdentifier) + if (!ifExists || catalog.tableExists(table)) { + logger.info(s"Destroying Hive view relation '$identifier' with VIEW $table") + catalog.dropView(table) provides.foreach(execution.refreshResource) } } @@ -238,7 +238,7 @@ case class HiveViewRelation( val select = sql.orElse(mapping.map(id => buildMappingSql(executor, id))) .getOrElse(throw new IllegalArgumentException("HiveView either requires explicit SQL SELECT statement or mapping")) - logger.debug(s"Hive SQL SELECT statement for VIEW $tableIdentifier: $select") + logger.debug(s"Hive SQL SELECT statement for VIEW $table: $select") select } @@ -266,8 +266,7 @@ class HiveViewRelationSpec extends RelationSpec with PartitionedRelationSpec{ override def instantiate(context: Context): HiveViewRelation = { HiveViewRelation( instanceProperties(context), - context.evaluate(database), - context.evaluate(view), + TableIdentifier(context.evaluate(view), context.evaluate(database)), partitions.map(_.instantiate(context)), context.evaluate(sql), context.evaluate(mapping).map(MappingOutputIdentifier.parse) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/IndexedRelationSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/IndexedRelationSpec.scala new file mode 100644 index 000000000..86fbb9417 --- /dev/null +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/IndexedRelationSpec.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.spec.relation + +import com.fasterxml.jackson.annotation.JsonProperty + +import com.dimajix.flowman.catalog.TableIndex +import com.dimajix.flowman.execution.Context + + +class IndexSpec { + @JsonProperty(value = "name", required = true) protected var name: String = _ + @JsonProperty(value = "columns", required = true) protected var columns: Seq[String] = Seq.empty + @JsonProperty(value = "unique", required = true) protected var unique: String = "false" + + def instantiate(context:Context) : TableIndex = { + TableIndex( + context.evaluate(name), + columns.map(context.evaluate), + context.evaluate(unique).toBoolean + ) + } +} + + +trait IndexedRelationSpec { this: RelationSpec => + @JsonProperty(value = "indexes", required = false) protected var indexes: Seq[IndexSpec] = Seq.empty +} diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 882ea191b..47c96e00d 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -36,13 +36,14 @@ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructType +import org.slf4j.Logger import org.slf4j.LoggerFactory import com.dimajix.common.SetIgnoreCase import com.dimajix.common.Trilean -import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.Execution @@ -77,17 +78,39 @@ import com.dimajix.flowman.types.{StructType => FlowmanStructType} class JdbcRelationBase( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, - override val partitions: Seq[PartitionField] = Seq(), + override val partitions: Seq[PartitionField] = Seq.empty, connection: Reference[Connection], - properties: Map[String,String] = Map(), - database: Option[String] = None, - table: Option[String] = None, + properties: Map[String,String] = Map.empty, + table: Option[TableIdentifier] = None, query: Option[String] = None, - mergeKey: Seq[String] = Seq(), - primaryKey: Seq[String] = Seq() + mergeKey: Seq[String] = Seq.empty, + primaryKey: Seq[String] = Seq.empty, + indexes: Seq[TableIndex] = Seq.empty ) extends BaseRelation with PartitionedRelation with SchemaRelation { - protected val logger = LoggerFactory.getLogger(getClass) - protected val tableIdentifier : TableIdentifier = TableIdentifier(table.getOrElse(""), database) + protected val logger : Logger = LoggerFactory.getLogger(getClass) + protected val tableIdentifier = table.getOrElse(TableIdentifier("")) + protected lazy val tableDefinition : Option[TableDefinition] = { + schema.map { schema => + val pk = if (primaryKey.nonEmpty) primaryKey else schema.primaryKey + + // Make Primary key columns not-nullable + val pkSet = SetIgnoreCase(pk) + val columns = fullSchema.get.fields.map { f => + if (pkSet.contains(f.name)) + f.copy(nullable=false) + else + f + } + + TableDefinition( + tableIdentifier, + columns, + schema.description, + pk, + indexes + ) + } + } if (query.nonEmpty && table.nonEmpty) throw new IllegalArgumentException(s"JDBC relation '$identifier' cannot have both a table and a SQL query defined") @@ -103,7 +126,7 @@ class JdbcRelationBase( override def provides: Set[ResourceIdentifier] = { // Only return a resource if a table is defined, which implies that this relation can be used for creating // and destroying JDBC tables - table.map(t => ResourceIdentifier.ofJdbcTable(t, database)).toSet + table.map(t => ResourceIdentifier.ofJdbcTable(t)).toSet } /** @@ -114,7 +137,7 @@ class JdbcRelationBase( override def requires: Set[ResourceIdentifier] = { // Only return a resource if a table is defined, which implies that this relation can be used for creating // and destroying JDBC tables - database.map(db => ResourceIdentifier.ofJdbcDatabase(db)).toSet ++ super.requires + table.flatMap(_.database.map(db => ResourceIdentifier.ofJdbcDatabase(db))).toSet ++ super.requires } /** @@ -135,7 +158,7 @@ class JdbcRelationBase( } else { val allPartitions = PartitionSchema(this.partitions).interpolate(partitions) - allPartitions.map(p => ResourceIdentifier.ofJdbcTablePartition(table.get, database, p.toMap)).toSet + allPartitions.map(p => ResourceIdentifier.ofJdbcTablePartition(tableIdentifier, p.toMap)).toSet } } @@ -232,7 +255,7 @@ class JdbcRelationBase( doAppend(execution, dfExt) } else { - throw new PartitionAlreadyExistsException(database.getOrElse(""), table.get, partition.mapValues(_.value)) + throw new PartitionAlreadyExistsException(tableIdentifier.database.getOrElse(""), tableIdentifier.table, partition.mapValues(_.value)) } case OutputMode.UPDATE => doUpdate(execution, df) @@ -369,14 +392,11 @@ class JdbcRelationBase( else { withConnection { (con, options) => if (JdbcUtils.tableExists(con, tableIdentifier, options)) { - if (schema.nonEmpty) { - val currentTable = JdbcUtils.getTable(con, tableIdentifier, options) - val targetTable = TableDefinition(tableIdentifier, fullSchema.get.fields) - - !TableChange.requiresMigration(currentTable, targetTable, migrationPolicy) - } - else { - true + tableDefinition match { + case Some(targetTable) => + val currentTable = JdbcUtils.getTable(con, tableIdentifier, options) + !TableChange.requiresMigration(currentTable, targetTable, migrationPolicy) + case None => true } } else { @@ -429,17 +449,13 @@ class JdbcRelationBase( protected def doCreate(con:java.sql.Connection, options:JDBCOptions): Unit = { logger.info(s"Creating JDBC relation '$identifier', this will create JDBC table $tableIdentifier with schema\n${this.schema.map(_.treeString).orNull}") - if (this.schema.isEmpty) { - throw new UnspecifiedSchemaException(identifier) - } - val schema = this.schema.get - val table = catalog.TableDefinition( - tableIdentifier, - schema.fields ++ partitions.map(_.field), - schema.description, - if (primaryKey.nonEmpty) primaryKey else schema.primaryKey - ) - JdbcUtils.createTable(con, table, options) + + tableDefinition match { + case Some(table) => + JdbcUtils.createTable(con, table, options) + case None => + throw new UnspecifiedSchemaException(identifier) + } } /** @@ -466,11 +482,10 @@ class JdbcRelationBase( throw new UnsupportedOperationException(s"Cannot migrate JDBC relation '$identifier' which is defined by an SQL query") // Only try migration if schema is explicitly specified - if (schema.isDefined) { + tableDefinition.foreach { targetTable => withConnection { (con, options) => if (JdbcUtils.tableExists(con, tableIdentifier, options)) { val currentTable = JdbcUtils.getTable(con, tableIdentifier, options) - val targetTable = TableDefinition(tableIdentifier, fullSchema.get.fields) if (TableChange.requiresMigration(currentTable, targetTable, migrationPolicy)) { doMigration(currentTable, targetTable, migrationPolicy, migrationStrategy) @@ -675,37 +690,37 @@ class JdbcRelationBase( case class JdbcRelation( override val instanceProperties:Relation.Properties, override val schema:Option[Schema] = None, - override val partitions: Seq[PartitionField] = Seq(), + override val partitions: Seq[PartitionField] = Seq.empty, connection: Reference[Connection], - properties: Map[String,String] = Map(), - database: Option[String] = None, - table: Option[String] = None, + properties: Map[String,String] = Map.empty, + table: Option[TableIdentifier] = None, query: Option[String] = None, - mergeKey: Seq[String] = Seq(), - primaryKey: Seq[String] = Seq() + mergeKey: Seq[String] = Seq.empty, + primaryKey: Seq[String] = Seq.empty, + indexes: Seq[TableIndex] = Seq.empty ) extends JdbcRelationBase( instanceProperties, schema, partitions, connection, properties, - database, table, query, mergeKey, - primaryKey + primaryKey, + indexes ) { } -class JdbcRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec { +class JdbcRelationSpec extends RelationSpec with PartitionedRelationSpec with SchemaRelationSpec with IndexedRelationSpec { @JsonProperty(value = "connection", required = true) private var connection: ConnectionReferenceSpec = _ - @JsonProperty(value = "properties", required = false) private var properties: Map[String, String] = Map() + @JsonProperty(value = "properties", required = false) private var properties: Map[String, String] = Map.empty @JsonProperty(value = "database", required = false) private var database: Option[String] = None @JsonProperty(value = "table", required = false) private var table: Option[String] = None @JsonProperty(value = "query", required = false) private var query: Option[String] = None - @JsonProperty(value = "mergeKey", required = false) private var mergeKey: Seq[String] = Seq() - @JsonProperty(value = "primaryKey", required = false) private var primaryKey: Seq[String] = Seq() + @JsonProperty(value = "mergeKey", required = false) private var mergeKey: Seq[String] = Seq.empty + @JsonProperty(value = "primaryKey", required = false) private var primaryKey: Seq[String] = Seq.empty /** * Creates the instance of the specified Relation with all variable interpolation being performed @@ -719,11 +734,11 @@ class JdbcRelationSpec extends RelationSpec with PartitionedRelationSpec with Sc partitions.map(_.instantiate(context)), connection.instantiate(context), context.evaluate(properties), - database.map(context.evaluate).filter(_.nonEmpty), - table.map(context.evaluate).filter(_.nonEmpty), - query.map(context.evaluate).filter(_.nonEmpty), + context.evaluate(table).map(t => TableIdentifier(t, context.evaluate(database))), + context.evaluate(query), mergeKey.map(context.evaluate), - primaryKey.map(context.evaluate) + primaryKey.map(context.evaluate), + indexes.map(_.instantiate(context)) ) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala index 5c8393b79..39c657625 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala @@ -16,6 +16,7 @@ package com.dimajix.flowman.spec.mapping +import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.DoubleType import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.types.StringType @@ -69,8 +70,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0007", + table = TableIdentifier("lala_0007", Some("default")), format = Some("parquet"), schema = Some(EmbeddedSchema( Schema.Properties(context), @@ -116,8 +116,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0007", + table = TableIdentifier("lala_0007", Some("default")), format = Some("parquet"), schema = Some(EmbeddedSchema( Schema.Properties(context), diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala index 3c57a3547..4ffd8afae 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala @@ -409,8 +409,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0006", + table = TableIdentifier("lala_0006", Some("default")), format = Some("parquet"), schema = Some(EmbeddedSchema( Schema.Properties(context), @@ -453,8 +452,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0007", + table = TableIdentifier("lala_0007", Some("default")), format = Some("avro"), schema = Some(EmbeddedSchema( Schema.Properties(context), @@ -496,8 +494,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0007", + table = TableIdentifier("lala_0007", Some("default")), format = Some("textfile"), rowFormat = Some("org.apache.hadoop.hive.serde2.OpenCSVSerde"), serdeProperties = Map( @@ -545,8 +542,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0008", + table = TableIdentifier("lala_0008", Some("default")), rowFormat = Some("org.apache.hadoop.hive.serde2.avro.AvroSerDe"), schema = Some(EmbeddedSchema( Schema.Properties(context), @@ -588,8 +584,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val relation = HiveTableRelation( Relation.Properties(context, "t0"), - database = Some("default"), - table = "lala_0009", + table = TableIdentifier("lala_0009", Some("default")), rowFormat = Some("org.apache.hadoop.hive.serde2.avro.AvroSerDe"), inputFormat = Some("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat"), outputFormat = Some("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat"), @@ -1024,8 +1019,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("f3", com.dimajix.flowman.types.StringType) ) )), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Create ================================================================================================ @@ -1111,8 +1105,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes partitions = Seq( PartitionField("part", com.dimajix.flowman.types.StringType) ), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Create ================================================================================================ @@ -1234,8 +1227,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes partitions = Seq( PartitionField("part", com.dimajix.flowman.types.StringType) ), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Create ================================================================================================= @@ -1340,8 +1332,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes partitions = Seq( PartitionField("part", com.dimajix.flowman.types.StringType) ), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Inspect =============================================================================================== @@ -1531,8 +1522,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("f3", com.dimajix.flowman.types.StringType) ) )), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Create ================================================================================================== @@ -1781,8 +1771,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("f3", com.dimajix.flowman.types.IntegerType) ) )), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) val relation_2 = HiveTableRelation( Relation.Properties(context, "rel_2"), @@ -1794,8 +1783,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("f4", com.dimajix.flowman.types.LongType) ) )), - table = "some_table", - database = Some("default") + table = TableIdentifier("some_table", Some("default")) ) // == Create =================================================================== @@ -1804,7 +1792,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes relation_1.conforms(execution, MigrationPolicy.RELAXED) should be (Yes) relation_1.conforms(execution, MigrationPolicy.STRICT) should be (Yes) session.catalog.tableExists(TableIdentifier("some_table", Some("default"))) should be (true) - session.catalog.getTable(relation_1.tableIdentifier).schema should be (StructType(Seq( + session.catalog.getTable(relation_1.table).schema should be (StructType(Seq( StructField("f1", StringType), StructField("f2", IntegerType), StructField("f3", IntegerType) @@ -1816,7 +1804,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes relation_1.migrate(execution, MigrationPolicy.RELAXED, MigrationStrategy.ALTER) relation_1.migrate(execution, MigrationPolicy.RELAXED, MigrationStrategy.ALTER_REPLACE) relation_1.migrate(execution, MigrationPolicy.RELAXED, MigrationStrategy.REPLACE) - session.catalog.getTable(relation_1.tableIdentifier).schema should be (StructType(Seq( + session.catalog.getTable(relation_1.table).schema should be (StructType(Seq( StructField("f1", StringType), StructField("f2", IntegerType), StructField("f3", IntegerType) @@ -1849,7 +1837,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes relation_2.conforms(execution, MigrationPolicy.RELAXED) should be (Yes) relation_2.conforms(execution, MigrationPolicy.STRICT) should be (No) - session.catalog.getTable(relation_2.tableIdentifier).schema should be (StructType(Seq( + session.catalog.getTable(relation_2.table).schema should be (StructType(Seq( StructField("f1", StringType), StructField("f2", IntegerType), StructField("f3", IntegerType), @@ -1864,7 +1852,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes relation_1.conforms(execution, MigrationPolicy.STRICT) should be (No) relation_2.conforms(execution, MigrationPolicy.RELAXED) should be (Yes) relation_2.conforms(execution, MigrationPolicy.STRICT) should be (Yes) - session.catalog.getTable(relation_2.tableIdentifier).schema should be (StructType(Seq( + session.catalog.getTable(relation_2.table).schema should be (StructType(Seq( StructField("f1", StringType), StructField("f2", ShortType), StructField("f4", LongType) @@ -2037,8 +2025,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val view = HiveViewRelation( Relation.Properties(context), - database = Some("default"), - table = "table_or_view", + table = TableIdentifier("table_or_view", Some("default")), mapping = Some(MappingOutputIdentifier("t0")) ) val table = HiveTableRelation( @@ -2051,8 +2038,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes Field("f3", com.dimajix.flowman.types.IntegerType) ) )), - table = "table_or_view", - database = Some("default") + table = TableIdentifier("table_or_view", Some("default")) ) // == Create VIEW ============================================================================================ diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala index 3a0934537..d41080c6a 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -442,8 +442,8 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa Field("f3", com.dimajix.flowman.types.StringType) ) )), - tablePrefix = "zz_", - view = "some_union_table_122" + tablePrefix = TableIdentifier("zz_"), + view = TableIdentifier("some_union_table_122") ) // == Create ================================================================================================ @@ -529,8 +529,8 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa partitions = Seq( PartitionField("part", com.dimajix.flowman.types.StringType) ), - tablePrefix = "zz_", - view = "some_union_table_123" + tablePrefix = TableIdentifier("zz_"), + view = TableIdentifier("some_union_table_123") ) // == Create ================================================================================================ diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala index 80ad1b782..dad05ad13 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,11 +72,8 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess val relation = HiveViewRelation( Relation.Properties(context), - Some("default"), - "v0", - Seq(), - None, - Some(MappingOutputIdentifier("t0")) + table = TableIdentifier("v0", Some("default")), + mapping = Some(MappingOutputIdentifier("t0")) ) relation.provides should be (Set(ResourceIdentifier.ofHiveTable("v0", Some("default")))) @@ -160,11 +157,8 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess val relation = HiveViewRelation( Relation.Properties(context), - Some("default"), - "v0", - Seq(), - None, - Some(MappingOutputIdentifier("union")) + table = TableIdentifier("v0", Some("default")), + mapping = Some(MappingOutputIdentifier("union")) ) relation.provides should be (Set(ResourceIdentifier.ofHiveTable("v0", Some("default")))) @@ -235,8 +229,7 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess val view = HiveViewRelation( Relation.Properties(context), - database = Some("default"), - table = "table_or_view", + table = TableIdentifier("table_or_view", Some("default")), mapping = Some(MappingOutputIdentifier("t0")) ) val table = HiveTableRelation( @@ -249,8 +242,7 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess Field("f3", com.dimajix.flowman.types.IntegerType) ) )), - table = "table_or_view", - database = Some("default") + table = TableIdentifier("table_or_view", Some("default")) ) // == Create TABLE ============================================================================================ @@ -326,8 +318,7 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess val view = HiveViewRelation( Relation.Properties(context), - database = Some("default"), - table = "view", + table = TableIdentifier("view", Some("default")), sql = Some("SELECT * FROM table") ) val table = HiveTableRelation( @@ -340,8 +331,7 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess Field("f3", com.dimajix.flowman.types.IntegerType) ) )), - table = "table", - database = Some("default") + table = TableIdentifier("table", Some("default")) ) val table2 = HiveTableRelation( Relation.Properties(context, "rel_1"), @@ -353,8 +343,7 @@ class HiveViewRelationTest extends AnyFlatSpec with Matchers with LocalSparkSess Field("f4", com.dimajix.flowman.types.IntegerType) ) )), - table = "table", - database = Some("default") + table =TableIdentifier("table", Some("default")) ) // == Create TABLE ============================================================================================ diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala index 3dc227aac..04c2dc475 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala @@ -40,6 +40,8 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.MigrationFailedException @@ -63,10 +65,12 @@ import com.dimajix.flowman.spec.schema.EmbeddedSchema import com.dimajix.flowman.types.DateType import com.dimajix.flowman.types.DoubleType import com.dimajix.flowman.types.Field +import com.dimajix.flowman.types.FloatType import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.SingleValue import com.dimajix.flowman.types.StringType import com.dimajix.flowman.types.StructType +import com.dimajix.flowman.types.VarcharType import com.dimajix.spark.sql.DataFrameBuilder import com.dimajix.spark.testing.LocalSparkSession @@ -115,6 +119,16 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession | type: string | - name: int_col | type: integer + | - name: float_col + | type: float + | primaryKey: + | - int_col + |indexes: + | - name: idx0 + | columns: [str_col, int_col] + | unique: false + |primaryKey: + | - str_col """.stripMargin val relationSpec = ObjectMapper.parse[RelationSpec](spec).asInstanceOf[JdbcRelationSpec] @@ -127,12 +141,16 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession Schema.Properties(context, name="embedded", kind="inline"), fields = Seq( Field("str_col", StringType), - Field("int_col", IntegerType) - ) + Field("int_col", IntegerType), + Field("float_col", FloatType) + ), + primaryKey = Seq("int_col") ))) relation.connection shouldBe a[ValueConnectionReference] relation.connection.identifier should be (ConnectionIdentifier("some_connection")) relation.connection.name should be ("some_connection") + relation.indexes should be (Seq(TableIndex("idx0", Seq("str_col", "int_col")))) + relation.primaryKey should be (Seq("str_col")) } it should "support the full lifecycle" in { @@ -853,7 +871,7 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession ) )), connection = ConnectionReference(context, ConnectionIdentifier("c0")), - table = Some("lala_004") + table = Some(TableIdentifier("lala_004")) ) val relation_t1 = JdbcRelation( Relation.Properties(context, "t1"), @@ -924,7 +942,7 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession ) )), connection = ConnectionReference(context, ConnectionIdentifier("c0")), - table = Some("lala_005") + table = Some(TableIdentifier("lala_005")) ) val rel1 = JdbcRelation( Relation.Properties(context, "t1"), @@ -936,7 +954,7 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession ) )), connection = ConnectionReference(context, ConnectionIdentifier("c0")), - table = Some("lala_005") + table = Some(TableIdentifier("lala_005")) ) // == Create ================================================================================================= @@ -1004,6 +1022,132 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession rel1.conforms(execution, MigrationPolicy.STRICT) should be (No) } + it should "support a primary key" in { + val db = tempDir.toPath.resolve("mydb") + val url = "jdbc:derby:" + db + ";create=true" + val driver = "org.apache.derby.jdbc.EmbeddedDriver" + + val spec = + s""" + |connections: + | c0: + | kind: jdbc + | driver: $driver + | url: $url + |""".stripMargin + val project = Module.read.string(spec).toProject("project") + + val session = Session.builder().withSparkSession(spark).build() + val execution = session.execution + val context = session.getContext(project) + + val rel0 = JdbcRelation( + Relation.Properties(context, "t0"), + schema = Some(EmbeddedSchema( + Schema.Properties(context), + fields = Seq( + Field("str_col", StringType), + Field("int_col", IntegerType), + Field("varchar_col", VarcharType(32)) + ) + )), + connection = ConnectionReference(context, ConnectionIdentifier("c0")), + table = Some(TableIdentifier("lala_005")), + primaryKey = Seq("int_col", "varchar_col") + ) + + // == Create ================================================================================================== + rel0.exists(execution) should be (No) + rel0.conforms(execution, MigrationPolicy.RELAXED) should be (No) + rel0.conforms(execution, MigrationPolicy.STRICT) should be (No) + rel0.create(execution) + rel0.exists(execution) should be (Yes) + rel0.conforms(execution, MigrationPolicy.RELAXED) should be (Yes) + rel0.conforms(execution, MigrationPolicy.STRICT) should be (Yes) + + // == Inspect ================================================================================================= + withConnection(url, "lala_005") { (con, options) => + JdbcUtils.getTable(con, TableIdentifier("lala_005"), options) + } should be ( + TableDefinition( + TableIdentifier("lala_005"), + columns = Seq( + Field("str_col", StringType), + Field("int_col", IntegerType, nullable=false), + Field("varchar_col", VarcharType(32), nullable=false) + ), + primaryKey = Seq("int_col", "varchar_col") + )) + + // == Destroy ================================================================================================= + rel0.exists(execution) should be (Yes) + rel0.destroy(execution) + rel0.exists(execution) should be (No) + } + + it should "support indexes" in { + val db = tempDir.toPath.resolve("mydb") + val url = "jdbc:derby:" + db + ";create=true" + val driver = "org.apache.derby.jdbc.EmbeddedDriver" + + val spec = + s""" + |connections: + | c0: + | kind: jdbc + | driver: $driver + | url: $url + |""".stripMargin + val project = Module.read.string(spec).toProject("project") + + val session = Session.builder().withSparkSession(spark).build() + val execution = session.execution + val context = session.getContext(project) + + val rel0 = JdbcRelation( + Relation.Properties(context, "t0"), + schema = Some(EmbeddedSchema( + Schema.Properties(context), + fields = Seq( + Field("str_col", StringType), + Field("int_col", IntegerType), + Field("varchar_col", VarcharType(32)) + ) + )), + connection = ConnectionReference(context, ConnectionIdentifier("c0")), + table = Some(TableIdentifier("lala_005")), + indexes = Seq(TableIndex("idx0",Seq("int_col", "varchar_col"))) + ) + + // == Create ================================================================================================== + rel0.exists(execution) should be (No) + rel0.conforms(execution, MigrationPolicy.RELAXED) should be (No) + rel0.conforms(execution, MigrationPolicy.STRICT) should be (No) + rel0.create(execution) + rel0.exists(execution) should be (Yes) + rel0.conforms(execution, MigrationPolicy.RELAXED) should be (Yes) + rel0.conforms(execution, MigrationPolicy.STRICT) should be (Yes) + + // == Inspect ================================================================================================= + withConnection(url, "lala_005") { (con, options) => + JdbcUtils.getTable(con, TableIdentifier("lala_005"), options) + } should be ( + TableDefinition( + TableIdentifier("lala_005"), + columns = Seq( + Field("str_col", StringType), + Field("int_col", IntegerType), + Field("varchar_col", VarcharType(32)) + ), + indexes = Seq(TableIndex("idx0",Seq("int_col", "varchar_col"))) + )) + + // == Destroy ================================================================================================= + rel0.exists(execution) should be (Yes) + rel0.destroy(execution) + rel0.exists(execution) should be (No) + } + private def withConnection[T](url:String, table:String)(fn:(Connection,JDBCOptions) => T) : T = { val props = Map( JDBCOptions.JDBC_URL -> url, From 6cc3b719e40f58ba1babfb25098ad910078fd6c5 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Sun, 27 Feb 2022 13:10:02 +0100 Subject: [PATCH 86/95] Rename 'tests' to 'checks' in documentation --- .../{data-qualioty.md => data-quality.md} | 27 ++++ docs/documenting/{tests.md => checks.md} | 28 ++-- docs/documenting/config.md | 11 +- examples/weather/documentation.yml | 4 +- examples/weather/mapping/aggregates.yml | 10 +- examples/weather/model/aggregates.yml | 12 +- examples/weather/model/measurements.yml | 20 +-- examples/weather/model/stations.yml | 2 +- ...om.dimajix.flowman.spi.ColumnCheckExecutor | 1 + ...com.dimajix.flowman.spi.ColumnTestExecutor | 1 - ...om.dimajix.flowman.spi.SchemaCheckExecutor | 1 + ...com.dimajix.flowman.spi.SchemaTestExecutor | 1 - ...stCollector.scala => CheckCollector.scala} | 6 +- ...TestExecutor.scala => CheckExecutor.scala} | 54 ++++---- .../{TestResult.scala => CheckResult.scala} | 26 ++-- .../{ColumnTest.scala => ColumnCheck.scala} | 130 +++++++++--------- .../flowman/documentation/ColumnDoc.scala | 8 +- .../flowman/documentation/Documenter.scala | 2 +- .../{SchemaTest.scala => SchemaCheck.scala} | 82 +++++------ .../flowman/documentation/SchemaDoc.scala | 10 +- .../flowman/documentation/velocity.scala | 40 +++--- ...ecutor.scala => ColumnCheckExecutor.scala} | 20 +-- ...ecutor.scala => SchemaCheckExecutor.scala} | 14 +- ...mnTestTest.scala => ColumnCheckTest.scala} | 102 +++++++------- ...maTestTest.scala => SchemaCheckTest.scala} | 44 +++--- ...lumnTestType.java => ColumnCheckType.java} | 8 +- ...hemaTestType.java => SchemaCheckType.java} | 6 +- ...dimajix.flowman.spi.ClassAnnotationHandler | 4 +- .../flowman/documentation/html/project.vtl | 36 ++--- .../dimajix/flowman/spec/ObjectMapper.scala | 8 +- .../spec/documentation/CollectorSpec.scala | 10 +- ...mnTestSpec.scala => ColumnCheckSpec.scala} | 64 ++++----- .../spec/documentation/ColumnDocSpec.scala | 6 +- .../spec/documentation/MappingDocSpec.scala | 12 +- .../spec/documentation/RelationDocSpec.scala | 8 +- ...maTestSpec.scala => SchemaCheckSpec.scala} | 40 +++--- .../spec/documentation/SchemaDocSpec.scala | 6 +- ...mnTestTest.scala => ColumnCheckTest.scala} | 42 +++--- .../spec/documentation/DocumenterTest.scala | 2 +- .../spec/documentation/MappingDocTest.scala | 14 +- .../spec/documentation/RelationDocTest.scala | 12 +- .../spec/target/DocumentTargetTest.scala | 4 +- 42 files changed, 489 insertions(+), 449 deletions(-) rename docs/cookbook/{data-qualioty.md => data-quality.md} (55%) rename docs/documenting/{tests.md => checks.md} (84%) create mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnCheckExecutor delete mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor create mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaCheckExecutor delete mode 100644 flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor rename flowman-core/src/main/scala/com/dimajix/flowman/documentation/{TestCollector.scala => CheckCollector.scala} (92%) rename flowman-core/src/main/scala/com/dimajix/flowman/documentation/{TestExecutor.scala => CheckExecutor.scala} (73%) rename flowman-core/src/main/scala/com/dimajix/flowman/documentation/{TestResult.scala => CheckResult.scala} (74%) rename flowman-core/src/main/scala/com/dimajix/flowman/documentation/{ColumnTest.scala => ColumnCheck.scala} (58%) rename flowman-core/src/main/scala/com/dimajix/flowman/documentation/{SchemaTest.scala => SchemaCheck.scala} (63%) rename flowman-core/src/main/scala/com/dimajix/flowman/spi/{ColumnTestExecutor.scala => ColumnCheckExecutor.scala} (71%) rename flowman-core/src/main/scala/com/dimajix/flowman/spi/{SchemaTestExecutor.scala => SchemaCheckExecutor.scala} (74%) rename flowman-core/src/test/scala/com/dimajix/flowman/documentation/{ColumnTestTest.scala => ColumnCheckTest.scala} (58%) rename flowman-core/src/test/scala/com/dimajix/flowman/documentation/{SchemaTestTest.scala => SchemaCheckTest.scala} (61%) rename flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/{ColumnTestType.java => ColumnCheckType.java} (75%) rename flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/{SchemaTestType.java => SchemaCheckType.java} (83%) rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/{ColumnTestSpec.scala => ColumnCheckSpec.scala} (66%) rename flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/{SchemaTestSpec.scala => SchemaCheckSpec.scala} (71%) rename flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/{ColumnTestTest.scala => ColumnCheckTest.scala} (66%) diff --git a/docs/cookbook/data-qualioty.md b/docs/cookbook/data-quality.md similarity index 55% rename from docs/cookbook/data-qualioty.md rename to docs/cookbook/data-quality.md index 873f9085b..df600145e 100644 --- a/docs/cookbook/data-qualioty.md +++ b/docs/cookbook/data-quality.md @@ -67,3 +67,30 @@ targets: This example will publish two metrics, `record_count` and `column_sum`, which then can be sent to a [metric sink](../spec/metric) configured in the [namespace](../spec/namespace.md). + + +## When to use what +All three approaches are complementary and can be used together. It all depends on what you want to achieve. + +### Checking Pre- and Post-Conditions +If you want to verify that certain pre- or post-conditions in the source or output data are met, then the +[`validate`](../spec/target/validate.md) and [`verify`](../spec/target/verify.md) targets should be used. They +will perform arbitrary tests either before the `CREATE` and `BUILD` phase (in case of the `validate` target) or after +the `BUILD` phase (in case of the `verify` target). In case any of the tests fail, the whole build will fail and not +proceed any processing. This approach can be used to only start the data transformations when input data is clean and +matches your expectations. + +### Continuous Monitoring of Data Quality +If you want to setup some continuous monitoring of your data quality (either input or output or both), then the +[`measure` target](../spec/target/measure.md) is the right choice. It will collect arbitrary numerical metrics from +the data and publish it to a metrics sink like Prometheus. Typically, metric collectors are used in conjunction with +a dashboard (like Grafana), which then can be used to display the whole history of these metrics over time. This way +you can see if data quality improves or gets worse, and many of these tools also allow you to set up alarms when +some threshold is reached. + +### Documenting Expectations with Reality Check +Finally, the whole documentation subsystem is the right tool for specifying your expectations on the data quality and +have these expectations automatically checked with the real data. In combination with continuous monitoring this can +help to better understand what might be going wrong. In contrast to pre/post-condition checking, a failed check in +the documentation will not fail the build - it will simply be marked as failed in the documentation, but that's all +what will happen. diff --git a/docs/documenting/tests.md b/docs/documenting/checks.md similarity index 84% rename from docs/documenting/tests.md rename to docs/documenting/checks.md index 1e1b9ad97..8c21af297 100644 --- a/docs/documenting/tests.md +++ b/docs/documenting/checks.md @@ -1,7 +1,7 @@ -# Testing Model Properties +# Checking Model Properties In addition to provide pure descriptions of model entities, the documentation framework in Flowman also provides -the ability to specify model properties (like unqiue values in a column, not null etc). These properties will not only +the ability to specify model properties (like unique values in a column, not null etc). These properties will not only be part of the documentation, they will also be verified as part of generating the documentation. @@ -27,30 +27,40 @@ relations: columns: - name: year description: "The year of the measurement, used for partitioning the data" - tests: + checks: - kind: notNull - name: usaf - tests: + checks: - kind: notNull - name: wban - tests: + checks: - kind: notNull - name: air_temperature_qual - tests: + checks: - kind: notNull - kind: values values: [0,1,2,3,4,5,6,7,8,9] - name: air_temperature - tests: + checks: - kind: expression expression: "air_temperature >= -100 OR air_temperature_qual <> 1" - kind: expression expression: "air_temperature <= 100 OR air_temperature_qual <> 1" + # Schema tests, which might involve multiple columns + checks: + kind: foreignKey + relation: stations + columns: + - usaf + - wban + references: + - usaf + - wban ``` -## Available Column Tests +## Available Column Checks -Flowman implements a couple of different test cases on a per column basis. +Flowman implements a couple of different check types on a per column basis. ### Not NULL diff --git a/docs/documenting/config.md b/docs/documenting/config.md index 2929702c5..6526b3133 100644 --- a/docs/documenting/config.md +++ b/docs/documenting/config.md @@ -15,8 +15,8 @@ collectors: - kind: mappings # Collect documentation of build targets - kind: targets - # Execute all tests - - kind: tests + # Execute all checks + - kind: checks generators: # Create an output file in the project directory @@ -34,9 +34,10 @@ generators: ## Collectors Flowman uses so called *collectors* which create an internal model of the documentation from the core entities like -relations, mappings and build targets. The default configuration uses the three collectors `relations`, `mappings` -and `targets`, with each of them being responsible for one entity type. If you really do not require documentation -for one of these targets, you may want to simply remove the corresponding collector from that list. +relations, mappings and build targets. The default configuration uses the four collectors `relations`, `mappings`, +`targets` and `checks`, with each of them being responsible for one entity type and the last one will execute all +data quality checks. If you really do not require documentation for one of these targets, you may want to simply +remove the corresponding collector from that list. ## File Generator Fields diff --git a/examples/weather/documentation.yml b/examples/weather/documentation.yml index db6312873..72ec3bf34 100644 --- a/examples/weather/documentation.yml +++ b/examples/weather/documentation.yml @@ -5,8 +5,8 @@ collectors: - kind: mappings # Collect documentation of build targets - kind: targets - # Execute all tests - - kind: tests + # Execute all checks + - kind: checks generators: # Create an output file in the project directory diff --git a/examples/weather/mapping/aggregates.yml b/examples/weather/mapping/aggregates.yml index 5b0f482c8..cbefd0ddd 100644 --- a/examples/weather/mapping/aggregates.yml +++ b/examples/weather/mapping/aggregates.yml @@ -17,24 +17,24 @@ mappings: description: "This mapping calculates the aggregated metrics per year and per country" columns: - name: country - tests: + checks: - kind: notNull - kind: unique - name: min_wind_speed description: Minimum wind speed - tests: + checks: - kind: expression expression: "min_wind_speed >= 0" - name: max_wind_speed description: Maximum wind speed - tests: + checks: - kind: expression expression: "max_wind_speed <= 60" - name: min_temperature - tests: + checks: - kind: expression expression: "min_temperature >= -100" - name: max_temperature - tests: + checks: - kind: expression expression: "max_temperature <= 100" diff --git a/examples/weather/model/aggregates.yml b/examples/weather/model/aggregates.yml index 6740ce84d..bdd95f36e 100644 --- a/examples/weather/model/aggregates.yml +++ b/examples/weather/model/aggregates.yml @@ -38,24 +38,24 @@ relations: description: "The aggregate table contains min/max temperature value per year and country" columns: - name: country - tests: + checks: - kind: notNull - name: year - tests: + checks: - kind: notNull - name: min_wind_speed - tests: + checks: - kind: expression expression: "min_wind_speed >= 0" - name: min_temperature - tests: + checks: - kind: expression expression: "min_temperature >= -100" - name: max_temperature - tests: + checks: - kind: expression expression: "max_temperature <= 100" - tests: + checks: kind: primaryKey columns: - country diff --git a/examples/weather/model/measurements.yml b/examples/weather/model/measurements.yml index c54364ae4..369af728b 100644 --- a/examples/weather/model/measurements.yml +++ b/examples/weather/model/measurements.yml @@ -23,38 +23,38 @@ relations: columns: - name: year description: "The year of the measurement, used for partitioning the data" - tests: + checks: - kind: notNull - kind: range lower: 1901 upper: 2022 - name: usaf - tests: + checks: - kind: notNull - name: wban - tests: + checks: - kind: notNull - name: date - tests: + checks: - kind: notNull - name: time - tests: + checks: - kind: notNull - name: wind_direction_qual - tests: + checks: - kind: notNull - name: wind_direction - tests: + checks: - kind: notNull - kind: expression expression: "(wind_direction >= 0 AND wind_direction <= 360) OR wind_direction_qual <> 1" - name: air_temperature_qual - tests: + checks: - kind: notNull - kind: values values: [0,1,2,3,4,5,6,7,8,9] - # Schema Tests, which might involve multiple columns - tests: + # Schema tests, which might involve multiple columns + checks: kind: foreignKey relation: stations columns: diff --git a/examples/weather/model/stations.yml b/examples/weather/model/stations.yml index d87b730d1..830ef88d2 100644 --- a/examples/weather/model/stations.yml +++ b/examples/weather/model/stations.yml @@ -9,7 +9,7 @@ relations: file: "${project.basedir}/schema/stations.avsc" documentation: - tests: + checks: kind: primaryKey columns: - usaf diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnCheckExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnCheckExecutor new file mode 100644 index 000000000..6b3e56004 --- /dev/null +++ b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnCheckExecutor @@ -0,0 +1 @@ +com.dimajix.flowman.documentation.DefaultColumnCheckExecutor diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor deleted file mode 100644 index da5f5a7af..000000000 --- a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ColumnTestExecutor +++ /dev/null @@ -1 +0,0 @@ -com.dimajix.flowman.documentation.DefaultColumnTestExecutor diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaCheckExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaCheckExecutor new file mode 100644 index 000000000..85fb4a1a8 --- /dev/null +++ b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaCheckExecutor @@ -0,0 +1 @@ +com.dimajix.flowman.documentation.DefaultSchemaCheckExecutor diff --git a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor b/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor deleted file mode 100644 index 5e1404ad8..000000000 --- a/flowman-core/src/main/resources/META-INF/services/com.dimajix.flowman.spi.SchemaTestExecutor +++ /dev/null @@ -1 +0,0 @@ -com.dimajix.flowman.documentation.DefaultSchemaTestExecutor diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckCollector.scala similarity index 92% rename from flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckCollector.scala index 5006ddf47..86a165090 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestCollector.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckCollector.scala @@ -22,11 +22,11 @@ import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph -class TestCollector extends Collector { +class CheckCollector extends Collector { private val logger = LoggerFactory.getLogger(getClass) /** - * This will execute all tests and change the documentation accordingly + * This will execute all checks and change the documentation accordingly * @param execution * @param graph * @param documentation @@ -34,7 +34,7 @@ class TestCollector extends Collector { */ override def collect(execution: Execution, graph: Graph, documentation: ProjectDoc): ProjectDoc = { val resolver = new ReferenceResolver(graph) - val executor = new TestExecutor(execution) + val executor = new CheckExecutor(execution) val mappings = documentation.mappings.map { m => resolver.resolve(m.reference) match { case None => diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckExecutor.scala similarity index 73% rename from flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckExecutor.scala index 44570d1d9..62db0ff3e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckExecutor.scala @@ -26,17 +26,17 @@ import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.Relation -import com.dimajix.flowman.spi.ColumnTestExecutor -import com.dimajix.flowman.spi.SchemaTestExecutor +import com.dimajix.flowman.spi.ColumnCheckExecutor +import com.dimajix.flowman.spi.SchemaCheckExecutor -class TestExecutor(execution: Execution) { +class CheckExecutor(execution: Execution) { private val logger = LoggerFactory.getLogger(getClass) - private val columnTestExecutors = ColumnTestExecutor.executors - private val schemaTestExecutors = SchemaTestExecutor.executors + private val columnTestExecutors = ColumnCheckExecutor.executors + private val schemaTestExecutors = SchemaCheckExecutor.executors /** - * Executes all tests for a relation as defined within the documentation + * Executes all checks for a relation as defined within the documentation * @param relation * @param doc * @return @@ -44,13 +44,13 @@ class TestExecutor(execution: Execution) { def executeTests(relation:Relation, doc:RelationDoc) : RelationDoc = { val schemaDoc = doc.schema.map { schema => if (containsTests(schema)) { - logger.info(s"Conducting tests on relation '${relation.identifier}'") + logger.info(s"Conducting checks on relation '${relation.identifier}'") try { val df = relation.read(execution, doc.partitions) runSchemaTests(relation.context, df, schema) } catch { case NonFatal(ex) => - logger.warn(s"Error executing tests for relation '${relation.identifier}': ${reasons(ex)}") + logger.warn(s"Error executing checks for relation '${relation.identifier}': ${reasons(ex)}") failSchemaTests(schema) } } @@ -62,7 +62,7 @@ class TestExecutor(execution: Execution) { } /** - * Executes all tests for a mapping as defined within the documentation + * Executes all checks for a mapping as defined within the documentation * @param relation * @param doc * @return @@ -71,13 +71,13 @@ class TestExecutor(execution: Execution) { val outputs = doc.outputs.map { output => val schema = output.schema.map { schema => if (containsTests(schema)) { - logger.info(s"Conducting tests on mapping '${mapping.identifier}'") + logger.info(s"Conducting checks on mapping '${mapping.identifier}'") try { val df = execution.instantiate(mapping, output.name) runSchemaTests(mapping.context, df, schema) } catch { case NonFatal(ex) => - logger.warn(s"Error executing tests for mapping '${mapping.identifier}': ${reasons(ex)}") + logger.warn(s"Error executing checks for mapping '${mapping.identifier}': ${reasons(ex)}") failSchemaTests(schema) } } @@ -91,35 +91,35 @@ class TestExecutor(execution: Execution) { } private def containsTests(doc:SchemaDoc) : Boolean = { - doc.tests.nonEmpty || containsTests(doc.columns) + doc.checks.nonEmpty || containsTests(doc.columns) } private def containsTests(docs:Seq[ColumnDoc]) : Boolean = { - docs.exists(col => col.tests.nonEmpty || containsTests(col.children)) + docs.exists(col => col.checks.nonEmpty || containsTests(col.children)) } private def failSchemaTests(schema:SchemaDoc) : SchemaDoc = { val columns = failColumnTests(schema.columns) - val tests = schema.tests.map { test => - val result = TestResult(Some(test.reference), status = TestStatus.ERROR) + val tests = schema.checks.map { test => + val result = CheckResult(Some(test.reference), status = CheckStatus.ERROR) test.withResult(result) } - schema.copy(columns=columns, tests=tests) + schema.copy(columns=columns, checks=tests) } private def failColumnTests(columns:Seq[ColumnDoc]) : Seq[ColumnDoc] = { columns.map(col => failColumnTests(col)) } private def failColumnTests(column:ColumnDoc) : ColumnDoc = { - val tests = column.tests.map { test => - val result = TestResult(Some(test.reference), status = TestStatus.ERROR) + val tests = column.checks.map { test => + val result = CheckResult(Some(test.reference), status = CheckStatus.ERROR) test.withResult(result) } val children = failColumnTests(column.children) - column.copy(children=children, tests=tests) + column.copy(children=children, checks=tests) } private def runSchemaTests(context:Context, df:DataFrame, schema:SchemaDoc) : SchemaDoc = { val columns = runColumnTests(context, df, schema.columns) - val tests = schema.tests.map { test => + val tests = schema.checks.map { test => logger.info(s" - Executing schema test '${test.name}'") val result = try { @@ -127,27 +127,27 @@ class TestExecutor(execution: Execution) { result match { case None => logger.warn(s"Could not find appropriate test executor for testing schema") - TestResult(Some(test.reference), status = TestStatus.NOT_RUN) + CheckResult(Some(test.reference), status = CheckStatus.NOT_RUN) case Some(result) => result.reparent(test.reference) } } catch { case NonFatal(ex) => logger.warn(s"Error executing column test: ${reasons(ex)}") - TestResult(Some(test.reference), status = TestStatus.ERROR) + CheckResult(Some(test.reference), status = CheckStatus.ERROR) } test.withResult(result) } - schema.copy(columns=columns, tests=tests) + schema.copy(columns=columns, checks=tests) } private def runColumnTests(context:Context, df:DataFrame, columns:Seq[ColumnDoc], path:String = "") : Seq[ColumnDoc] = { columns.map(col => runColumnTests(context, df, col, path)) } private def runColumnTests(context:Context, df:DataFrame, column:ColumnDoc, path:String) : ColumnDoc = { val columnPath = path + column.name - val tests = column.tests.map { test => + val tests = column.checks.map { test => logger.info(s" - Executing test '${test.name}' on column ${columnPath}") val result = try { @@ -155,19 +155,19 @@ class TestExecutor(execution: Execution) { result match { case None => logger.warn(s"Could not find appropriate test executor for testing column $columnPath") - TestResult(Some(test.reference), status = TestStatus.NOT_RUN) + CheckResult(Some(test.reference), status = CheckStatus.NOT_RUN) case Some(result) => result.reparent(test.reference) } } catch { case NonFatal(ex) => logger.warn(s"Error executing column test: ${reasons(ex)}") - TestResult(Some(test.reference), status = TestStatus.ERROR) + CheckResult(Some(test.reference), status = CheckStatus.ERROR) } test.withResult(result) } val children = runColumnTests(context, df, column.children, path + column.name + ".") - column.copy(children=children, tests=tests) + column.copy(children=children, checks=tests) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckResult.scala similarity index 74% rename from flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckResult.scala index 29babf3ee..721d15725 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/TestResult.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/CheckResult.scala @@ -17,29 +17,29 @@ package com.dimajix.flowman.documentation -sealed abstract class TestStatus extends Product with Serializable { +sealed abstract class CheckStatus extends Product with Serializable { def success : Boolean def failure : Boolean def run : Boolean } -object TestStatus { - final case object FAILED extends TestStatus { +object CheckStatus { + final case object FAILED extends CheckStatus { def success : Boolean = false def failure : Boolean = true def run : Boolean = true } - final case object SUCCESS extends TestStatus { + final case object SUCCESS extends CheckStatus { def success : Boolean = true def failure : Boolean = false def run : Boolean = true } - final case object ERROR extends TestStatus { + final case object ERROR extends CheckStatus { def success : Boolean = false def failure : Boolean = true def run : Boolean = true } - final case object NOT_RUN extends TestStatus { + final case object NOT_RUN extends CheckStatus { def success : Boolean = false def failure : Boolean = false def run : Boolean = false @@ -47,7 +47,7 @@ object TestStatus { } -final case class TestResultReference( +final case class CheckResultReference( parent:Option[Reference] ) extends Reference { override def toString: String = { @@ -56,21 +56,21 @@ final case class TestResultReference( case None => "" } } - override def kind : String = "test_result" + override def kind : String = "check_result" } -final case class TestResult( +final case class CheckResult( parent:Some[Reference], - status:TestStatus, + status:CheckStatus, description:Option[String] = None, details:Option[Fragment] = None ) extends Fragment { - override def reference: TestResultReference = TestResultReference(parent) + override def reference: CheckResultReference = CheckResultReference(parent) override def fragments: Seq[Fragment] = details.toSeq - override def reparent(parent:Reference) : TestResult = { - val ref = TestResultReference(Some(parent)) + override def reparent(parent:Reference) : CheckResult = { + val ref = CheckResultReference(Some(parent)) copy( parent = Some(parent), details = details.map(_.reparent(ref)) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnCheck.scala similarity index 58% rename from flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnCheck.scala index 441e1870a..a69240b1f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnCheck.scala @@ -26,155 +26,155 @@ import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.RelationIdentifier -import com.dimajix.flowman.spi.ColumnTestExecutor +import com.dimajix.flowman.spi.ColumnCheckExecutor -final case class ColumnTestReference( +final case class ColumnCheckReference( override val parent:Option[Reference] ) extends Reference { override def toString: String = { parent match { - case Some(ref) => ref.toString + "/test" + case Some(ref) => ref.toString + "/check" case None => "" } } - override def kind : String = "column_test" + override def kind : String = "column_check" } -abstract class ColumnTest extends Fragment with Product with Serializable { +abstract class ColumnCheck extends Fragment with Product with Serializable { def name : String - def result : Option[TestResult] - def withResult(result:TestResult) : ColumnTest + def result : Option[CheckResult] + def withResult(result:CheckResult) : ColumnCheck - override def reparent(parent: Reference): ColumnTest + override def reparent(parent: Reference): ColumnCheck override def parent: Option[Reference] - override def reference: ColumnTestReference = ColumnTestReference(parent) + override def reference: ColumnCheckReference = ColumnCheckReference(parent) override def fragments: Seq[Fragment] = result.toSeq } -final case class NotNullColumnTest( +final case class NotNullColumnCheck( parent:Option[Reference], description: Option[String] = None, - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name : String = "IS NOT NULL" - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class UniqueColumnTest( +final case class UniqueColumnCheck( parent:Option[Reference], description: Option[String] = None, - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name : String = "HAS UNIQUE VALUES" - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): UniqueColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): UniqueColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class RangeColumnTest( +final case class RangeColumnCheck( parent:Option[Reference], description: Option[String] = None, lower:Any, upper:Any, - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name : String = s"IS BETWEEN $lower AND $upper" - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): RangeColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): RangeColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class ValuesColumnTest( +final case class ValuesColumnCheck( parent:Option[Reference], description: Option[String] = None, values: Seq[Any] = Seq(), - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name : String = s"IS IN (${values.mkString(",")})" - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ValuesColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ValuesColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class ForeignKeyColumnTest( +final case class ForeignKeyColumnCheck( parent:Option[Reference], description: Option[String] = None, relation: Option[RelationIdentifier] = None, mapping: Option[MappingOutputIdentifier] = None, column: Option[String] = None, - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name : String = { val otherEntity = relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("") val otherColumn = column.getOrElse("") s"FOREIGN KEY REFERENCES ${otherEntity} (${otherColumn})" } - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ForeignKeyColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ForeignKeyColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class ExpressionColumnTest( +final case class ExpressionColumnCheck( parent:Option[Reference], description: Option[String] = None, expression: String, - result:Option[TestResult] = None -) extends ColumnTest { + result:Option[CheckResult] = None +) extends ColumnCheck { override def name: String = expression - override def withResult(result: TestResult): ColumnTest = copy(result=Some(result)) - override def reparent(parent: Reference): ExpressionColumnTest = { - val ref = ColumnTestReference(Some(parent)) + override def withResult(result: CheckResult): ColumnCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ExpressionColumnCheck = { + val ref = ColumnCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -class DefaultColumnTestExecutor extends ColumnTestExecutor { - override def execute(execution: Execution, context:Context, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] = { - test match { - case _: NotNullColumnTest => - executePredicateTest(df, test, df(column).isNotNull) +class DefaultColumnCheckExecutor extends ColumnCheckExecutor { + override def execute(execution: Execution, context:Context, df: DataFrame, column:String, check: ColumnCheck): Option[CheckResult] = { + check match { + case _: NotNullColumnCheck => + executePredicateTest(df, check, df(column).isNotNull) - case _: UniqueColumnTest => + case _: UniqueColumnCheck => val agg = df.filter(df(column).isNotNull).groupBy(df(column)).count() val result = agg.groupBy(agg(agg.columns(1)) > 1).count().collect() val numSuccess = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) - val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val status = if (numFailed > 0) CheckStatus.FAILED else CheckStatus.SUCCESS val description = s"$numSuccess values are unique, $numFailed values are non-unique" - Some(TestResult(Some(test.reference), status, Some(description))) + Some(CheckResult(Some(check.reference), status, Some(description))) - case v: ValuesColumnTest => + case v: ValuesColumnCheck => val dt = df.schema(column).dataType val values = v.values.map(v => lit(v).cast(dt)) - executePredicateTest(df.filter(df(column).isNotNull), test, df(column).isin(values:_*)) + executePredicateTest(df.filter(df(column).isNotNull), check, df(column).isin(values:_*)) - case v: RangeColumnTest => + case v: RangeColumnCheck => val dt = df.schema(column).dataType val lower = lit(v.lower).cast(dt) val upper = lit(v.upper).cast(dt) - executePredicateTest(df.filter(df(column).isNotNull), test, df(column).between(lower, upper)) + executePredicateTest(df.filter(df(column).isNotNull), check, df(column).between(lower, upper)) - case v: ExpressionColumnTest => - executePredicateTest(df, test, expr(v.expression).cast(BooleanType)) + case v: ExpressionColumnCheck => + executePredicateTest(df, check, expr(v.expression).cast(BooleanType)) - case f:ForeignKeyColumnTest => + case f:ForeignKeyColumnCheck => val otherDf = f.relation.map { rel => val relation = context.getRelation(rel) @@ -182,21 +182,21 @@ class DefaultColumnTestExecutor extends ColumnTestExecutor { }.orElse(f.mapping.map { map=> val mapping = context.getMapping(map.mapping) execution.instantiate(mapping, map.output) - }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test of column '$column' in test ${test.reference.toString}")) + }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test of column '$column' in check ${check.reference.toString}")) val otherColumn = f.column.getOrElse(column) val joined = df.join(otherDf, df(column) === otherDf(otherColumn), "left") - executePredicateTest(joined.filter(df(column).isNotNull), test,otherDf(otherColumn).isNotNull) + executePredicateTest(joined.filter(df(column).isNotNull), check,otherDf(otherColumn).isNotNull) case _ => None } } - private def executePredicateTest(df: DataFrame, test:ColumnTest, predicate:Column) : Option[TestResult] = { + private def executePredicateTest(df: DataFrame, test:ColumnCheck, predicate:Column) : Option[CheckResult] = { val result = df.groupBy(predicate).count().collect() val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) - val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val status = if (numFailed > 0) CheckStatus.FAILED else CheckStatus.SUCCESS val description = s"$numSuccess records passed, $numFailed records failed" - Some(TestResult(Some(test.reference), status, Some(description))) + Some(CheckResult(Some(test.reference), status, Some(description))) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala index 293bee278..bfb0ee7aa 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/ColumnDoc.scala @@ -59,7 +59,7 @@ final case class ColumnDoc( parent:Option[Reference], field:Field, children:Seq[ColumnDoc] = Seq(), - tests:Seq[ColumnTest] = Seq() + checks:Seq[ColumnCheck] = Seq() ) extends EntityDoc { override def reference: ColumnReference = ColumnReference(parent, name) override def fragments: Seq[Fragment] = children @@ -68,7 +68,7 @@ final case class ColumnDoc( copy( parent = Some(parent), children = children.map(_.reparent(ref)), - tests = tests.map(_.reparent(ref)) + checks = checks.map(_.reparent(ref)) ) } @@ -101,11 +101,11 @@ final case class ColumnDoc( else this.children ++ other.children val desc = other.description.orElse(description) - val tsts = tests ++ other.tests + val tsts = checks ++ other.checks val ftyp = if (field.ftype == NullType) other.field.ftype else field.ftype val nll = if (field.ftype == NullType) other.field.nullable else field.nullable val fld = field.copy(ftype=ftyp, nullable=nll, description=desc) - copy(field=fld, children=childs, tests=tsts) + copy(field=fld, children=childs, checks=tsts) } /** diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala index 57779e3b8..9edc26527 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/Documenter.scala @@ -42,7 +42,7 @@ object Documenter { new RelationCollector(), new MappingCollector(), new TargetCollector(), - new TestCollector() + new CheckCollector() ) Documenter( collectors=collectors diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaCheck.scala similarity index 63% rename from flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaCheck.scala index 89d0af15b..225921d66 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaTest.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaCheck.scala @@ -25,97 +25,97 @@ import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.RelationIdentifier -import com.dimajix.flowman.spi.SchemaTestExecutor +import com.dimajix.flowman.spi.SchemaCheckExecutor -final case class SchemaTestReference( +final case class SchemaCheckReference( override val parent:Option[Reference] ) extends Reference { override def toString: String = { parent match { - case Some(ref) => ref.toString + "/test" + case Some(ref) => ref.toString + "/check" case None => "" } } - override def kind : String = "schema_test" + override def kind : String = "schema_check" } -abstract class SchemaTest extends Fragment with Product with Serializable { +abstract class SchemaCheck extends Fragment with Product with Serializable { def name : String - def result : Option[TestResult] - def withResult(result:TestResult) : SchemaTest + def result : Option[CheckResult] + def withResult(result:CheckResult) : SchemaCheck override def parent: Option[Reference] - override def reference: SchemaTestReference = SchemaTestReference(parent) + override def reference: SchemaCheckReference = SchemaCheckReference(parent) override def fragments: Seq[Fragment] = result.toSeq - override def reparent(parent: Reference): SchemaTest + override def reparent(parent: Reference): SchemaCheck } -final case class PrimaryKeySchemaTest( +final case class PrimaryKeySchemaCheck( parent:Option[Reference], description: Option[String] = None, columns:Seq[String] = Seq.empty, - result:Option[TestResult] = None -) extends SchemaTest { + result:Option[CheckResult] = None +) extends SchemaCheck { override def name : String = s"PRIMARY KEY (${columns.mkString(",")})" - override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) - override def reparent(parent: Reference): PrimaryKeySchemaTest = { - val ref = SchemaTestReference(Some(parent)) + override def withResult(result: CheckResult): SchemaCheck = copy(result=Some(result)) + override def reparent(parent: Reference): PrimaryKeySchemaCheck = { + val ref = SchemaCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class ForeignKeySchemaTest( +final case class ForeignKeySchemaCheck( parent:Option[Reference], description: Option[String] = None, columns: Seq[String] = Seq.empty, relation: Option[RelationIdentifier] = None, mapping: Option[MappingOutputIdentifier] = None, references: Seq[String] = Seq.empty, - result:Option[TestResult] = None -) extends SchemaTest { + result:Option[CheckResult] = None +) extends SchemaCheck { override def name : String = { val otherEntity = relation.map(_.toString).orElse(mapping.map(_.toString)).getOrElse("") val otherColumns = if (references.isEmpty) columns else references s"FOREIGN KEY (${columns.mkString(",")}) REFERENCES ${otherEntity}(${otherColumns.mkString(",")})" } - override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) - override def reparent(parent: Reference): ForeignKeySchemaTest = { - val ref = SchemaTestReference(Some(parent)) + override def withResult(result: CheckResult): SchemaCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ForeignKeySchemaCheck = { + val ref = SchemaCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -final case class ExpressionSchemaTest( +final case class ExpressionSchemaCheck( parent:Option[Reference], description: Option[String] = None, expression: String, - result:Option[TestResult] = None -) extends SchemaTest { + result:Option[CheckResult] = None +) extends SchemaCheck { override def name: String = expression - override def withResult(result: TestResult): SchemaTest = copy(result=Some(result)) - override def reparent(parent: Reference): ExpressionSchemaTest = { - val ref = SchemaTestReference(Some(parent)) + override def withResult(result: CheckResult): SchemaCheck = copy(result=Some(result)) + override def reparent(parent: Reference): ExpressionSchemaCheck = { + val ref = SchemaCheckReference(Some(parent)) copy(parent=Some(parent), result=result.map(_.reparent(ref))) } } -class DefaultSchemaTestExecutor extends SchemaTestExecutor { - override def execute(execution: Execution, context:Context, df: DataFrame, test: SchemaTest): Option[TestResult] = { - test match { - case p:PrimaryKeySchemaTest => +class DefaultSchemaCheckExecutor extends SchemaCheckExecutor { + override def execute(execution: Execution, context:Context, df: DataFrame, check: SchemaCheck): Option[CheckResult] = { + check match { + case p:PrimaryKeySchemaCheck => val cols = p.columns.map(df(_)) val agg = df.filter(cols.map(_.isNotNull).reduce(_ || _)).groupBy(cols:_*).count() val result = agg.groupBy(agg(agg.columns(cols.length)) > 1).count().collect() val numSuccess = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) - val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val status = if (numFailed > 0) CheckStatus.FAILED else CheckStatus.SUCCESS val description = s"$numSuccess keys are unique, $numFailed keys are non-unique" - Some(TestResult(Some(test.reference), status, Some(description))) + Some(CheckResult(Some(check.reference), status, Some(description))) - case f:ForeignKeySchemaTest => + case f:ForeignKeySchemaCheck => val otherDf = f.relation.map { rel => val relation = context.getRelation(rel) @@ -123,7 +123,7 @@ class DefaultSchemaTestExecutor extends SchemaTestExecutor { }.orElse(f.mapping.map { map=> val mapping = context.getMapping(map.mapping) execution.instantiate(mapping, map.output) - }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test ${test.reference.toString}")) + }).getOrElse(throw new IllegalArgumentException(s"Need either mapping or relation in foreignKey test ${check.reference.toString}")) val cols = f.columns.map(df(_)) val otherCols = if (f.references.nonEmpty) @@ -131,21 +131,21 @@ class DefaultSchemaTestExecutor extends SchemaTestExecutor { else f.columns.map(otherDf(_)) val joined = df.join(otherDf, cols.zip(otherCols).map(lr => lr._1 === lr._2).reduce(_ && _), "left") - executePredicateTest(joined, test, otherCols.map(_.isNotNull).reduce(_ || _)) + executePredicateTest(joined, check, otherCols.map(_.isNotNull).reduce(_ || _)) - case e:ExpressionSchemaTest => - executePredicateTest(df, test, expr(e.expression).cast(BooleanType)) + case e:ExpressionSchemaCheck => + executePredicateTest(df, check, expr(e.expression).cast(BooleanType)) case _ => None } } - private def executePredicateTest(df: DataFrame, test:SchemaTest, predicate:Column) : Option[TestResult] = { + private def executePredicateTest(df: DataFrame, test:SchemaCheck, predicate:Column) : Option[CheckResult] = { val result = df.groupBy(predicate).count().collect() val numSuccess = result.find(_.getBoolean(0) == true).map(_.getLong(1)).getOrElse(0L) val numFailed = result.find(_.getBoolean(0) == false).map(_.getLong(1)).getOrElse(0L) - val status = if (numFailed > 0) TestStatus.FAILED else TestStatus.SUCCESS + val status = if (numFailed > 0) CheckStatus.FAILED else CheckStatus.SUCCESS val description = s"$numSuccess records passed, $numFailed records failed" - Some(TestResult(Some(test.reference), status, Some(description))) + Some(CheckResult(Some(test.reference), status, Some(description))) } } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala index 47eecffd7..60b0eec35 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/SchemaDoc.scala @@ -84,16 +84,16 @@ final case class SchemaDoc( parent:Option[Reference], description:Option[String] = None, columns:Seq[ColumnDoc] = Seq(), - tests:Seq[SchemaTest] = Seq() + checks:Seq[SchemaCheck] = Seq() ) extends EntityDoc { override def reference: SchemaReference = SchemaReference(parent) - override def fragments: Seq[Fragment] = columns ++ tests + override def fragments: Seq[Fragment] = columns ++ checks override def reparent(parent: Reference): SchemaDoc = { val ref = SchemaReference(Some(parent)) copy( parent = Some(parent), columns = columns.map(_.reparent(ref)), - tests = tests.map(_.reparent(ref)) + checks = checks.map(_.reparent(ref)) ) } @@ -118,9 +118,9 @@ final case class SchemaDoc( */ def merge(other:SchemaDoc) : SchemaDoc = { val desc = other.description.orElse(this.description) - val tsts = tests ++ other.tests + val tsts = checks ++ other.checks val cols = ColumnDoc.merge(columns, other.columns) - val result = copy(description=desc, columns=cols, tests=tsts) + val result = copy(description=desc, columns=cols, checks=tsts) parent.orElse(other.parent) .map(result.reparent) .getOrElse(result) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala index ad44d3e57..7dfb71496 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/documentation/velocity.scala @@ -28,6 +28,7 @@ final case class ReferenceWrapper(reference:Reference) { def getKind() : String = reference.kind def getSql() : String = reference match { case m:MappingReference => m.sql + case m:MappingOutputReference => m.sql case m:RelationReference => m.sql case m:ColumnReference => m.sql case m:SchemaReference => m.sql @@ -43,7 +44,7 @@ class FragmentWrapper(fragment:Fragment) { } -final case class TestResultWrapper(result:TestResult) extends FragmentWrapper(result) { +final case class CheckResultWrapper(result:CheckResult) extends FragmentWrapper(result) { override def toString: String = result.status.toString def getStatus() : String = result.status.toString @@ -52,14 +53,14 @@ final case class TestResultWrapper(result:TestResult) extends FragmentWrapper(re } -final case class ColumnTestWrapper(test:ColumnTest) extends FragmentWrapper(test) { - override def toString: String = test.name +final case class ColumnCheckWrapper(check:ColumnCheck) extends FragmentWrapper(check) { + override def toString: String = check.name - def getName() : String = test.name - def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull - def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") - def getSuccess() : Boolean = test.result.exists(_.success) - def getFailure() : Boolean = test.result.exists(_.failure) + def getName() : String = check.name + def getResult() : CheckResultWrapper = check.result.map(CheckResultWrapper).orNull + def getStatus() : String = check.result.map(_.status.toString).getOrElse("NOT_RUN") + def getSuccess() : Boolean = check.result.exists(_.success) + def getFailure() : Boolean = check.result.exists(_.failure) } @@ -73,24 +74,24 @@ final case class ColumnDocWrapper(column:ColumnDoc) extends FragmentWrapper(colu def getSparkType() : String = column.sparkType def getCatalogType() : String = column.catalogType def getColumns() : java.util.List[ColumnDocWrapper] = column.children.map(ColumnDocWrapper).asJava - def getTests() : java.util.List[ColumnTestWrapper] = column.tests.map(ColumnTestWrapper).asJava + def getChecks() : java.util.List[ColumnCheckWrapper] = column.checks.map(ColumnCheckWrapper).asJava } -final case class SchemaTestWrapper(test:SchemaTest) extends FragmentWrapper(test) { - override def toString: String = test.name +final case class SchemaCheckWrapper(check:SchemaCheck) extends FragmentWrapper(check) { + override def toString: String = check.name - def getName() : String = test.name - def getResult() : TestResultWrapper = test.result.map(TestResultWrapper).orNull - def getStatus() : String = test.result.map(_.status.toString).getOrElse("NOT_RUN") - def getSuccess() : Boolean = test.result.exists(_.success) - def getFailure() : Boolean = test.result.exists(_.failure) + def getName() : String = check.name + def getResult() : CheckResultWrapper = check.result.map(CheckResultWrapper).orNull + def getStatus() : String = check.result.map(_.status.toString).getOrElse("NOT_RUN") + def getSuccess() : Boolean = check.result.exists(_.success) + def getFailure() : Boolean = check.result.exists(_.failure) } final case class SchemaDocWrapper(schema:SchemaDoc) extends FragmentWrapper(schema) { def getColumns() : java.util.List[ColumnDocWrapper] = schema.columns.map(ColumnDocWrapper).asJava - def getTests() : java.util.List[SchemaTestWrapper] = schema.tests.map(SchemaTestWrapper).asJava + def getChecks() : java.util.List[SchemaCheckWrapper] = schema.checks.map(SchemaCheckWrapper).asJava } @@ -165,9 +166,10 @@ final case class ProjectDocWrapper(project:ProjectDoc) extends FragmentWrapper(p case t:TargetDoc => TargetDocWrapper(t) case p:TargetPhaseDoc => TargetPhaseDocWrapper(p) case s:SchemaDoc => SchemaDocWrapper(s) - case t:TestResult => TestResultWrapper(t) + case s:SchemaCheck => SchemaCheckWrapper(s) + case t:CheckResult => CheckResultWrapper(t) case c:ColumnDoc => ColumnDocWrapper(c) - case t:ColumnTest => ColumnTestWrapper(t) + case t:ColumnCheck => ColumnCheckWrapper(t) case f:Fragment => new FragmentWrapper(f) }.orNull } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnCheckExecutor.scala similarity index 71% rename from flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnCheckExecutor.scala index f4dd910be..61c0e0da3 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnTestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/ColumnCheckExecutor.scala @@ -22,29 +22,29 @@ import scala.collection.JavaConverters._ import org.apache.spark.sql.DataFrame -import com.dimajix.flowman.documentation.ColumnTest -import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.documentation.ColumnCheck +import com.dimajix.flowman.documentation.CheckResult import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.graph.Graph -object ColumnTestExecutor { - def executors : Seq[ColumnTestExecutor] = { - val loader = ServiceLoader.load(classOf[ColumnTestExecutor]) +object ColumnCheckExecutor { + def executors : Seq[ColumnCheckExecutor] = { + val loader = ServiceLoader.load(classOf[ColumnCheckExecutor]) loader.iterator().asScala.toSeq } } -trait ColumnTestExecutor { +trait ColumnCheckExecutor { /** - * Executes a column test + * Executes a column check * @param execution - execution to use * @param context - context that can be used for resource lookups like relations or mappings - * @param df - DataFrame containing the output to test - * @param column - Path of the column to test + * @param df - DataFrame containing the output to check + * @param column - Path of the column to check * @param test - Test to execute * @return */ - def execute(execution: Execution, context:Context, df: DataFrame, column:String, test: ColumnTest): Option[TestResult] + def execute(execution: Execution, context:Context, df: DataFrame, column:String, test: ColumnCheck): Option[CheckResult] } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaCheckExecutor.scala similarity index 74% rename from flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala rename to flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaCheckExecutor.scala index 255e3900b..f8535d869 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaTestExecutor.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/spi/SchemaCheckExecutor.scala @@ -22,19 +22,19 @@ import scala.collection.JavaConverters._ import org.apache.spark.sql.DataFrame -import com.dimajix.flowman.documentation.SchemaTest -import com.dimajix.flowman.documentation.TestResult +import com.dimajix.flowman.documentation.SchemaCheck +import com.dimajix.flowman.documentation.CheckResult import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution -object SchemaTestExecutor { - def executors : Seq[SchemaTestExecutor] = { - val loader = ServiceLoader.load(classOf[SchemaTestExecutor]) +object SchemaCheckExecutor { + def executors : Seq[SchemaCheckExecutor] = { + val loader = ServiceLoader.load(classOf[SchemaCheckExecutor]) loader.iterator().asScala.toSeq } } -trait SchemaTestExecutor { - def execute(execution: Execution, context:Context, df:DataFrame, test:SchemaTest) : Option[TestResult] +trait SchemaCheckExecutor { + def execute(execution: Execution, context:Context, df:DataFrame, test:SchemaCheck) : Option[CheckResult] } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnCheckTest.scala similarity index 58% rename from flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala rename to flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnCheckTest.scala index 37a982b14..56484affa 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/ColumnCheckTest.scala @@ -30,32 +30,32 @@ import com.dimajix.flowman.model.Prototype import com.dimajix.spark.testing.LocalSparkSession -class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { - "A NotNullColumnTest" should "be executable" in { +class ColumnCheckTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { + "A NotNullColumnCheck" should "be executable" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq((Some(1),2), (None,3))) - val test = NotNullColumnTest(None) + val test = NotNullColumnCheck(None) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_3", test)) } - "A UniqueColumnTest" should "be executable" in { + "A UniqueColumnCheck" should "be executable" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,3), @@ -63,36 +63,36 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc (None,3,5) )) - val test = UniqueColumnTest(None) + val test = UniqueColumnCheck(None) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 values are unique, 0 values are non-unique")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 values are unique, 0 values are non-unique")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 values are unique, 1 values are non-unique")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 values are unique, 1 values are non-unique")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("3 values are unique, 0 values are non-unique")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("3 values are unique, 0 values are non-unique")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } - "A ValuesColumnTest" should "be executable" in { + "A ValuesColumnCheck" should "be executable" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = ValuesColumnTest(None, values=Seq(1,2)) + val test = ValuesColumnCheck(None, values=Seq(1,2)) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -102,43 +102,43 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = ValuesColumnTest(None, values=Seq(1,2)) + val test = ValuesColumnCheck(None, values=Seq(1,2)) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } - "A RangeColumnTest" should "be executable" in { + "A RangeColumnCheck" should "be executable" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = RangeColumnTest(None, lower=1, upper=2) + val test = RangeColumnCheck(None, lower=1, upper=2) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } @@ -148,42 +148,42 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = RangeColumnTest(None, lower="1.0", upper="2.2") + val test = RangeColumnCheck(None, lower="1.0", upper="2.2") val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) } - "An ExpressionColumnTest" should "succeed" in { + "An ExpressionColumnCheck" should "succeed" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = ExpressionColumnTest(None, expression="_2 > _3") + val test = ExpressionColumnCheck(None, expression="_2 > _3") val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) val result4 = testExecutor.execute(execution, context, df, "_4", test) - result4 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result4 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) } it should "fail" in { @@ -192,23 +192,23 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test = ExpressionColumnTest(None, expression="_2 < _3") + val test = ExpressionColumnCheck(None, expression="_2 < _3") val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("0 records passed, 2 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("0 records passed, 2 records failed")))) val result4 = testExecutor.execute(execution, context, df, "_4", test) - result4 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + result4 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("0 records passed, 2 records failed")))) } - "A ForeignKeyColumnTest" should "work" in { + "A ForeignKeyColumnCheck" should "work" in { val mappingSpec = mock[Prototype[Mapping]] val mapping = mock[Mapping] @@ -222,7 +222,7 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc val context = session.getContext(project) val execution = session.execution - val testExecutor = new DefaultColumnTestExecutor + val testExecutor = new DefaultColumnCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),1,1), @@ -243,13 +243,13 @@ class ColumnTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc (mapping.identifier _).expects().returns(MappingIdentifier("project/mapping")) (mapping.execute _).expects(*,*).returns(Map("main" -> otherDf)) - val test = ForeignKeyColumnTest(None, mapping=Some(MappingOutputIdentifier("mapping")), column=Some("_1")) + val test = ForeignKeyColumnCheck(None, mapping=Some(MappingOutputIdentifier("mapping")), column=Some("_1")) val result1 = testExecutor.execute(execution, context, df, "_1", test) - result1 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("1 records passed, 0 records failed")))) val result2 = testExecutor.execute(execution, context, df, "_2", test) - result2 should be (Some(TestResult(Some(test.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result2 should be (Some(CheckResult(Some(test.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) val result3 = testExecutor.execute(execution, context, df, "_3", test) - result3 should be (Some(TestResult(Some(test.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result3 should be (Some(CheckResult(Some(test.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, "_4", test)) } } diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaCheckTest.scala similarity index 61% rename from flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala rename to flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaCheckTest.scala index 8b6c1b3b2..d9663f438 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaTestTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/documentation/SchemaCheckTest.scala @@ -30,14 +30,14 @@ import com.dimajix.flowman.model.Prototype import com.dimajix.spark.testing.LocalSparkSession -class SchemaTestTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { - "A PrimaryKeySchemaTest" should "be executable" in { +class SchemaCheckTest extends AnyFlatSpec with Matchers with MockFactory with LocalSparkSession { + "A PrimaryKeySchemaCheck" should "be executable" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultSchemaTestExecutor + val testExecutor = new DefaultSchemaCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,3), @@ -45,38 +45,38 @@ class SchemaTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc (None,3,5) )) - val test1 = PrimaryKeySchemaTest(None, columns=Seq("_1","_3")) + val test1 = PrimaryKeySchemaCheck(None, columns=Seq("_1","_3")) val result1 = testExecutor.execute(execution, context, df, test1) - result1 should be (Some(TestResult(Some(test1.reference), TestStatus.SUCCESS, description=Some("3 keys are unique, 0 keys are non-unique")))) + result1 should be (Some(CheckResult(Some(test1.reference), CheckStatus.SUCCESS, description=Some("3 keys are unique, 0 keys are non-unique")))) - val test2 = PrimaryKeySchemaTest(None, columns=Seq("_1","_2")) + val test2 = PrimaryKeySchemaCheck(None, columns=Seq("_1","_2")) val result2 = testExecutor.execute(execution, context, df, test2) - result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 keys are unique, 1 keys are non-unique")))) + result2 should be (Some(CheckResult(Some(test1.reference), CheckStatus.FAILED, description=Some("1 keys are unique, 1 keys are non-unique")))) } - "An ExpressionSchemaTest" should "work" in { + "An ExpressionSchemaCheck" should "work" in { val session = Session.builder() .withSparkSession(spark) .build() val execution = session.execution val context = session.context - val testExecutor = new DefaultSchemaTestExecutor + val testExecutor = new DefaultSchemaCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),2,1), (None,3,2) )) - val test1 = ExpressionSchemaTest(None, expression="_2 > _3") + val test1 = ExpressionSchemaCheck(None, expression="_2 > _3") val result1 = testExecutor.execute(execution, context, df, test1) - result1 should be (Some(TestResult(Some(test1.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result1 should be (Some(CheckResult(Some(test1.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) - val test2 = ExpressionSchemaTest(None, expression="_2 < _3") + val test2 = ExpressionSchemaCheck(None, expression="_2 < _3") val result2 = testExecutor.execute(execution, context, df, test2) - result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("0 records passed, 2 records failed")))) + result2 should be (Some(CheckResult(Some(test1.reference), CheckStatus.FAILED, description=Some("0 records passed, 2 records failed")))) } - "A ForeignKeySchemaTest" should "work" in { + "A ForeignKeySchemaCheck" should "work" in { val mappingSpec = mock[Prototype[Mapping]] val mapping = mock[Mapping] @@ -90,7 +90,7 @@ class SchemaTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc val context = session.getContext(project) val execution = session.execution - val testExecutor = new DefaultSchemaTestExecutor + val testExecutor = new DefaultSchemaCheckExecutor val df = spark.createDataFrame(Seq( (Some(1),1,1), @@ -111,19 +111,19 @@ class SchemaTestTest extends AnyFlatSpec with Matchers with MockFactory with Loc (mapping.identifier _).expects().returns(MappingIdentifier("project/mapping")) (mapping.execute _).expects(*,*).returns(Map("main" -> otherDf)) - val test1 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_1")) + val test1 = ForeignKeySchemaCheck(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_1")) val result1 = testExecutor.execute(execution, context, df, test1) - result1 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result1 should be (Some(CheckResult(Some(test1.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) - val test2 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_3"), references=Seq("_2")) + val test2 = ForeignKeySchemaCheck(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_3"), references=Seq("_2")) val result2 = testExecutor.execute(execution, context, df, test2) - result2 should be (Some(TestResult(Some(test1.reference), TestStatus.FAILED, description=Some("1 records passed, 1 records failed")))) + result2 should be (Some(CheckResult(Some(test1.reference), CheckStatus.FAILED, description=Some("1 records passed, 1 records failed")))) - val test3 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2")) + val test3 = ForeignKeySchemaCheck(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2")) val result3 = testExecutor.execute(execution, context, df, test3) - result3 should be (Some(TestResult(Some(test3.reference), TestStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) + result3 should be (Some(CheckResult(Some(test3.reference), CheckStatus.SUCCESS, description=Some("2 records passed, 0 records failed")))) - val test4 = ForeignKeySchemaTest(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2"), references=Seq("_3")) + val test4 = ForeignKeySchemaCheck(None, mapping=Some(MappingOutputIdentifier("mapping")), columns=Seq("_2"), references=Seq("_3")) an[Exception] should be thrownBy(testExecutor.execute(execution, context, df, test4)) } } diff --git a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnCheckType.java similarity index 75% rename from flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java rename to flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnCheckType.java index e16e73e03..67ae63442 100644 --- a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnTestType.java +++ b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/ColumnCheckType.java @@ -23,14 +23,14 @@ /** - * This annotation marks a specific class as a [[ColumnTest]] to be used in a data flow spec. The specific ColumnTest itself has - * to derive from the ColumnTest class + * This annotation marks a specific class as a [[ColumnCheck]] to be used in a data flow spec. The specific ColumnCheck itself has + * to derive from the ColumnCheck class */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface ColumnTestType { +public @interface ColumnCheckType { /** - * Specifies the kind of the column test which is used in data flow specifications. + * Specifies the kind of the column check which is used in data flow specifications. * @return */ String kind(); diff --git a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaCheckType.java similarity index 83% rename from flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java rename to flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaCheckType.java index 85b6a5847..b87c4807a 100644 --- a/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaTestType.java +++ b/flowman-spec/src/main/java/com/dimajix/flowman/spec/annotation/SchemaCheckType.java @@ -23,12 +23,12 @@ /** - * This annotation marks a specific class as a [[SchemaTest]] to be used in a data flow spec. The specific SchemaTest itself has - * to derive from the SchemaTest class + * This annotation marks a specific class as a [[SchemaCheck]] to be used in a data flow spec. The specific SchemaCheck itself has + * to derive from the SchemaCheck class */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE}) -public @interface SchemaTestType { +public @interface SchemaCheckType { /** * Specifies the kind of the schema test which is used in data flow specifications. * @return diff --git a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler index 1c947a9f1..684781b65 100644 --- a/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler +++ b/flowman-spec/src/main/resources/META-INF/services/com.dimajix.flowman.spi.ClassAnnotationHandler @@ -12,5 +12,5 @@ com.dimajix.flowman.spec.target.TargetSpecAnnotationHandler com.dimajix.flowman.spec.assertion.AssertionSpecAnnotationHandler com.dimajix.flowman.spec.storage.ParcelSpecAnnotationHandler com.dimajix.flowman.spec.documentation.GeneratorSpecAnnotationHandler -com.dimajix.flowman.spec.documentation.ColumnTestSpecAnnotationHandler -com.dimajix.flowman.spec.documentation.SchemaTestSpecAnnotationHandler +com.dimajix.flowman.spec.documentation.ColumnCheckSpecAnnotationHandler +com.dimajix.flowman.spec.documentation.SchemaCheckSpecAnnotationHandler diff --git a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl index 95cc3417e..45aecb410 100644 --- a/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl +++ b/flowman-spec/src/main/resources/com/dimajix/flowman/documentation/html/project.vtl @@ -41,17 +41,17 @@ border-left: none; } - table.columnTests { + table.columnChecks { width: 100%; } - table.columnTests tbody td { + table.columnChecks tbody td { font-size: 12px; border-width: 0; } - table.columnTests tr { + table.columnChecks tr { background: transparent; } - table.columnTests tr:nth-child(even) { + table.columnChecks tr:nth-child(even) { background: transparent; } @@ -148,8 +148,8 @@ -#macro(testStatus $test) -#if(${test.success})#elseif(${test.failure})#else#end${test.status} +#macro(testStatus $check) +#if(${check.success})#elseif(${check.failure})#else#end${check.status} #end #macro(schema $schema) @@ -160,7 +160,7 @@

- + @@ -172,12 +172,12 @@
${test.name}${test.status}#testStatus($test) #if(${test.result})${test.result.description}#end
${test.name}${test.status}#testStatus($test) #if(${test.result})${test.result.description}#end
Data Type Constraints DescriptionTestsQuality Checks
#if(!$column.nullable) NOT NULL #end ${column.description} - - #foreach($test in ${column.tests}) +
+ #foreach($check in ${column.checks}) - - - + + + #end
${test.name}#testStatus($test)#if(${test.result})${test.result.description}#end${check.name}#testStatus($check)#if(${check.result})${check.result.description}#end
@@ -187,21 +187,21 @@ #end
-#if($schema.tests) +#if($schema.checks) - + - #foreach($test in ${schema.tests}) + #foreach($check in ${schema.checks}) - - - + + + #end diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala index c2bcd71b1..9d2ca895a 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/ObjectMapper.scala @@ -24,9 +24,9 @@ import com.dimajix.flowman.spec.assertion.AssertionSpec import com.dimajix.flowman.spec.catalog.CatalogSpec import com.dimajix.flowman.spec.connection.ConnectionSpec import com.dimajix.flowman.spec.dataset.DatasetSpec -import com.dimajix.flowman.spec.documentation.ColumnTestSpec +import com.dimajix.flowman.spec.documentation.ColumnCheckSpec import com.dimajix.flowman.spec.documentation.GeneratorSpec -import com.dimajix.flowman.spec.documentation.SchemaTestSpec +import com.dimajix.flowman.spec.documentation.SchemaCheckSpec import com.dimajix.flowman.spec.history.HistorySpec import com.dimajix.flowman.spec.mapping.MappingSpec import com.dimajix.flowman.spec.measure.MeasureSpec @@ -71,8 +71,8 @@ object ObjectMapper extends CoreObjectMapper { val metricSinkTypes = MetricSinkSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val parcelTypes = ParcelSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val generatorTypes = GeneratorSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val columnTestTypes = ColumnTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) - val schemaTestTypes = SchemaTestSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val columnTestTypes = ColumnCheckSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) + val schemaTestTypes = SchemaCheckSpec.subtypes.map(kv => new NamedType(kv._2, kv._1)) val mapper = super.mapper mapper.registerSubtypes(stateStoreTypes: _*) mapper.registerSubtypes(catalogTypes: _*) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala index 9b4b4e531..9af9d8a50 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/CollectorSpec.scala @@ -23,7 +23,7 @@ import com.dimajix.flowman.documentation.Collector import com.dimajix.flowman.documentation.MappingCollector import com.dimajix.flowman.documentation.RelationCollector import com.dimajix.flowman.documentation.TargetCollector -import com.dimajix.flowman.documentation.TestCollector +import com.dimajix.flowman.documentation.CheckCollector import com.dimajix.flowman.execution.Context import com.dimajix.flowman.spec.Spec @@ -33,7 +33,7 @@ import com.dimajix.flowman.spec.Spec new JsonSubTypes.Type(name = "mappings", value = classOf[MappingCollectorSpec]), new JsonSubTypes.Type(name = "relations", value = classOf[RelationCollectorSpec]), new JsonSubTypes.Type(name = "targets", value = classOf[TargetCollectorSpec]), - new JsonSubTypes.Type(name = "tests", value = classOf[TestCollectorSpec]) + new JsonSubTypes.Type(name = "checks", value = classOf[CheckCollectorSpec]) )) abstract class CollectorSpec extends Spec[Collector] { override def instantiate(context: Context): Collector @@ -57,8 +57,8 @@ final class TargetCollectorSpec extends CollectorSpec { } } -final class TestCollectorSpec extends CollectorSpec { - override def instantiate(context: Context): TestCollector = { - new TestCollector() +final class CheckCollectorSpec extends CollectorSpec { + override def instantiate(context: Context): CheckCollector = { + new CheckCollector() } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnCheckSpec.scala similarity index 66% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnCheckSpec.scala index 3d9a71840..c7addfd20 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnCheckSpec.scala @@ -22,85 +22,85 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo import com.dimajix.common.TypeRegistry import com.dimajix.flowman.documentation.ColumnReference -import com.dimajix.flowman.documentation.ColumnTest -import com.dimajix.flowman.documentation.ExpressionColumnTest -import com.dimajix.flowman.documentation.ForeignKeyColumnTest -import com.dimajix.flowman.documentation.NotNullColumnTest -import com.dimajix.flowman.documentation.RangeColumnTest -import com.dimajix.flowman.documentation.UniqueColumnTest -import com.dimajix.flowman.documentation.ValuesColumnTest +import com.dimajix.flowman.documentation.ColumnCheck +import com.dimajix.flowman.documentation.ExpressionColumnCheck +import com.dimajix.flowman.documentation.ForeignKeyColumnCheck +import com.dimajix.flowman.documentation.NotNullColumnCheck +import com.dimajix.flowman.documentation.RangeColumnCheck +import com.dimajix.flowman.documentation.UniqueColumnCheck +import com.dimajix.flowman.documentation.ValuesColumnCheck import com.dimajix.flowman.execution.Context import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.RelationIdentifier -import com.dimajix.flowman.spec.annotation.ColumnTestType +import com.dimajix.flowman.spec.annotation.ColumnCheckType import com.dimajix.flowman.spi.ClassAnnotationHandler -object ColumnTestSpec extends TypeRegistry[ColumnTestSpec] { +object ColumnCheckSpec extends TypeRegistry[ColumnCheckSpec] { } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") @JsonSubTypes(value = Array( - new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionColumnTestSpec]), - new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeyColumnTestSpec]), - new JsonSubTypes.Type(name = "notNull", value = classOf[NotNullColumnTestSpec]), - new JsonSubTypes.Type(name = "unique", value = classOf[UniqueColumnTestSpec]), - new JsonSubTypes.Type(name = "range", value = classOf[RangeColumnTestSpec]), - new JsonSubTypes.Type(name = "values", value = classOf[ValuesColumnTestSpec]) + new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionColumnCheckSpec]), + new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeyColumnCheckSpec]), + new JsonSubTypes.Type(name = "notNull", value = classOf[NotNullColumnCheckSpec]), + new JsonSubTypes.Type(name = "unique", value = classOf[UniqueColumnCheckSpec]), + new JsonSubTypes.Type(name = "range", value = classOf[RangeColumnCheckSpec]), + new JsonSubTypes.Type(name = "values", value = classOf[ValuesColumnCheckSpec]) )) -abstract class ColumnTestSpec { - def instantiate(context: Context, parent:ColumnReference): ColumnTest +abstract class ColumnCheckSpec { + def instantiate(context: Context, parent:ColumnReference): ColumnCheck } -class ColumnTestSpecAnnotationHandler extends ClassAnnotationHandler { - override def annotation: Class[_] = classOf[ColumnTestType] +class ColumnCheckSpecAnnotationHandler extends ClassAnnotationHandler { + override def annotation: Class[_] = classOf[ColumnCheckType] override def register(clazz: Class[_]): Unit = - ColumnTestSpec.register(clazz.getAnnotation(classOf[ColumnTestType]).kind(), clazz.asInstanceOf[Class[_ <: ColumnTestSpec]]) + ColumnCheckSpec.register(clazz.getAnnotation(classOf[ColumnCheckType]).kind(), clazz.asInstanceOf[Class[_ <: ColumnCheckSpec]]) } -class NotNullColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): NotNullColumnTest = NotNullColumnTest(Some(parent)) +class NotNullColumnCheckSpec extends ColumnCheckSpec { + override def instantiate(context: Context, parent:ColumnReference): NotNullColumnCheck = NotNullColumnCheck(Some(parent)) } -class UniqueColumnTestSpec extends ColumnTestSpec { - override def instantiate(context: Context, parent:ColumnReference): UniqueColumnTest = UniqueColumnTest(Some(parent)) +class UniqueColumnCheckSpec extends ColumnCheckSpec { + override def instantiate(context: Context, parent:ColumnReference): UniqueColumnCheck = UniqueColumnCheck(Some(parent)) } -class RangeColumnTestSpec extends ColumnTestSpec { +class RangeColumnCheckSpec extends ColumnCheckSpec { @JsonProperty(value="lower", required=true) private var lower:String = "" @JsonProperty(value="upper", required=true) private var upper:String = "" - override def instantiate(context: Context, parent:ColumnReference): RangeColumnTest = RangeColumnTest( + override def instantiate(context: Context, parent:ColumnReference): RangeColumnCheck = RangeColumnCheck( Some(parent), None, context.evaluate(lower), context.evaluate(upper) ) } -class ValuesColumnTestSpec extends ColumnTestSpec { +class ValuesColumnCheckSpec extends ColumnCheckSpec { @JsonProperty(value="values", required=false) private var values:Seq[String] = Seq() - override def instantiate(context: Context, parent:ColumnReference): ValuesColumnTest = ValuesColumnTest( + override def instantiate(context: Context, parent:ColumnReference): ValuesColumnCheck = ValuesColumnCheck( Some(parent), values=values.map(context.evaluate) ) } -class ExpressionColumnTestSpec extends ColumnTestSpec { +class ExpressionColumnCheckSpec extends ColumnCheckSpec { @JsonProperty(value="expression", required=true) private var expression:String = _ - override def instantiate(context: Context, parent:ColumnReference): ExpressionColumnTest = ExpressionColumnTest( + override def instantiate(context: Context, parent:ColumnReference): ExpressionColumnCheck = ExpressionColumnCheck( Some(parent), expression=context.evaluate(expression) ) } -class ForeignKeyColumnTestSpec extends ColumnTestSpec { +class ForeignKeyColumnCheckSpec extends ColumnCheckSpec { @JsonProperty(value="mapping", required=false) private var mapping:Option[String] = None @JsonProperty(value="relation", required=false) private var relation:Option[String] = None @JsonProperty(value="column", required=false) private var column:Option[String] = None - override def instantiate(context: Context, parent:ColumnReference): ForeignKeyColumnTest = ForeignKeyColumnTest( + override def instantiate(context: Context, parent:ColumnReference): ForeignKeyColumnCheck = ForeignKeyColumnCheck( Some(parent), relation=context.evaluate(relation).map(RelationIdentifier(_)), mapping=context.evaluate(mapping).map(MappingOutputIdentifier(_)), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala index 4a682527e..391a92f43 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/ColumnDocSpec.scala @@ -29,7 +29,7 @@ class ColumnDocSpec { @JsonProperty(value="name", required=true) private var name:String = _ @JsonProperty(value="description", required=false) private var description:Option[String] = None @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() - @JsonProperty(value="tests", required=false) private var tests:Seq[ColumnTestSpec] = Seq() + @JsonProperty(value="checks", required=false) private var checks:Seq[ColumnCheckSpec] = Seq() def instantiate(context: Context, parent:Reference): ColumnDoc = { val doc = ColumnDoc( @@ -41,11 +41,11 @@ class ColumnDocSpec { def ref = doc.reference val cols = columns.map(_.instantiate(context, ref)) - val tests = this.tests.map(_.instantiate(context, ref)) + val tests = this.checks.map(_.instantiate(context, ref)) doc.copy( children = cols, - tests = tests + checks = tests ) } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala index 08859a53b..55606f17b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/MappingDocSpec.scala @@ -31,7 +31,7 @@ import com.dimajix.flowman.spec.Spec class MappingOutputDocSpec { @JsonProperty(value="description", required=false) private var description:Option[String] = None @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() - @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + @JsonProperty(value="checks", required=false) private var checks:Seq[SchemaCheckSpec] = Seq() def instantiate(context: Context, parent:MappingReference, name:String): MappingOutputDoc = { val doc = MappingOutputDoc( @@ -43,7 +43,7 @@ class MappingOutputDocSpec { val ref = doc.reference val schema = - if (columns.nonEmpty || tests.nonEmpty) { + if (columns.nonEmpty || checks.nonEmpty) { val schema = SchemaDoc( Some(ref), None, @@ -52,10 +52,10 @@ class MappingOutputDocSpec { ) val ref2 = schema.reference val cols = columns.map(_.instantiate(context, ref2)) - val tests = this.tests.map(_.instantiate(context, ref2)) + val tests = this.checks.map(_.instantiate(context, ref2)) Some(schema.copy( columns=cols, - tests=tests + checks=tests )) } else { @@ -73,7 +73,7 @@ class MappingDocSpec extends Spec[MappingDoc] { @JsonProperty(value="description", required=false) private var description:Option[String] = None @JsonProperty(value="outputs", required=false) private var outputs:Map[String,MappingOutputDocSpec] = Map() @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() - @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaCheckSpec] = Seq() def instantiate(context: Context): MappingDoc = { val doc = MappingDoc( @@ -101,7 +101,7 @@ class MappingDocSpec extends Spec[MappingDoc] { output.copy( schema = Some(schema.copy( columns=cols, - tests=tsts + checks=tsts )) ) ) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala index dec6b36bb..967c02db0 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/RelationDocSpec.scala @@ -28,7 +28,7 @@ import com.dimajix.flowman.spec.Spec class RelationDocSpec extends Spec[RelationDoc] { @JsonProperty(value="description", required=false) private var description:Option[String] = None @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() - @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + @JsonProperty(value="checks", required=false) private var checks:Seq[SchemaCheckSpec] = Seq() override def instantiate(context: Context): RelationDoc = { val doc = RelationDoc( @@ -39,16 +39,16 @@ class RelationDocSpec extends Spec[RelationDoc] { val ref = doc.reference val schema = - if (columns.nonEmpty || tests.nonEmpty) { + if (columns.nonEmpty || checks.nonEmpty) { val schema = SchemaDoc( Some(ref) ) val ref2 = schema.reference val cols = columns.map(_.instantiate(context, ref2)) - val tests = this.tests.map(_.instantiate(context, ref2)) + val tests = this.checks.map(_.instantiate(context, ref2)) Some(schema.copy( columns=cols, - tests=tests + checks=tests )) } else { diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaCheckSpec.scala similarity index 71% rename from flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala rename to flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaCheckSpec.scala index 897f310de..8060b7c9b 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaTestSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaCheckSpec.scala @@ -21,64 +21,64 @@ import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo import com.dimajix.common.TypeRegistry -import com.dimajix.flowman.documentation.ExpressionSchemaTest -import com.dimajix.flowman.documentation.ForeignKeySchemaTest -import com.dimajix.flowman.documentation.PrimaryKeySchemaTest +import com.dimajix.flowman.documentation.ExpressionSchemaCheck +import com.dimajix.flowman.documentation.ForeignKeySchemaCheck +import com.dimajix.flowman.documentation.PrimaryKeySchemaCheck import com.dimajix.flowman.documentation.SchemaReference -import com.dimajix.flowman.documentation.SchemaTest +import com.dimajix.flowman.documentation.SchemaCheck import com.dimajix.flowman.execution.Context import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.RelationIdentifier -import com.dimajix.flowman.spec.annotation.SchemaTestType +import com.dimajix.flowman.spec.annotation.SchemaCheckType import com.dimajix.flowman.spi.ClassAnnotationHandler -object SchemaTestSpec extends TypeRegistry[SchemaTestSpec] { +object SchemaCheckSpec extends TypeRegistry[SchemaCheckSpec] { } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "kind") @JsonSubTypes(value = Array( - new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionSchemaTestSpec]), - new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeySchemaTestSpec]), - new JsonSubTypes.Type(name = "primaryKey", value = classOf[PrimaryKeySchemaTestSpec]) + new JsonSubTypes.Type(name = "expression", value = classOf[ExpressionSchemaCheckSpec]), + new JsonSubTypes.Type(name = "foreignKey", value = classOf[ForeignKeySchemaCheckSpec]), + new JsonSubTypes.Type(name = "primaryKey", value = classOf[PrimaryKeySchemaCheckSpec]) )) -abstract class SchemaTestSpec { - def instantiate(context: Context, parent:SchemaReference): SchemaTest +abstract class SchemaCheckSpec { + def instantiate(context: Context, parent:SchemaReference): SchemaCheck } -class SchemaTestSpecAnnotationHandler extends ClassAnnotationHandler { - override def annotation: Class[_] = classOf[SchemaTestType] +class SchemaCheckSpecAnnotationHandler extends ClassAnnotationHandler { + override def annotation: Class[_] = classOf[SchemaCheckType] override def register(clazz: Class[_]): Unit = - SchemaTestSpec.register(clazz.getAnnotation(classOf[SchemaTestType]).kind(), clazz.asInstanceOf[Class[_ <: SchemaTestSpec]]) + SchemaCheckSpec.register(clazz.getAnnotation(classOf[SchemaCheckType]).kind(), clazz.asInstanceOf[Class[_ <: SchemaCheckSpec]]) } -class PrimaryKeySchemaTestSpec extends SchemaTestSpec { +class PrimaryKeySchemaCheckSpec extends SchemaCheckSpec { @JsonProperty(value="columns", required=false) private var columns:Seq[String] = Seq.empty - override def instantiate(context: Context, parent:SchemaReference): PrimaryKeySchemaTest = PrimaryKeySchemaTest( + override def instantiate(context: Context, parent:SchemaReference): PrimaryKeySchemaCheck = PrimaryKeySchemaCheck( Some(parent), columns = columns.map(context.evaluate) ) } -class ExpressionSchemaTestSpec extends SchemaTestSpec { +class ExpressionSchemaCheckSpec extends SchemaCheckSpec { @JsonProperty(value="expression", required=true) private var expression:String = _ - override def instantiate(context: Context, parent:SchemaReference): ExpressionSchemaTest = ExpressionSchemaTest( + override def instantiate(context: Context, parent:SchemaReference): ExpressionSchemaCheck = ExpressionSchemaCheck( Some(parent), expression = context.evaluate(expression) ) } -class ForeignKeySchemaTestSpec extends SchemaTestSpec { +class ForeignKeySchemaCheckSpec extends SchemaCheckSpec { @JsonProperty(value="mapping", required=false) private var mapping:Option[String] = None @JsonProperty(value="relation", required=false) private var relation:Option[String] = None @JsonProperty(value="columns", required=false) private var columns:Seq[String] = Seq.empty @JsonProperty(value="references", required=false) private var references:Seq[String] = Seq.empty - override def instantiate(context: Context, parent:SchemaReference): ForeignKeySchemaTest = ForeignKeySchemaTest( + override def instantiate(context: Context, parent:SchemaReference): ForeignKeySchemaCheck = ForeignKeySchemaCheck( Some(parent), columns=columns.map(context.evaluate), relation=context.evaluate(relation).map(RelationIdentifier(_)), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala index 38436fd02..948ae4010 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/documentation/SchemaDocSpec.scala @@ -26,7 +26,7 @@ import com.dimajix.flowman.execution.Context class SchemaDocSpec { @JsonProperty(value="description", required=false) private var description:Option[String] = None @JsonProperty(value="columns", required=false) private var columns:Seq[ColumnDocSpec] = Seq() - @JsonProperty(value="tests", required=false) private var tests:Seq[SchemaTestSpec] = Seq() + @JsonProperty(value="checks", required=false) private var checks:Seq[SchemaCheckSpec] = Seq() def instantiate(context: Context, parent:Reference): SchemaDoc = { val doc = SchemaDoc( @@ -36,10 +36,10 @@ class SchemaDocSpec { val ref = doc.reference val cols = columns.map(_.instantiate(context, ref)) - val tests = this.tests.map(_.instantiate(context, ref)) + val tests = this.checks.map(_.instantiate(context, ref)) doc.copy( columns = cols, - tests = tests + checks = tests ) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnCheckTest.scala similarity index 66% rename from flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala rename to flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnCheckTest.scala index a327edf47..81eee97b3 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnTestTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/ColumnCheckTest.scala @@ -20,32 +20,32 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.documentation.ColumnReference -import com.dimajix.flowman.documentation.ExpressionColumnTest -import com.dimajix.flowman.documentation.RangeColumnTest -import com.dimajix.flowman.documentation.UniqueColumnTest -import com.dimajix.flowman.documentation.ValuesColumnTest +import com.dimajix.flowman.documentation.ExpressionColumnCheck +import com.dimajix.flowman.documentation.RangeColumnCheck +import com.dimajix.flowman.documentation.UniqueColumnCheck +import com.dimajix.flowman.documentation.ValuesColumnCheck import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper -class ColumnTestTest extends AnyFlatSpec with Matchers { - "A ColumnTest" should "be deserializable" in { +class ColumnCheckTest extends AnyFlatSpec with Matchers { + "A ColumnCheck" should "be deserializable" in { val yaml = """ |kind: unique """.stripMargin - val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[UniqueColumnTestSpec] + val spec = ObjectMapper.parse[ColumnCheckSpec](yaml) + spec shouldBe a[UniqueColumnCheckSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (UniqueColumnTest( + test should be (UniqueColumnCheck( Some(ColumnReference(None, "col0")) )) } - "A RangeColumnTest" should "be deserializable" in { + "A RangeColumnCheck" should "be deserializable" in { val yaml = """ |kind: range @@ -53,49 +53,49 @@ class ColumnTestTest extends AnyFlatSpec with Matchers { |upper: 23 """.stripMargin - val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[RangeColumnTestSpec] + val spec = ObjectMapper.parse[ColumnCheckSpec](yaml) + spec shouldBe a[RangeColumnCheckSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (RangeColumnTest( + test should be (RangeColumnCheck( Some(ColumnReference(None, "col0")), lower="7", upper="23" )) } - "A ValuesColumnTest" should "be deserializable" in { + "A ValuesColumnCheck" should "be deserializable" in { val yaml = """ |kind: values |values: ['a', 12, null] """.stripMargin - val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[ValuesColumnTestSpec] + val spec = ObjectMapper.parse[ColumnCheckSpec](yaml) + spec shouldBe a[ValuesColumnCheckSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (ValuesColumnTest( + test should be (ValuesColumnCheck( Some(ColumnReference(None, "col0")), values = Seq("a", "12", null) )) } - "A ExpressionColumnTest" should "be deserializable" in { + "A ExpressionColumnCheck" should "be deserializable" in { val yaml = """ |kind: expression |expression: "col1 < col2" """.stripMargin - val spec = ObjectMapper.parse[ColumnTestSpec](yaml) - spec shouldBe a[ExpressionColumnTestSpec] + val spec = ObjectMapper.parse[ColumnCheckSpec](yaml) + spec shouldBe a[ExpressionColumnCheckSpec] val context = RootContext.builder().build() val test = spec.instantiate(context, ColumnReference(None, "col0")) - test should be (ExpressionColumnTest( + test should be (ExpressionColumnCheck( Some(ColumnReference(None, "col0")), expression = "col1 < col2" )) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala index ca2e51dda..58740320b 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/DocumenterTest.scala @@ -34,7 +34,7 @@ class DocumenterTest extends AnyFlatSpec with Matchers { | # Collect documentation of build targets | - kind: targets | # Execute all tests - | - kind: tests + | - kind: checks | |generators: | # Create an output file in the project directory diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala index bcad1723a..a761bacaa 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/MappingDocTest.scala @@ -20,7 +20,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.documentation.ColumnReference -import com.dimajix.flowman.documentation.NotNullColumnTest +import com.dimajix.flowman.documentation.NotNullColumnCheck import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper @@ -33,7 +33,7 @@ class MappingDocTest extends AnyFlatSpec with Matchers { |columns: | - name: col_a | description: "This is column a" - | tests: + | checks: | - kind: notNull |outputs: | other: @@ -56,9 +56,9 @@ class MappingDocTest extends AnyFlatSpec with Matchers { mainSchema.columns.size should be (1) mainSchema.columns(0).name should be ("col_a") mainSchema.columns(0).description should be (Some("This is column a")) - mainSchema.columns(0).tests.size should be (1) - mainSchema.columns(0).tests(0) shouldBe a[NotNullColumnTest] - mainSchema.tests.size should be (0) + mainSchema.columns(0).checks.size should be (1) + mainSchema.columns(0).checks(0) shouldBe a[NotNullColumnCheck] + mainSchema.checks.size should be (0) val other = mapping.outputs.find(_.name == "other").get other.description should be (Some("This is an additional output")) @@ -66,7 +66,7 @@ class MappingDocTest extends AnyFlatSpec with Matchers { otherSchema.columns.size should be (1) otherSchema.columns(0).name should be ("col_x") otherSchema.columns(0).description should be (Some("Column of other output")) - otherSchema.columns(0).tests.size should be (0) - otherSchema.tests.size should be (0) + otherSchema.columns(0).checks.size should be (0) + otherSchema.checks.size should be (0) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala index 5da396742..cf2293ca5 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/documentation/RelationDocTest.scala @@ -19,7 +19,7 @@ package com.dimajix.flowman.spec.documentation import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import com.dimajix.flowman.documentation.NotNullColumnTest +import com.dimajix.flowman.documentation.NotNullColumnCheck import com.dimajix.flowman.execution.RootContext import com.dimajix.flowman.spec.ObjectMapper @@ -32,7 +32,7 @@ class RelationDocTest extends AnyFlatSpec with Matchers { |columns: | - name: col_a | description: "This is column a" - | tests: + | checks: | - kind: notNull | - name: col_x | description: "Column of other output" @@ -51,11 +51,11 @@ class RelationDocTest extends AnyFlatSpec with Matchers { mainSchema.columns.size should be (2) mainSchema.columns(0).name should be ("col_a") mainSchema.columns(0).description should be (Some("This is column a")) - mainSchema.columns(0).tests.size should be (1) - mainSchema.columns(0).tests(0) shouldBe a[NotNullColumnTest] + mainSchema.columns(0).checks.size should be (1) + mainSchema.columns(0).checks(0) shouldBe a[NotNullColumnCheck] mainSchema.columns(1).name should be ("col_x") mainSchema.columns(1).description should be (Some("Column of other output")) - mainSchema.columns(1).tests.size should be (0) - mainSchema.tests.size should be (0) + mainSchema.columns(1).checks.size should be (0) + mainSchema.checks.size should be (0) } } diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala index fece1a85b..afec1de0d 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/target/DocumentTargetTest.scala @@ -36,8 +36,8 @@ class DocumentTargetTest extends AnyFlatSpec with Matchers { | - kind: mappings | # Collect documentation of build targets | - kind: targets - | # Execute all tests - | - kind: tests + | # Execute all checks + | - kind: checks | | generators: | # Create an output file in the project directory From b80d1dce28b6e97aab6b7af4427a62baa55f09d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 04:29:59 +0000 Subject: [PATCH 87/95] Bump url-parse from 1.5.7 to 1.5.10 in /flowman-studio-ui Bumps [url-parse](https://github.com/unshiftio/url-parse) from 1.5.7 to 1.5.10. - [Release notes](https://github.com/unshiftio/url-parse/releases) - [Commits](https://github.com/unshiftio/url-parse/compare/1.5.7...1.5.10) --- updated-dependencies: - dependency-name: url-parse dependency-type: indirect ... Signed-off-by: dependabot[bot] --- flowman-studio-ui/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flowman-studio-ui/package-lock.json b/flowman-studio-ui/package-lock.json index 198a8356a..aff032287 100644 --- a/flowman-studio-ui/package-lock.json +++ b/flowman-studio-ui/package-lock.json @@ -15328,9 +15328,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dev": true, "dependencies": { "querystringify": "^2.1.1", @@ -29127,9 +29127,9 @@ } }, "url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dev": true, "requires": { "querystringify": "^2.1.1", From 36b256a77beff28e5493c10da2e7ed7c8a4c8790 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 04:30:13 +0000 Subject: [PATCH 88/95] Bump url-parse from 1.5.7 to 1.5.10 in /flowman-server-ui Bumps [url-parse](https://github.com/unshiftio/url-parse) from 1.5.7 to 1.5.10. - [Release notes](https://github.com/unshiftio/url-parse/releases) - [Commits](https://github.com/unshiftio/url-parse/compare/1.5.7...1.5.10) --- updated-dependencies: - dependency-name: url-parse dependency-type: indirect ... Signed-off-by: dependabot[bot] --- flowman-server-ui/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flowman-server-ui/package-lock.json b/flowman-server-ui/package-lock.json index 2de734f9f..520f418ed 100644 --- a/flowman-server-ui/package-lock.json +++ b/flowman-server-ui/package-lock.json @@ -15255,9 +15255,9 @@ } }, "node_modules/url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dev": true, "dependencies": { "querystringify": "^2.1.1", @@ -29298,9 +29298,9 @@ } }, "url-parse": { - "version": "1.5.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.7.tgz", - "integrity": "sha512-HxWkieX+STA38EDk7CE9MEryFeHCKzgagxlGvsdS7WBImq9Mk+PGwiT56w82WI3aicwJA8REp42Cxo98c8FZMA==", + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dev": true, "requires": { "querystringify": "^2.1.1", From b92112fed9d5c7df58c4a9f74f42b9524650774c Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 28 Feb 2022 12:37:00 +0100 Subject: [PATCH 89/95] Use custom 'TableIdentifier' class instead of Sparks own class --- CHANGELOG.md | 1 + .../dimajix/flowman/catalog/HiveCatalog.scala | 53 ++++++++-------- .../flowman/catalog/TableDefinition.scala | 6 +- .../flowman/catalog/TableIdentifier.scala | 60 +++++++++++++++++++ .../dimajix/flowman/jdbc/BaseDialect.scala | 7 +-- .../dimajix/flowman/jdbc/DerbyDialect.scala | 12 ++-- .../dimajix/flowman/jdbc/HiveDialect.scala | 7 ++- .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 2 +- .../flowman/jdbc/MsSqlServerDialect.scala | 8 +-- .../dimajix/flowman/jdbc/MySQLDialect.scala | 5 +- .../com/dimajix/flowman/jdbc/SqlDialect.scala | 3 +- .../dimajix/flowman/jdbc/SqlStatements.scala | 2 +- .../flowman/model/ResourceIdentifier.scala | 12 ++-- .../flowman/catalog/TableChangeTest.scala | 1 - .../flowman/catalog/TableIdentifierTest.scala | 56 +++++++++++++++++ .../flowman/jdbc/BaseDialectTest.scala | 4 +- .../dimajix/flowman/jdbc/DerbyJdbcTest.scala | 2 +- .../com/dimajix/flowman/jdbc/H2JdbcTest.scala | 3 +- .../dimajix/flowman/jdbc/JdbcUtilsTest.scala | 4 +- .../flowman/dsl/relation/HiveTable.scala | 4 +- .../flowman/dsl/relation/HiveUnionTable.scala | 2 +- .../flowman/dsl/relation/HiveView.scala | 3 +- .../spec/relation/DeltaFileRelation.scala | 11 ++-- .../flowman/spec/relation/DeltaRelation.scala | 8 +-- .../spec/relation/DeltaTableRelation.scala | 16 +++-- .../spec/target/DeltaVacuumTarget.scala | 6 +- .../relation/DeltaTableRelationTest.scala | 12 ++-- .../spec/target/DeltaVacuumTargetTest.scala | 7 +-- .../spec/relation/SqlServerRelation.scala | 3 +- .../spec/mapping/ReadHiveMapping.scala | 30 +++++----- .../flowman/spec/relation/HiveRelation.scala | 2 +- .../spec/relation/HiveTableRelation.scala | 6 +- .../relation/HiveUnionTableRelation.scala | 5 +- .../spec/relation/HiveViewRelation.scala | 2 +- .../flowman/spec/relation/JdbcRelation.scala | 8 +-- .../flowman/spec/mapping/ReadHiveTest.scala | 11 ++-- .../spec/relation/HiveTableRelationTest.scala | 32 +++++----- .../relation/HiveUnionTableRelationTest.scala | 26 ++++---- .../spec/relation/HiveViewRelationTest.scala | 2 +- .../spec/relation/JdbcRelationTest.scala | 4 +- 40 files changed, 270 insertions(+), 178 deletions(-) create mode 100644 flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableIdentifier.scala create mode 100644 flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableIdentifierTest.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index 30e5afc97..af21450f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Implement schema cache in Executor to speed up documentation and similar tasks * Add new config variables `flowman.execution.mapping.schemaCache` and `flowman.execution.relation.schemaCache` * Add new config variable `flowman.default.target.verifyPolicy` to ignore empty tables during VERIFY phase +* Implement initial support for indexes in JDBC relations # Version 0.21.2 - 2022-02-14 diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala index d0a814540..f5ee93c97 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/HiveCatalog.scala @@ -23,7 +23,6 @@ import scala.collection.mutable import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession import org.apache.spark.sql.SparkShim -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.DatabaseAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException import org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException @@ -138,7 +137,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex val dbName = formatDatabaseName(database) catalog.externalCatalog .listTables(dbName) - .map(name =>TableIdentifier(name, Some(database))) + .map(name =>TableIdentifier(name, Seq(database))) } /** @@ -151,7 +150,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex val dbName = formatDatabaseName(database) catalog.externalCatalog .listTables(dbName, pattern) - .map(name =>TableIdentifier(name, Some(database))) + .map(name =>TableIdentifier(name, Seq(database))) } /** @@ -164,7 +163,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex def createTable(table:CatalogTable, ignoreIfExists:Boolean) : Unit = { require(table != null) - val exists = tableExists(table.identifier) + val exists = tableExists(TableIdentifier.of(table)) if (!ignoreIfExists && exists) { throw new TableAlreadyExistsException(table.identifier.database.getOrElse(""), table.identifier.table) } @@ -205,7 +204,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex } if (config.flowmanConf.hiveAnalyzeTable) { - val cmd = AnalyzeTableCommand(table, false) + val cmd = AnalyzeTableCommand(table.toSpark, false) cmd.run(spark) } @@ -222,7 +221,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex * @return */ def tableExists(name:TableIdentifier) : Boolean = { - catalog.tableExists(name) + catalog.tableExists(name.toSpark) } /** @@ -237,7 +236,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex val db = formatDatabaseName(name.database.getOrElse(catalog.getCurrentDatabase)) val table = formatTableName(name.table) requireDbExists(db) - requireTableExists(TableIdentifier(table, Some(db))) + requireTableExists(TableIdentifier(table, Seq(db))) catalog.externalCatalog.getTable(db, table) } @@ -276,7 +275,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex // Delete all partitions if (catalogTable.partitionSchema != null && catalogTable.partitionSchema.fields.nonEmpty) { - catalog.listPartitions(table).foreach { p => + catalog.listPartitions(table.toSpark).foreach { p => val location = new Path(p.location) val fs = location.getFileSystem(hadoopConf) FileUtils.deleteLocation(fs, location) @@ -284,7 +283,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex } // Delete table itself - val cmd = DropTableCommand(table, ignoreIfNotExists, false, true) + val cmd = DropTableCommand(table.toSpark, ignoreIfNotExists, false, true) cmd.run(spark) // Delete location to cleanup any remaining files @@ -312,7 +311,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex // First drop partitions if (catalogTable.partitionSchema != null && catalogTable.partitionSchema.fields.nonEmpty) { - dropPartitions(table, catalog.listPartitions(table).map(p => PartitionSpec(p.spec))) + dropPartitions(table, catalog.listPartitions(table.toSpark).map(p => PartitionSpec(p.spec))) } // Then cleanup directory from any remainders @@ -349,18 +348,18 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Updating nullability of column ${u.column} to ${u.nullable} in Hive table '$table'") val field = tableColumns.getOrElse(u.column.toLowerCase(Locale.ROOT), throw new IllegalArgumentException(s"Table column ${u.column} does not exist in table $table")) .copy(nullable = u.nullable) - val cmd = AlterTableChangeColumnCommand(table, u.column, field) + val cmd = AlterTableChangeColumnCommand(table.toSpark, u.column, field) cmd.run(spark) case u:UpdateColumnComment => logger.info(s"Updating comment of column ${u.column} in Hive table '$table'") val field = tableColumns.getOrElse(u.column.toLowerCase(Locale.ROOT), throw new IllegalArgumentException(s"Table column ${u.column} does not exist in table $table")) .withComment(u.comment.getOrElse("")) - val cmd = AlterTableChangeColumnCommand(table, u.column, field) + val cmd = AlterTableChangeColumnCommand(table.toSpark, u.column, field) cmd.run(spark) case x:TableChange => throw new UnsupportedOperationException(s"Unsupported table change $x for Hive table $table") } - val cmd = AlterTableAddColumnsCommand(table, colsToAdd) + val cmd = AlterTableAddColumnsCommand(table.toSpark, colsToAdd) cmd.run(spark) externalCatalogs.foreach(_.alterTable(catalogTable)) @@ -381,7 +380,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex val catalogTable = getTable(table) require(catalogTable.tableType != CatalogTableType.VIEW) - val cmd = AlterTableAddColumnsCommand(table, colsToAdd) + val cmd = AlterTableAddColumnsCommand(table.toSpark, colsToAdd) cmd.run(spark) externalCatalogs.foreach(_.alterTable(catalogTable)) @@ -398,7 +397,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex def partitionExists(table:TableIdentifier, partition:PartitionSpec) : Boolean = { require(table != null) require(partition != null) - catalog.listPartitions(table, Some(partition.mapValues(_.toString).toMap).filter(_.nonEmpty)).nonEmpty + catalog.listPartitions(table.toSpark, Some(partition.mapValues(_.toString).toMap).filter(_.nonEmpty)).nonEmpty } /** @@ -409,7 +408,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex @throws[NoSuchTableException] @throws[NoSuchPartitionException] def getPartition(table: TableIdentifier, partition:PartitionSpec): CatalogTablePartition = { - catalog.getPartition(table, partition.mapValues(_.toString).toMap) + catalog.getPartition(table.toSpark, partition.mapValues(_.toString).toMap) } /** @@ -455,14 +454,14 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Adding partition ${partition.spec} to table $table at '$location'") val sparkPartition = partition.mapValues(_.toString).toMap - val cmd = AlterTableAddPartitionCommand(table, Seq((sparkPartition, Some(location.toString))), false) + val cmd = AlterTableAddPartitionCommand(table.toSpark, Seq((sparkPartition, Some(location.toString))), false) cmd.run(spark) analyzePartition(table, sparkPartition) externalCatalogs.foreach { ec => val catalogTable = getTable(table) - val catalogPartition = catalog.getPartition(table, sparkPartition) + val catalogPartition = catalog.getPartition(table.toSpark, sparkPartition) ec.addPartition(catalogTable, catalogPartition) } } @@ -483,7 +482,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex val sparkPartition = partition.mapValues(_.toString).toMap if (partitionExists(table, partition)) { logger.info(s"Replacing partition ${partition.spec} of table $table with location '$location'") - val cmd = AlterTableSetLocationCommand(table, Some(sparkPartition), location.toString) + val cmd = AlterTableSetLocationCommand(table.toSpark, Some(sparkPartition), location.toString) cmd.run(spark) refreshPartition(table, partition) @@ -516,14 +515,14 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex externalCatalogs.foreach { ec => val catalogTable = getTable(table) - val catalogPartition = catalog.getPartition(table, sparkPartition) + val catalogPartition = catalog.getPartition(table.toSpark, sparkPartition) ec.alterPartition(catalogTable, catalogPartition) } } private def analyzePartition(table:TableIdentifier, sparkPartition:Map[String,String]) : Unit = { def doIt(): Unit = { - val cmd = AnalyzePartitionCommand(table, sparkPartition.map { case (k, v) => k -> Some(v) }, false) + val cmd = AnalyzePartitionCommand(table.toSpark, sparkPartition.map { case (k, v) => k -> Some(v) }, false) cmd.run(spark) } @@ -560,7 +559,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex externalCatalogs.foreach { ec => val sparkPartition = partition.mapValues(_.toString).toMap val catalogTable = getTable(table) - val catalogPartition = catalog.getPartition(table, sparkPartition) + val catalogPartition = catalog.getPartition(table.toSpark, sparkPartition) ec.truncatePartition(catalogTable, catalogPartition) } } @@ -599,7 +598,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex // Convert to Spark partitions val sparkPartitions = dropPartitions.map(_.mapValues(_.toString).toMap) // Convert to external catalog partitions which can be reused in the last step - val catalogPartitions = sparkPartitions.map(catalog.getPartition(table, _)).filter(_ != null) + val catalogPartitions = sparkPartitions.map(catalog.getPartition(table.toSpark, _)).filter(_ != null) logger.info(s"Dropping partitions ${dropPartitions.map(_.spec).mkString(",")} from Hive table $table") catalogPartitions.foreach { partition => @@ -609,7 +608,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex } // Note that "purge" is not supported with Hive < 1.2 - val cmd = AlterTableDropPartitionCommand(table, sparkPartitions, ignoreIfNotExists, purge = false, retainData = false) + val cmd = AlterTableDropPartitionCommand(table.toSpark, sparkPartitions, ignoreIfNotExists, purge = false, retainData = false) cmd.run(spark) externalCatalogs.foreach { ec => @@ -632,7 +631,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Creating Hive view $table") val plan = spark.sql(select).queryExecution.analyzed - val cmd = SparkShim.createView(table, select, plan, false, false) + val cmd = SparkShim.createView(table.toSpark, select, plan, false, false) cmd.run(spark) // Publish view to external catalog @@ -648,7 +647,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex logger.info(s"Redefining Hive view $table") val plan = spark.sql(select).queryExecution.analyzed - val cmd = SparkShim.alterView(table, select, plan) + val cmd = SparkShim.alterView(table.toSpark, select, plan) cmd.run(spark) // Publish view to external catalog @@ -675,7 +674,7 @@ final class HiveCatalog(val spark:SparkSession, val config:Configuration, val ex require(catalogTable.tableType == CatalogTableType.VIEW) // Delete table itself - val cmd = DropTableCommand(table, ignoreIfNotExists, true, false) + val cmd = DropTableCommand(table.toSpark, ignoreIfNotExists, true, false) cmd.run(spark) // Remove table from external catalog diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala index 0220d7686..fc4b2d19d 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableDefinition.scala @@ -18,7 +18,6 @@ package com.dimajix.flowman.catalog import java.util.Locale -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import com.dimajix.flowman.types.Field @@ -27,8 +26,9 @@ import com.dimajix.flowman.types.StructType object TableDefinition { def ofTable(table:CatalogTable) : TableDefinition = { - val sourceSchema = com.dimajix.flowman.types.StructType.of(table.dataSchema) - TableDefinition(table.identifier, sourceSchema.fields) + val id = table.identifier + val schema = com.dimajix.flowman.types.StructType.of(table.dataSchema) + TableDefinition(TableIdentifier(id.table, id.database.toSeq), schema.fields) } } final case class TableDefinition( diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableIdentifier.scala b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableIdentifier.scala new file mode 100644 index 000000000..8deec49cf --- /dev/null +++ b/flowman-core/src/main/scala/com/dimajix/flowman/catalog/TableIdentifier.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2018-2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.catalog + +import org.apache.spark.sql.catalyst.catalog.CatalogTable + + +object TableIdentifier { + def apply(table: String, db:Option[String]) : TableIdentifier = TableIdentifier(table, db.toSeq) + + def empty : TableIdentifier = TableIdentifier("", Seq.empty) + + def of(table:CatalogTable) : TableIdentifier = { + of(table.identifier) + } + def of(id: org.apache.spark.sql.catalyst.TableIdentifier) : TableIdentifier = { + TableIdentifier(id.table, id.database.toSeq) + } +} +final case class TableIdentifier( + table: String, + space: Seq[String] = Seq.empty +) { + private def quoteIdentifier(name: String): String = s"`${name.replace("`", "``")}`" + + def quotedString: String = { + val replacedId = quoteIdentifier(table) + val replacedSpace = space.map(quoteIdentifier) + + if (replacedSpace.nonEmpty) s"${replacedSpace.mkString(".")}.$replacedId" else replacedId + } + + def unquotedString: String = { + if (space.nonEmpty) s"${space.mkString(".")}.$table" else table + } + + def toSpark : org.apache.spark.sql.catalyst.TableIdentifier = { + org.apache.spark.sql.catalyst.TableIdentifier(table, database) + } + + def quotedDatabase : Option[String] = if (space.nonEmpty) Some(space.map(quoteIdentifier).mkString(".")) else None + def unquotedDatabase : Option[String] = if (space.nonEmpty) Some(space.mkString(".")) else None + def database : Option[String] = unquotedDatabase + + override def toString: String = quotedString +} diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala index 69404c4f5..258960679 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala @@ -23,13 +23,11 @@ import java.util.Locale import org.apache.commons.lang3.StringUtils import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.jdbc.JdbcType import org.apache.spark.sql.types.StructType -import com.dimajix.common.MapIgnoreCase import com.dimajix.common.SetIgnoreCase import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange @@ -39,6 +37,7 @@ import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause @@ -163,8 +162,8 @@ abstract class BaseDialect extends SqlDialect { * @return */ override def quote(table:TableIdentifier) : String = { - if (table.database.isDefined) - quoteIdentifier(table.database.get) + "." + quoteIdentifier(table.table) + if (table.space.nonEmpty) + table.space.map(quoteIdentifier).mkString(".") + "." + quoteIdentifier(table.table) else quoteIdentifier(table.table) } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/DerbyDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/DerbyDialect.scala index 19a6aaac8..23c8f293f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/DerbyDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/DerbyDialect.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,11 @@ package com.dimajix.flowman.jdbc -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.jdbc.JdbcType import com.dimajix.flowman.catalog.TableChange -import com.dimajix.flowman.catalog.TableChange.AddColumn -import com.dimajix.flowman.catalog.TableChange.DropColumn -import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment -import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.types.BooleanType import com.dimajix.flowman.types.ByteType import com.dimajix.flowman.types.DecimalType @@ -45,8 +41,8 @@ object DerbyDialect extends BaseDialect { * @return */ override def quote(table:TableIdentifier) : String = { - if (table.database.isDefined) - table.database.get + "." + table.table + if (table.space.nonEmpty) + table.space.mkString(".") + "." + table.table else table.table } diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/HiveDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/HiveDialect.scala index 333edd190..f9a897396 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/HiveDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/HiveDialect.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,11 @@ package com.dimajix.flowman.jdbc object HiveDialect extends BaseDialect { override def canHandle(url : String): Boolean = url.startsWith("jdbc:hive") + def quote(table:org.apache.spark.sql.catalyst.TableIdentifier): String = { + table.database.map(db => quoteIdentifier(db) + "." + quoteIdentifier(table.table)) + .getOrElse(quoteIdentifier(table.table)) + } + /** * Quotes the identifier. This is used to put quotes around the identifier in case the column * name is a reserved keyword, or in case it contains characters that require quotes (e.g. space). diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index 05f0a0781..5b86a7d2c 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -30,7 +30,6 @@ import scala.util.Try import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils.createConnectionFactory import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils.savePartition @@ -55,6 +54,7 @@ import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.types.Field diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala index 36ade3d23..0324fdb3f 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MsSqlServerDialect.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,11 @@ package com.dimajix.flowman.jdbc -import java.sql.SQLFeatureNotSupportedException import java.util.Locale -import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.jdbc.JdbcType -import org.apache.spark.sql.types.StructType -import com.dimajix.flowman.execution.MergeClause +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.types.BinaryType import com.dimajix.flowman.types.BooleanType import com.dimajix.flowman.types.FieldType diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala index 495c8ed97..4581a21fb 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/MySQLDialect.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2019 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,7 @@ package com.dimajix.flowman.jdbc import java.sql.SQLFeatureNotSupportedException import java.sql.Types -import org.apache.spark.sql.catalyst.TableIdentifier - +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.types.FieldType import com.dimajix.flowman.types.LongType import com.dimajix.flowman.types.BooleanType diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlDialect.scala index e8d75268f..efb0a8f73 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlDialect.scala @@ -16,11 +16,10 @@ package com.dimajix.flowman.jdbc -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.jdbc.JdbcType -import org.apache.spark.sql.types.DataType import com.dimajix.flowman.catalog.TableChange +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.types.FieldType diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala index 567cb1285..c42b6630e 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/SqlStatements.scala @@ -17,10 +17,10 @@ package com.dimajix.flowman.jdbc import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.StructType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.MergeClause diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala b/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala index 6696f9650..fa3a07555 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/model/ResourceIdentifier.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import java.util.regex.Pattern import scala.annotation.tailrec import org.apache.hadoop.fs.Path -import org.apache.spark.sql.catalyst.TableIdentifier +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.hadoop.GlobPattern @@ -38,13 +38,13 @@ object ResourceIdentifier { def ofHiveDatabase(database:String): RegexResourceIdentifier = RegexResourceIdentifier("hiveDatabase", database) def ofHiveTable(table:TableIdentifier): RegexResourceIdentifier = - ofHiveTable(table.table, table.database) + ofHiveTable(table.table, table.space.headOption) def ofHiveTable(table:String): RegexResourceIdentifier = RegexResourceIdentifier("hiveTable", table) def ofHiveTable(table:String, database:Option[String]): RegexResourceIdentifier = RegexResourceIdentifier("hiveTable", fqTable(table, database)) def ofHivePartition(table:TableIdentifier, partition:Map[String,Any]): RegexResourceIdentifier = - ofHivePartition(table.table, table.database, partition) + ofHivePartition(table.table, table.space.headOption, partition) def ofHivePartition(table:String, partition:Map[String,Any]): RegexResourceIdentifier = RegexResourceIdentifier("hiveTablePartition", table, partition.map { case(k,v) => k -> v.toString }) def ofHivePartition(table:String, database:Option[String], partition:Map[String,Any]): RegexResourceIdentifier = @@ -52,13 +52,13 @@ object ResourceIdentifier { def ofJdbcDatabase(database:String): RegexResourceIdentifier = RegexResourceIdentifier("jdbcDatabase", database) def ofJdbcTable(table:TableIdentifier): RegexResourceIdentifier = - ofJdbcTable(table.table, table.database) + ofJdbcTable(table.table, table.space.headOption) def ofJdbcTable(table:String, database:Option[String]): RegexResourceIdentifier = RegexResourceIdentifier("jdbcTable", fqTable(table, database)) def ofJdbcQuery(query:String): SimpleResourceIdentifier = SimpleResourceIdentifier("jdbcQuery", "") def ofJdbcTablePartition(table:TableIdentifier, partition:Map[String,Any]): RegexResourceIdentifier = - ofJdbcTablePartition(table.table, table.database, partition) + ofJdbcTablePartition(table.table, table.space.headOption, partition) def ofJdbcTablePartition(table:String, database:Option[String], partition:Map[String,Any]): RegexResourceIdentifier = RegexResourceIdentifier("jdbcTablePartition", fqTable(table, database), partition.map { case(k,v) => k -> v.toString }) def ofURL(url:URL): RegexResourceIdentifier = diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala index 294040b72..1cd47e068 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableChangeTest.scala @@ -16,7 +16,6 @@ package com.dimajix.flowman.catalog -import org.apache.spark.sql.catalyst.TableIdentifier import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableIdentifierTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableIdentifierTest.scala new file mode 100644 index 000000000..a58d4b5e0 --- /dev/null +++ b/flowman-core/src/test/scala/com/dimajix/flowman/catalog/TableIdentifierTest.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2022 Kaya Kupferschmidt + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dimajix.flowman.catalog + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + + +class TableIdentifierTest extends AnyFlatSpec with Matchers { + "The TableIdentifier" should "work without a namespace" in { + val id = TableIdentifier("some_table") + id.toString should be ("`some_table`") + id.quotedString should be ("`some_table`") + id.unquotedString should be ("some_table") + id.database should be (None) + id.quotedDatabase should be (None) + id.unquotedDatabase should be (None) + id.toSpark should be (org.apache.spark.sql.catalyst.TableIdentifier("some_table", None)) + } + + it should "work with a single namespace" in { + val id = TableIdentifier("some_table", Some("db")) + id.toString should be ("`db`.`some_table`") + id.quotedString should be ("`db`.`some_table`") + id.unquotedString should be ("db.some_table") + id.database should be (Some("db")) + id.quotedDatabase should be (Some("`db`")) + id.unquotedDatabase should be (Some("db")) + id.toSpark should be (org.apache.spark.sql.catalyst.TableIdentifier("some_table", Some("db"))) + } + + it should "work with a nested namespace" in { + val id = TableIdentifier("some_table", Seq("db","ns")) + id.toString should be ("`db`.`ns`.`some_table`") + id.quotedString should be ("`db`.`ns`.`some_table`") + id.unquotedString should be ("db.ns.some_table") + id.database should be (Some("db.ns")) + id.quotedDatabase should be (Some("`db`.`ns`")) + id.unquotedDatabase should be (Some("db.ns")) + id.toSpark should be (org.apache.spark.sql.catalyst.TableIdentifier("some_table", Some("db.ns"))) + } +} diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala index b8def97e7..afc25d3dc 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/BaseDialectTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.dimajix.flowman.jdbc -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.expr import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.types.StringType @@ -28,6 +27,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.UpdateClause diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala index ab1c454f0..d4d3c919d 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/DerbyJdbcTest.scala @@ -18,12 +18,12 @@ package com.dimajix.flowman.jdbc import java.nio.file.Path -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala index 99a3bc10e..006f3db3d 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/H2JdbcTest.scala @@ -20,7 +20,6 @@ import java.nio.file.Path import java.util.Properties import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.apache.spark.sql.functions.col import org.apache.spark.sql.functions.expr @@ -29,9 +28,9 @@ import org.apache.spark.sql.types.StructField import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause diff --git a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala index ec2047f96..84b6dc650 100644 --- a/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala +++ b/flowman-core/src/test/scala/com/dimajix/flowman/jdbc/JdbcUtilsTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2018-2021 Kaya Kupferschmidt + * Copyright 2018-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ package com.dimajix.flowman.jdbc import java.nio.file.Path -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -26,6 +25,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.flowman.catalog import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.types.BooleanType import com.dimajix.flowman.types.Field diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala index d676f2a16..a6e35e9ed 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveTable.scala @@ -17,8 +17,8 @@ package com.dimajix.flowman.dsl.relation import org.apache.hadoop.fs.Path -import org.apache.spark.sql.catalyst.TableIdentifier +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Relation @@ -48,7 +48,7 @@ case class HiveTable( props, schema = schema.map(s => s.instantiate(props.context)), partitions = partitions, - table = TableIdentifier(table, database), + table = TableIdentifier(table, database.toSeq), external = external, location = location, format = format, diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala index 2d708c648..6429193c7 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveUnionTable.scala @@ -17,8 +17,8 @@ package com.dimajix.flowman.dsl.relation import org.apache.hadoop.fs.Path -import org.apache.spark.sql.catalyst.TableIdentifier +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Relation diff --git a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala index 906f0c8f4..02d0155f1 100644 --- a/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala +++ b/flowman-dsl/src/main/scala/com/dimajix/flowman/dsl/relation/HiveView.scala @@ -16,8 +16,7 @@ package com.dimajix.flowman.dsl.relation -import org.apache.spark.sql.catalyst.TableIdentifier - +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.dsl.RelationGen import com.dimajix.flowman.model.MappingOutputIdentifier import com.dimajix.flowman.model.PartitionField diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala index 26b28acac..0acec4fba 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaFileRelation.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,11 +22,8 @@ import java.nio.file.FileAlreadyExistsException import com.fasterxml.jackson.annotation.JsonProperty import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path -import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.catalog.DeltaTableV2 -import org.apache.spark.sql.functions.col import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.streaming.Trigger import org.apache.spark.sql.types.StructType @@ -39,9 +36,9 @@ import com.dimajix.common.Yes import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution -import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.MigrationStrategy import com.dimajix.flowman.execution.OutputMode @@ -225,8 +222,8 @@ case class DeltaFileRelation( val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) - val sourceTable = TableDefinition(TableIdentifier(""), sourceSchema.fields) - val targetTable = TableDefinition(TableIdentifier(""), targetSchema.fields) + val sourceTable = TableDefinition(TableIdentifier.empty, sourceSchema.fields) + val targetTable = TableDefinition(TableIdentifier.empty, targetSchema.fields) !TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) } else { diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala index f6cfa135d..03cd6be33 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaRelation.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.catalog.DeltaTableV2 import org.apache.spark.sql.delta.commands.AlterTableAddColumnsDeltaCommand import org.apache.spark.sql.delta.commands.AlterTableChangeColumnDeltaCommand @@ -34,6 +33,7 @@ import org.apache.spark.sql.types.StructType import org.slf4j.LoggerFactory import com.dimajix.common.SetIgnoreCase +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableChange.AddColumn import com.dimajix.flowman.catalog.TableChange.DropColumn @@ -126,8 +126,8 @@ abstract class DeltaRelation(options: Map[String,String], mergeKey: Seq[String]) val table = deltaCatalogTable(execution) val sourceSchema = com.dimajix.flowman.types.StructType.of(table.schema()) val targetSchema = com.dimajix.flowman.types.SchemaUtils.replaceCharVarchar(fullSchema.get) - val sourceTable = TableDefinition(TableIdentifier(""), sourceSchema.fields) - val targetTable = TableDefinition(TableIdentifier(""), targetSchema.fields) + val sourceTable = TableDefinition(TableIdentifier.empty, sourceSchema.fields) + val targetTable = TableDefinition(TableIdentifier.empty, targetSchema.fields) val requiresMigration = TableChange.requiresMigration(sourceTable, targetTable, migrationPolicy) diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala index 06df5a95f..86e715bb8 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/relation/DeltaTableRelation.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,7 @@ package com.dimajix.flowman.spec.relation import com.fasterxml.jackson.annotation.JsonProperty import io.delta.tables.DeltaTable import org.apache.hadoop.fs.Path -import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.delta.catalog.DeltaTableV2 @@ -37,9 +35,9 @@ import com.dimajix.common.Yes import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution -import com.dimajix.flowman.execution.MergeClause import com.dimajix.flowman.execution.MigrationFailedException import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.MigrationStrategy @@ -83,7 +81,7 @@ case class DeltaTableRelation( * @return */ override def requires: Set[ResourceIdentifier] = { - table.database.map(ResourceIdentifier.ofHiveDatabase).toSet + table.space.headOption.map(ResourceIdentifier.ofHiveDatabase).toSet } /** @@ -177,7 +175,7 @@ case class DeltaTableRelation( */ override def readStream(execution: Execution): DataFrame = { logger.info(s"Streaming from Delta table relation '$identifier' at $table") - val location = DeltaUtils.getLocation(execution, table) + val location = DeltaUtils.getLocation(execution, table.toSpark) readStreamFrom(execution, location) } @@ -190,7 +188,7 @@ case class DeltaTableRelation( */ override def writeStream(execution: Execution, df: DataFrame, mode: OutputMode, trigger: Trigger, checkpointLocation: Path): StreamingQuery = { logger.info(s"Streaming to Delta table relation '$identifier' $table") - val location = DeltaUtils.getLocation(execution, table) + val location = DeltaUtils.getLocation(execution, table.toSpark) writeStreamTo(execution, df, location, mode, trigger, checkpointLocation) } @@ -259,7 +257,7 @@ case class DeltaTableRelation( } else if (partitions.nonEmpty) { val partitionSpec = PartitionSchema(partitions).spec(partition) - DeltaUtils.isLoaded(execution, table, partitionSpec) + DeltaUtils.isLoaded(execution, table.toSpark, partitionSpec) } else { val location = catalog.getTableLocation(table) @@ -287,7 +285,7 @@ case class DeltaTableRelation( DeltaUtils.createTable( execution, - Some(table), + Some(table.toSpark), location, sparkSchema, partitions, diff --git a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala index 272eb05fe..9ccc9c092 100644 --- a/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala +++ b/flowman-plugins/delta/src/main/scala/com/dimajix/flowman/spec/target/DeltaVacuumTarget.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,11 +21,9 @@ import java.time.Duration import com.fasterxml.jackson.annotation.JsonProperty import io.delta.tables.DeltaTable import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.functions import org.apache.spark.sql.functions.col -import org.apache.spark.sql.functions.count import org.apache.spark.sql.functions.lit import org.slf4j.LoggerFactory @@ -127,7 +125,7 @@ case class DeltaVacuumTarget( private def compact(deltaTable:DeltaTable) : Unit = { val spark = deltaTable.toDF.sparkSession val deltaLog = relation.value match { - case table:DeltaTableRelation => DeltaLog.forTable(spark, table.table) + case table:DeltaTableRelation => DeltaLog.forTable(spark, table.table.toSpark) case files:DeltaFileRelation => DeltaLog.forTable(spark, files.location.toString) case rel:Relation => throw new IllegalArgumentException(s"DeltaVacuumTarget only supports relations of type deltaTable and deltaFiles, but it was given relation '${rel.identifier}' of kind '${rel.kind}'") } diff --git a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala index 9a45bbdf8..b96369133 100644 --- a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala +++ b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/relation/DeltaTableRelationTest.scala @@ -1,5 +1,5 @@ /* - * Copyright 2021 Kaya Kupferschmidt + * Copyright 2021-2022 Kaya Kupferschmidt * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.apache.spark.sql.Row import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.CatalogTableType @@ -40,6 +39,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause import com.dimajix.flowman.execution.MigrationFailedException @@ -50,14 +50,12 @@ import com.dimajix.flowman.execution.Session import com.dimajix.flowman.execution.UpdateClause import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Relation -import com.dimajix.flowman.model.RelationIdentifier import com.dimajix.flowman.model.Schema import com.dimajix.flowman.spec.ObjectMapper import com.dimajix.flowman.spec.schema.EmbeddedSchema import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.SingleValue import com.dimajix.flowman.{types => ftypes} -import com.dimajix.spark.sql.SchemaUtils import com.dimajix.spark.sql.streaming.StreamingUtils import com.dimajix.spark.testing.LocalSparkSession import com.dimajix.spark.testing.QueryTest @@ -139,7 +137,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("delta_table", Some("default"))) - table_1.identifier should be (TableIdentifier("delta_table", Some("default"))) + table_1.identifier should be (TableIdentifier("delta_table", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) table_1.schema should be (StructType(Seq())) table_1.dataSchema should be (StructType(Seq())) @@ -251,7 +249,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("delta_table2", Some("default"))) - table_1.identifier should be (TableIdentifier("delta_table2", Some("default"))) + table_1.identifier should be (TableIdentifier("delta_table2", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.EXTERNAL) table_1.schema should be (StructType(Seq())) table_1.dataSchema should be (StructType(Seq())) @@ -1264,7 +1262,7 @@ class DeltaTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSe // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("delta_table", Some("default"))) - table_1.identifier should be (TableIdentifier("delta_table", Some("default"))) + table_1.identifier should be (TableIdentifier("delta_table", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) table_1.schema should be (StructType(Seq())) table_1.dataSchema should be (StructType(Seq())) diff --git a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala index 03d792877..7434ffafa 100644 --- a/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala +++ b/flowman-plugins/delta/src/test/scala/com/dimajix/flowman/spec/target/DeltaVacuumTargetTest.scala @@ -22,16 +22,18 @@ import java.time.Duration import io.delta.sql.DeltaSparkSessionExtension import org.apache.hadoop.fs.Path import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.functions.col +import org.apache.spark.sql.{types => stypes} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Unknown import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Phase import com.dimajix.flowman.execution.Session +import com.dimajix.flowman.model.PartitionField import com.dimajix.flowman.model.Prototype import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.Schema @@ -45,9 +47,6 @@ import com.dimajix.flowman.types.Field import com.dimajix.flowman.types.IntegerType import com.dimajix.flowman.types.StringType import com.dimajix.spark.testing.LocalSparkSession -import org.apache.spark.sql.{types => stypes} - -import com.dimajix.flowman.model.PartitionField class DeltaVacuumTargetTest extends AnyFlatSpec with Matchers with LocalSparkSession { diff --git a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala index c66fece31..65dac09ac 100644 --- a/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala +++ b/flowman-plugins/mssqlserver/src/main/scala/com/dimajix/flowman/spec/relation/SqlServerRelation.scala @@ -21,11 +21,10 @@ import scala.collection.mutable import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SaveMode -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions import com.dimajix.flowman.catalog -import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala index c8d73d156..49cec00ab 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/mapping/ReadHiveMapping.scala @@ -16,12 +16,17 @@ package com.dimajix.flowman.spec.mapping +import scala.collection.immutable.ListMap + import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.slf4j.LoggerFactory +import com.dimajix.jackson.ListMapDeserializer + +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.BaseMapping @@ -36,24 +41,21 @@ import com.dimajix.spark.sql.SchemaUtils case class ReadHiveMapping( instanceProperties:Mapping.Properties, - database: Option[String] = None, - table: String, + table: TableIdentifier, columns:Seq[Field] = Seq(), filter:Option[String] = None ) extends BaseMapping { private val logger = LoggerFactory.getLogger(classOf[ReadHiveMapping]) - def tableIdentifier: TableIdentifier = new TableIdentifier(table, database) - /** * Returns a list of physical resources required by this mapping. This list will only be non-empty for mappings * which actually read from physical data. * @return */ override def requires : Set[ResourceIdentifier] = { - Set(ResourceIdentifier.ofHiveTable(table, database)) ++ - database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet + Set(ResourceIdentifier.ofHiveTable(table)) ++ + table.database.map(db => ResourceIdentifier.ofHiveDatabase(db)).toSet } /** @@ -75,10 +77,10 @@ extends BaseMapping { require(input != null) val schema = if (columns.nonEmpty) Some(spark.sql.types.StructType(columns.map(_.sparkField))) else None - logger.info(s"Reading Hive table $tableIdentifier with filter '${filter.getOrElse("")}'") + logger.info(s"Reading Hive table $table with filter '${filter.getOrElse("")}'") val reader = execution.spark.read - val tableDf = reader.table(tableIdentifier.unquotedString) + val tableDf = reader.table(table.unquotedString) val df = SchemaUtils.applySchema(tableDf, schema) // Apply optional filter @@ -101,7 +103,7 @@ extends BaseMapping { StructType(columns) } else { - val tableDf = execution.spark.read.table(tableIdentifier.unquotedString) + val tableDf = execution.spark.read.table(table.unquotedString) StructType.of(tableDf.schema) } @@ -115,7 +117,8 @@ extends BaseMapping { class ReadHiveMappingSpec extends MappingSpec { @JsonProperty(value = "database", required = false) private var database: Option[String] = None @JsonProperty(value = "table", required = true) private var table: String = "" - @JsonProperty(value = "columns", required=false) private var columns:Map[String,String] = Map() + @JsonDeserialize(using = classOf[ListMapDeserializer]) // Old Jackson in old Spark doesn't support ListMap + @JsonProperty(value = "columns", required=false) private var columns:ListMap[String,String] = ListMap() @JsonProperty(value = "filter", required=false) private var filter:Option[String] = None /** @@ -126,9 +129,8 @@ class ReadHiveMappingSpec extends MappingSpec { override def instantiate(context: Context): ReadHiveMapping = { ReadHiveMapping( instanceProperties(context), - context.evaluate(database), - context.evaluate(table), - context.evaluate(columns).map { case(name,typ) => Field(name, FieldType.of(typ))}.toSeq, + TableIdentifier(context.evaluate(table), context.evaluate(database)), + columns.toSeq.map { case(name,typ) => Field(name, FieldType.of(typ))}, context.evaluate(filter) ) } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala index dfc93174f..8f162c066 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveRelation.scala @@ -17,10 +17,10 @@ package com.dimajix.flowman.spec.relation import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.slf4j.Logger import com.dimajix.common.Trilean +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.model.BaseRelation import com.dimajix.flowman.model.PartitionedRelation diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala index 364b6de53..e75a93e0f 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveTableRelation.scala @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty import org.apache.hadoop.fs.Path import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SparkShim -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat @@ -50,6 +49,7 @@ import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MigrationFailedException @@ -465,7 +465,7 @@ case class HiveTableRelation( // Configure catalog table by assembling all options val catalogTable = CatalogTable( - identifier = table, + identifier = table.toSpark, tableType = if (external) CatalogTableType.EXTERNAL @@ -697,7 +697,7 @@ class HiveTableRelationSpec extends RelationSpec with SchemaRelationSpec with Pa instanceProperties(context), schema.map(_.instantiate(context)), partitions.map(_.instantiate(context)), - TableIdentifier(context.evaluate(table), context.evaluate(database)), + TableIdentifier(context.evaluate(table), context.evaluate(database).toSeq), context.evaluate(external).toBoolean, context.evaluate(location).map(p => new Path(p)), context.evaluate(format), diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala index 79d439aea..013aca5f8 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelation.scala @@ -19,7 +19,6 @@ package com.dimajix.flowman.spec.relation import com.fasterxml.jackson.annotation.JsonProperty import org.apache.hadoop.fs.Path import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.types.StructField import org.apache.spark.sql.types.StructType @@ -30,6 +29,7 @@ import com.dimajix.common.No import com.dimajix.common.SetIgnoreCase import com.dimajix.common.Trilean import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.ExecutionException @@ -48,7 +48,6 @@ import com.dimajix.flowman.model.Relation import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Schema import com.dimajix.flowman.model.SchemaRelation -import com.dimajix.flowman.spec.schema.EmbeddedSchema import com.dimajix.flowman.transforms.SchemaEnforcer import com.dimajix.flowman.transforms.UnionTransformer import com.dimajix.flowman.types.FieldValue @@ -519,7 +518,7 @@ case class HiveUnionTableRelation( private def doMigrateAlterTable(execution:Execution, table:CatalogTable, rawMissingFields:Seq[StructField], migrationStrategy:MigrationStrategy) : Unit = { doMigrate(migrationStrategy) { val catalog = execution.catalog - val id = table.identifier + val id = TableIdentifier.of(table.identifier) val targetSchema = table.dataSchema val missingFields = HiveTableRelation.cleanupFields(rawMissingFields) val newSchema = StructType(targetSchema.fields ++ missingFields) diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala index 420decc8d..0e10b1370 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/HiveViewRelation.scala @@ -18,12 +18,12 @@ package com.dimajix.flowman.spec.relation import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.slf4j.LoggerFactory import com.dimajix.common.Trilean import com.dimajix.flowman.catalog.HiveCatalog +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.Execution import com.dimajix.flowman.execution.MappingUtils diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 47c96e00d..017e0f5e4 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -29,7 +29,6 @@ import com.fasterxml.jackson.annotation.JsonProperty import org.apache.spark.sql.Column import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SaveMode -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.Expression @@ -43,6 +42,7 @@ import com.dimajix.common.SetIgnoreCase import com.dimajix.common.Trilean import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.Context import com.dimajix.flowman.execution.DeleteClause @@ -87,9 +87,9 @@ class JdbcRelationBase( primaryKey: Seq[String] = Seq.empty, indexes: Seq[TableIndex] = Seq.empty ) extends BaseRelation with PartitionedRelation with SchemaRelation { - protected val logger : Logger = LoggerFactory.getLogger(getClass) - protected val tableIdentifier = table.getOrElse(TableIdentifier("")) - protected lazy val tableDefinition : Option[TableDefinition] = { + protected val logger: Logger = LoggerFactory.getLogger(getClass) + protected val tableIdentifier: TableIdentifier = table.getOrElse(TableIdentifier.empty) + protected lazy val tableDefinition: Option[TableDefinition] = { schema.map { schema => val pk = if (primaryKey.nonEmpty) primaryKey else schema.primaryKey diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala index 39c657625..829f7ac18 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/mapping/ReadHiveTest.scala @@ -16,7 +16,6 @@ package com.dimajix.flowman.spec.mapping -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.types.DoubleType import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.types.StringType @@ -25,6 +24,7 @@ import org.apache.spark.sql.types.StructType import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.Session import com.dimajix.flowman.model.Mapping import com.dimajix.flowman.model.MappingIdentifier @@ -58,8 +58,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { mapping shouldBe a[ReadHiveMapping] val rrm = mapping.asInstanceOf[ReadHiveMapping] - rrm.database should be (Some("default")) - rrm.table should be ("t0") + rrm.table should be (TableIdentifier("t0", Some("default"))) rrm.filter should be (Some("landing_date > 123")) } @@ -84,8 +83,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { val mapping = ReadHiveMapping( Mapping.Properties(context, "readHive"), - Some("default"), - "lala_0007" + TableIdentifier("lala_0007", Some("default")) ) mapping.requires should be (Set( @@ -130,8 +128,7 @@ class ReadHiveTest extends AnyFlatSpec with Matchers with LocalSparkSession { val mapping = ReadHiveMapping( Mapping.Properties(context, "readHive"), - Some("default"), - "lala_0007", + table = TableIdentifier("lala_0007", Some("default")), columns = Seq( Field("int_col", ftypes.DoubleType) ) diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala index 4ffd8afae..a51b1a2bd 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveTableRelationTest.scala @@ -21,7 +21,6 @@ import java.io.File import org.apache.hadoop.fs.Path import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException @@ -40,6 +39,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.MigrationFailedException import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.MigrationStrategy @@ -112,7 +112,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0001", Some("default"))) table.provider should be (Some("hive")) table.comment should be(Some("This is a test table")) - table.identifier should be (TableIdentifier("lala_0001", Some("default"))) + table.identifier should be (TableIdentifier("lala_0001", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -194,7 +194,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0002", Some("default"))) table.provider should be (Some("hive")) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0002", Some("default"))) + table.identifier should be (TableIdentifier("lala_0002", Some("default")).toSpark) table.tableType should be (CatalogTableType.EXTERNAL) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -262,7 +262,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0003", Some("default"))) table.provider should be (Some("hive")) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0003", Some("default"))) + table.identifier should be (TableIdentifier("lala_0003", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -326,7 +326,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0004", Some("default"))) table.provider should be (Some("hive")) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0004", Some("default"))) + table.identifier should be (TableIdentifier("lala_0004", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -385,7 +385,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0005", Some("default"))) table.provider should be (Some("hive")) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0005", Some("default"))) + table.identifier should be (TableIdentifier("lala_0005", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -426,7 +426,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // == Check ================================================================================================= val table = session.catalog.getTable(TableIdentifier("lala_0006", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0006", Some("default"))) + table.identifier should be (TableIdentifier("lala_0006", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -469,7 +469,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // == Check ================================================================================================= val table = session.catalog.getTable(TableIdentifier("lala_0007", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0007", Some("default"))) + table.identifier should be (TableIdentifier("lala_0007", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -516,7 +516,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // == Check ================================================================================================= val table = session.catalog.getTable(TableIdentifier("lala_0007", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0007", Some("default"))) + table.identifier should be (TableIdentifier("lala_0007", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) SchemaUtils.dropMetadata(table.schema) should be (StructType(Seq( StructField("str_col", StringType), @@ -559,7 +559,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // == Check ================================================================================================= val table = session.catalog.getTable(TableIdentifier("lala_0008", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0008", Some("default"))) + table.identifier should be (TableIdentifier("lala_0008", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -603,7 +603,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // == Check ================================================================================================= val table = session.catalog.getTable(TableIdentifier("lala_0009", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0009", Some("default"))) + table.identifier should be (TableIdentifier("lala_0009", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -860,7 +860,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes val table = session.catalog.getTable(TableIdentifier("lala_0012", Some("default"))) table.comment should be(None) - table.identifier should be (TableIdentifier("lala_0012", Some("default"))) + table.identifier should be (TableIdentifier("lala_0012", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType( StructField("str_col", StringType) :: @@ -1462,7 +1462,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // Inspect Hive table val table = session.catalog.getTable(TableIdentifier("lala", Some("default"))) - table.identifier should be (TableIdentifier("lala", Some("default"))) + table.identifier should be (TableIdentifier("lala", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -1534,7 +1534,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("some_table", Some("default"))) - table_1.identifier should be (TableIdentifier("some_table", Some("default"))) + table_1.identifier should be (TableIdentifier("some_table", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(table_1.schema) should be(StructType(Seq( @@ -1646,7 +1646,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) - table_1.identifier should be (TableIdentifier("lala", Some("default"))) + table_1.identifier should be (TableIdentifier("lala", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) table_1.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -1681,7 +1681,7 @@ class HiveTableRelationTest extends AnyFlatSpec with Matchers with LocalSparkSes // Inspect Hive table val table_2 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) - table_2.identifier should be (TableIdentifier("lala", Some("default"))) + table_2.identifier should be (TableIdentifier("lala", Some("default")).toSpark) table_2.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(table_2.schema) should be(StructType(Seq( diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala index d41080c6a..f30fd079a 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveUnionTableRelationTest.scala @@ -17,7 +17,6 @@ package com.dimajix.flowman.spec.relation import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.analysis.PartitionAlreadyExistsException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException @@ -37,6 +36,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.MigrationStrategy import com.dimajix.flowman.execution.OutputMode @@ -118,7 +118,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view.provider should be (None) view.comment should be (None) - view.identifier should be (TableIdentifier("lala", Some("default"))) + view.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view.tableType should be (CatalogTableType.VIEW) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(view.schema) should be(StructType(Seq( @@ -143,7 +143,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val table = session.catalog.getTable(TableIdentifier("lala_1", Some("default"))) table.provider should be (Some("hive")) table.comment should be(Some("This is a test table")) - table.identifier should be (TableIdentifier("lala_1", Some("default"))) + table.identifier should be (TableIdentifier("lala_1", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(table.schema) should be(StructType(Seq( @@ -352,7 +352,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view.provider should be (None) view.comment should be (None) - view.identifier should be (TableIdentifier("lala", Some("default"))) + view.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view.tableType should be (CatalogTableType.VIEW) view.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -366,7 +366,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val table = session.catalog.getTable(TableIdentifier("lala_1", Some("default"))) table.provider should be (Some("hive")) table.comment should be(Some("This is a test table")) - table.identifier should be (TableIdentifier("lala_1", Some("default"))) + table.identifier should be (TableIdentifier("lala_1", Some("default")).toSpark) table.tableType should be (CatalogTableType.MANAGED) table.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -701,7 +701,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view_1 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view_1.provider should be (None) view_1.comment should be (None) - view_1.identifier should be (TableIdentifier("lala", Some("default"))) + view_1.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view_1.tableType should be (CatalogTableType.VIEW) view_1.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -713,7 +713,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("lala_1", Some("default"))) - table_1.identifier should be (TableIdentifier("lala_1", Some("default"))) + table_1.identifier should be (TableIdentifier("lala_1", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) table_1.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -754,7 +754,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view_2 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view_2.provider should be (None) view_2.comment should be (None) - view_2.identifier should be (TableIdentifier("lala", Some("default"))) + view_2.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view_2.tableType should be (CatalogTableType.VIEW) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(view_2.schema) should be(StructType(Seq( @@ -777,7 +777,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa // Inspect Hive table val table_2 = session.catalog.getTable(TableIdentifier("lala_1", Some("default"))) - table_2.identifier should be (TableIdentifier("lala_1", Some("default"))) + table_2.identifier should be (TableIdentifier("lala_1", Some("default")).toSpark) table_2.tableType should be (CatalogTableType.MANAGED) if (hiveVarcharSupported) { SchemaUtils.dropMetadata(table_2.schema) should be(StructType(Seq( @@ -922,7 +922,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view_1 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view_1.provider should be (None) view_1.comment should be (None) - view_1.identifier should be (TableIdentifier("lala", Some("default"))) + view_1.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view_1.tableType should be (CatalogTableType.VIEW) view_1.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -934,7 +934,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa // Inspect Hive table val table_1 = session.catalog.getTable(TableIdentifier("lala_1", Some("default"))) - table_1.identifier should be (TableIdentifier("lala_1", Some("default"))) + table_1.identifier should be (TableIdentifier("lala_1", Some("default")).toSpark) table_1.tableType should be (CatalogTableType.MANAGED) table_1.schema should be (StructType(Seq( StructField("str_col", StringType), @@ -975,7 +975,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa val view_2 = session.catalog.getTable(TableIdentifier("lala", Some("default"))) view_2.provider should be (None) view_2.comment should be (None) - view_2.identifier should be (TableIdentifier("lala", Some("default"))) + view_2.identifier should be (TableIdentifier("lala", Some("default")).toSpark) view_2.tableType should be (CatalogTableType.VIEW) view_2.schema should be (StructType( StructField("str_col", StringType) :: @@ -988,7 +988,7 @@ class HiveUnionTableRelationTest extends AnyFlatSpec with Matchers with LocalSpa // Inspect Hive table val table_2 = session.catalog.getTable(TableIdentifier("lala_2", Some("default"))) - table_2.identifier should be (TableIdentifier("lala_2", Some("default"))) + table_2.identifier should be (TableIdentifier("lala_2", Some("default")).toSpark) table_2.tableType should be (CatalogTableType.MANAGED) table_2.schema should be (StructType(Seq( StructField("str_col", StringType), diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala index dad05ad13..f2d300019 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/HiveViewRelationTest.scala @@ -16,7 +16,6 @@ package com.dimajix.flowman.spec.relation -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.CatalogTableType @@ -25,6 +24,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.execution.MigrationPolicy import com.dimajix.flowman.execution.Session import com.dimajix.flowman.model.MappingOutputIdentifier diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala index 04c2dc475..9759c46b4 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala @@ -23,11 +23,9 @@ import java.sql.Statement import java.util.Properties import scala.collection.JavaConverters._ -import scala.collection.mutable import scala.util.control.NonFatal import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.jdbc.DriverRegistry import org.apache.spark.sql.execution.datasources.jdbc.DriverWrapper import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions @@ -41,6 +39,7 @@ import org.scalatest.matchers.should.Matchers import com.dimajix.common.No import com.dimajix.common.Yes import com.dimajix.flowman.catalog.TableDefinition +import com.dimajix.flowman.catalog.TableIdentifier import com.dimajix.flowman.catalog.TableIndex import com.dimajix.flowman.execution.DeleteClause import com.dimajix.flowman.execution.InsertClause @@ -60,7 +59,6 @@ import com.dimajix.flowman.model.ResourceIdentifier import com.dimajix.flowman.model.Schema import com.dimajix.flowman.model.ValueConnectionReference import com.dimajix.flowman.spec.ObjectMapper -import com.dimajix.flowman.spec.connection.JdbcConnection import com.dimajix.flowman.spec.schema.EmbeddedSchema import com.dimajix.flowman.types.DateType import com.dimajix.flowman.types.DoubleType From 6e169c2442a2f69f5d579aa85d7451421a952d7e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 28 Feb 2022 12:42:55 +0100 Subject: [PATCH 90/95] Fix wrong deserialization of Sequence files with Text types --- .../spark/sql/sources/sequencefile/SequenceFileOptions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowman-spark-extensions/src/main/scala/com/dimajix/spark/sql/sources/sequencefile/SequenceFileOptions.scala b/flowman-spark-extensions/src/main/scala/com/dimajix/spark/sql/sources/sequencefile/SequenceFileOptions.scala index abea493ca..c2414b4cb 100644 --- a/flowman-spark-extensions/src/main/scala/com/dimajix/spark/sql/sources/sequencefile/SequenceFileOptions.scala +++ b/flowman-spark-extensions/src/main/scala/com/dimajix/spark/sql/sources/sequencefile/SequenceFileOptions.scala @@ -93,7 +93,7 @@ object WritableConverter { case StringType => WritableConverter( classOf[Text], classOf[String], - (w:Writable) => UTF8String.fromBytes(w.asInstanceOf[Text].getBytes), + (w:Writable) => UTF8String.fromBytes(w.asInstanceOf[Text].copyBytes()), (row:InternalRow) => if (row.isNullAt(idx)) new Text() else new Text(row.getString(idx)), () => new Text() ) From 430ce7c070287ee37e9ae0f3c4fce867636b43da Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Mon, 28 Feb 2022 15:29:33 +0100 Subject: [PATCH 91/95] Fix NPE when retrieving indexes from JDBC tables --- .../com/dimajix/flowman/jdbc/JdbcUtils.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala index 5b86a7d2c..d2965bc1a 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/JdbcUtils.scala @@ -187,13 +187,15 @@ object JdbcUtils { private def getPrimaryKey(meta: DatabaseMetaData, table:TableIdentifier) : Seq[String] = { val pkrs = meta.getPrimaryKeys(null, table.database.orNull, table.table) - val pk = mutable.ListBuffer[String]() + val pk = mutable.ListBuffer[(Short,String)]() while(pkrs.next()) { val col = pkrs.getString(4) - pk.append(col) + val seq = pkrs.getShort(5) + // val name = pkrs.getString(6) + pk.append((seq,col)) } pkrs.close() - pk + pk.sortBy(_._1).map(_._2) } private def getIndexes(meta: DatabaseMetaData, table:TableIdentifier) : Seq[TableIndex] = { @@ -201,15 +203,16 @@ object JdbcUtils { val idxcols = mutable.ListBuffer[(String, String, Boolean)]() while(idxrs.next()) { val unique = !idxrs.getBoolean(4) - val name = idxrs.getString(6) + val name = idxrs.getString(6) // May be null for statistics val col = idxrs.getString(9) idxcols.append((name, col, unique)) } idxrs.close() - idxcols.groupBy(_._1).map { case(name,cols) => - TableIndex(name, cols.map(_._2), cols.foldLeft(false)(_ || _._3)) - }.toSeq + idxcols.filter(_._1 != null) + .groupBy(_._1).map { case(name,cols) => + TableIndex(name, cols.map(_._2), cols.foldLeft(false)(_ || _._3)) + }.toSeq } /** From e9fbb8f7dcbdf9e12ae770e3885fa4bb6beb46f1 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Mar 2022 07:04:43 +0100 Subject: [PATCH 92/95] Add support for creating/dropping indexes in migrations of JDBC tables --- .../main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala | 8 ++++++++ .../com/dimajix/flowman/spec/relation/JdbcRelation.scala | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala index 258960679..ea6c868a1 100644 --- a/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala +++ b/flowman-core/src/main/scala/com/dimajix/flowman/jdbc/BaseDialect.scala @@ -32,7 +32,11 @@ import com.dimajix.common.SetIgnoreCase import com.dimajix.flowman.catalog.PartitionSpec import com.dimajix.flowman.catalog.TableChange import com.dimajix.flowman.catalog.TableChange.AddColumn +import com.dimajix.flowman.catalog.TableChange.CreateIndex +import com.dimajix.flowman.catalog.TableChange.CreatePrimaryKey import com.dimajix.flowman.catalog.TableChange.DropColumn +import com.dimajix.flowman.catalog.TableChange.DropIndex +import com.dimajix.flowman.catalog.TableChange.DropPrimaryKey import com.dimajix.flowman.catalog.TableChange.UpdateColumnComment import com.dimajix.flowman.catalog.TableChange.UpdateColumnNullability import com.dimajix.flowman.catalog.TableChange.UpdateColumnType @@ -204,6 +208,10 @@ abstract class BaseDialect extends SqlDialect { case _:UpdateColumnNullability => true case _:UpdateColumnType => true case _:UpdateColumnComment => true + case _:CreateIndex => true + case _:DropIndex => true + case _:CreatePrimaryKey => true + case _:DropPrimaryKey => true case x:TableChange => throw new UnsupportedOperationException(s"Table change ${x} not supported") } } diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index 017e0f5e4..ccfccea90 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -448,7 +448,9 @@ class JdbcRelationBase( } protected def doCreate(con:java.sql.Connection, options:JDBCOptions): Unit = { - logger.info(s"Creating JDBC relation '$identifier', this will create JDBC table $tableIdentifier with schema\n${this.schema.map(_.treeString).orNull}") + val pk = tableDefinition.filter(_.primaryKey.nonEmpty).map(t => s"\n Primary key ${t.primaryKey.mkString(",")}").getOrElse("") + val idx = tableDefinition.map(t => t.indexes.map(i => s"\n Index '${i.name}' on ${i.columns.mkString(",")}").foldLeft("")(_ + _)).getOrElse("") + logger.info(s"Creating JDBC relation '$identifier', this will create JDBC table $tableIdentifier with schema\n${schema.map(_.treeString).orNull}$pk$idx") tableDefinition match { case Some(table) => From 984dc62cb46d675d3961791dece89b24abcc8b6e Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Mar 2022 08:43:44 +0100 Subject: [PATCH 93/95] Implement upsert operation for JDBC --- docs/spec/relation/deltaFile.md | 3 +- docs/spec/relation/deltaTable.md | 3 +- docs/spec/relation/file.md | 9 +- docs/spec/relation/generic.md | 4 + docs/spec/relation/hiveTable.md | 1 - docs/spec/relation/hiveUnionTable.md | 15 ++- docs/spec/relation/{jdbc.md => jdbcTable.md} | 82 +++++++++++---- docs/spec/relation/kafka.md | 27 +++-- docs/spec/relation/local.md | 11 +++ docs/spec/relation/sqlserver.md | 17 +++- .../flowman/spec/relation/JdbcRelation.scala | 49 +++++---- .../flowman/spec/relation/RelationSpec.scala | 1 + .../spec/relation/JdbcRelationTest.scala | 99 +++++++++++++++++++ .../spec/relation/LocalRelationTest.scala | 4 + 14 files changed, 254 insertions(+), 71 deletions(-) rename docs/spec/relation/{jdbc.md => jdbcTable.md} (66%) diff --git a/docs/spec/relation/deltaFile.md b/docs/spec/relation/deltaFile.md index 90bf11f55..bab75bfad 100644 --- a/docs/spec/relation/deltaFile.md +++ b/docs/spec/relation/deltaFile.md @@ -99,7 +99,8 @@ The `deltaFile` relation supports the following output modes in a [`relation` ta | `overwrite_dynamic` | no | - | | `append` | yes | Append new records to the existing table | | `update` | yes | Updates existing records, either using `mergeKey` or the primary key of the specified `schema` | -| `merge` | no | - | + +In addition, the `deltaFile` relation also supports complex merge operations in a [`merge` target](../target/merge.md). ### Stream Writing In addition to batch writing, the Delta file relation also supports stream writing via the diff --git a/docs/spec/relation/deltaTable.md b/docs/spec/relation/deltaTable.md index aa01e2f80..563ce8c03 100644 --- a/docs/spec/relation/deltaTable.md +++ b/docs/spec/relation/deltaTable.md @@ -114,7 +114,8 @@ The `deltaTable` relation supports the following output modes in a [`relation` t | `overwrite_dynamic` | no | - | | `append` | yes | Append new records to the existing table | | `update` | yes | Updates existing records, either using `mergeKey` or the primary key of the specified `schema` | -| `merge` | no | - | + +In addition, the `deltaFile` relation also supports complex merge operations in a [`merge` target](../target/merge.md). ### Stream Writing In addition to batch writing, the Delta table relation also supports stream writing via the diff --git a/docs/spec/relation/file.md b/docs/spec/relation/file.md index c69b0a048..3fec6f44f 100644 --- a/docs/spec/relation/file.md +++ b/docs/spec/relation/file.md @@ -75,8 +75,12 @@ relations: Please see the section [Partitioning](#Partitioning) below. +## Automatic Migrations +The `file` relation does not support any automatic migration like adding/removing columns. + + ## Schema Conversion -The file relation fully supports automatic schema conversion on input and output operations as described in the +The `file` relation fully supports automatic schema conversion on input and output operations as described in the corresponding section of [relations](index.md). @@ -93,7 +97,6 @@ The `file` relation supports the following output modes in a [`relation` target] | `overwrite_dynamic` | yes | Overwrite only partitions dynamically determined by the data itself | | `append` | yes | Append new records to the existing files | | `update` | no | - | -| `merge` | no | - | ### Stream Writing In addition to batch writing, the file relation also supports stream writing via the @@ -126,7 +129,7 @@ in all situations where only schema information is required. ### Partitioning -Flowman also supports partitioning, i.e. written to different sub directories. You can explicitly specify a *partition +Flowman also supports partitioning, i.e. written to different subdirectories. You can explicitly specify a *partition pattern* via the `pattern` field, but it is highly recommended to NOT explicitly set this field and let Spark manage partitions itself. This way Spark can infer partition values from directory names and will also list directories more efficiently. diff --git a/docs/spec/relation/generic.md b/docs/spec/relation/generic.md index b81a1a2d7..3b71c35f4 100644 --- a/docs/spec/relation/generic.md +++ b/docs/spec/relation/generic.md @@ -34,3 +34,7 @@ relations: * `format` **(optional)** *(string)* *(default: empty)*: Specifies the name of the Spark data source format to use. + + +## Automatic Migrations +The `generic` relation does not support any automatic migration like adding/removing columns. diff --git a/docs/spec/relation/hiveTable.md b/docs/spec/relation/hiveTable.md index 029651abe..aaeea8d29 100644 --- a/docs/spec/relation/hiveTable.md +++ b/docs/spec/relation/hiveTable.md @@ -154,7 +154,6 @@ The `hive` relation supports the following output modes in a [`relation` target] | `overwrite_dynamic` | yes | Overwrite only the partitions dynamically inferred from the data. | | `append` | yes | Append new records to the existing table | | `update` | no | - | -| `merge` | no | - | ## Remarks diff --git a/docs/spec/relation/hiveUnionTable.md b/docs/spec/relation/hiveUnionTable.md index 10a13a2de..9aefdfdea 100644 --- a/docs/spec/relation/hiveUnionTable.md +++ b/docs/spec/relation/hiveUnionTable.md @@ -116,14 +116,13 @@ following changes to a data schema are supported ## Output Modes The `hiveUnionTable` relation supports the following output modes in a [`relation` target](../target/relation.md): -|Output Mode |Supported | Comments| ---- | --- | --- -|`errorIfExists`|yes|Throw an error if the Hive table already exists| -|`ignoreIfExists`|yes|Do nothing if the Hive table already exists| -|`overwrite`|yes|Overwrite the whole table or the specified partitions| -|`append`|yes|Append new records to the existing table| -|`update`|no|-| -|`merge`|no|-| +| Output Mode | Supported | Comments | +|------------------|-----------|-------------------------------------------------------| +| `errorIfExists` | yes | Throw an error if the Hive table already exists | +| `ignoreIfExists` | yes | Do nothing if the Hive table already exists | +| `overwrite` | yes | Overwrite the whole table or the specified partitions | +| `append` | yes | Append new records to the existing table | +| `update` | no | - | ## Remarks diff --git a/docs/spec/relation/jdbc.md b/docs/spec/relation/jdbcTable.md similarity index 66% rename from docs/spec/relation/jdbc.md rename to docs/spec/relation/jdbcTable.md index a35b8c897..0d04491a3 100644 --- a/docs/spec/relation/jdbc.md +++ b/docs/spec/relation/jdbcTable.md @@ -1,6 +1,6 @@ -# JDBC Relations +# JDBC Table Relations -The JDBC relation allows you to access databases using a JDBC driver. Note that you need to put an appropriate JDBC +The `jdbcTable` relation allows you to access databases using a JDBC driver. Note that you need to put an appropriate JDBC driver onto the classpath of Flowman. This can be done by using an appropriate plugin. @@ -18,7 +18,7 @@ connections: relations: frontend_users: - kind: jdbc + kind: jdbcTable # Specify the name of the connection to use connection: frontend # Specify the table @@ -28,12 +28,15 @@ relations: file: "${project.basedir}/schema/users.avsc" primaryKey: - user_id + indexes: + - name: "users_idx0" + columns: [user_first_name, user_last_name] ``` It is also possible to directly embed the connection as follows: ```yaml relations: frontend_users: - kind: jdbc + kind: jdbcTable # Specify the name of the connection to use connection: kind: jdbc @@ -47,9 +50,38 @@ relations: For most cases, it is recommended not to embed the connection, since this prevents reusing the same connection in multiple places. +It is also possible to access the results of an arbitrary SQL query, which is executed inside the target database: +```yaml +relations: + lineitem: + kind: jdbc + connection: frontend + query: " + SELECT + CONCAT('DIR_', li.id) AS lineitem, + li.campaign_id AS campaign, + IF(c.demand_type_system = 1, 'S', IF(li.demand_type_system = 1, 'S', 'D')) AS demand_type + FROM + line_item AS li + INNER JOIN + campaign c + ON c.id = li.campaign_id + " + schema: + kind: embedded + fields: + - name: lineitem + type: string + - name: campaign + type: long + - name: demand_type + type: string +``` +The schema is still optional in this case, but it will help [mocking](mock.md) the relation for unittests. + ## Fields - * `kind` **(mandatory)** *(type: string)*: `jdbc` + * `kind` **(mandatory)** *(type: string)*: `jdbcTable` or `jdbc` * `schema` **(optional)** *(type: schema)* *(default: empty)*: Explicitly specifies the schema of the JDBC source. Alternatively Flowman will automatically @@ -75,8 +107,14 @@ as the fallback for merge/upsert operations, when no `mergeKey` and no explicit table is accessed without any specific qualification, meaning that the default database will be used or the one specified in the connection. - * `table` **(mandatory)** *(type: string)*: - Specifies the name of the table in the relational database. + * `table` **(optional)** *(type: string)*: + Specifies the name of the table in the relational database. You either need to specify this `table` property +or the `query` property. + + * `query` **(optional)** *(type: string)*: +As an alternative to directly accessing a table, you can also specify an SQL query which will be executed by the +database for retrieving data. Of course, then only read operations are possible. You either need to specify this +`query` property or the `table` property. * `properties` **(optional)** *(type: map:string)* *(default: empty)*: Specifies any additional properties passed to the JDBC connection. Note that both the JDBC @@ -85,6 +123,9 @@ as the fallback for merge/upsert operations, when no `mergeKey` and no explicit The connection properties are applied first, then the relation properties. This means that a relation property can overwrite a connection property if it has the same name. + * `indexes` **(optional)** *(type: list:index)* *(default: empty)*: + Specifies a list of database indexes to be created. Each index has the properties `name`, `columns` and `unique`. + ## Automatic Migrations Flowman supports some automatic migrations, specifically with the migration strategies `ALTER`, `ALTER_REPLACE` @@ -96,9 +137,11 @@ The migration strategy `ALTER` supports the following alterations for JDBC relat * Adding new columns * Dropping columns * Changing the column type +* Adding / dropping indexes +* Changing the primary key Note that although Flowman will try to apply these changes, not all SQL databases support all of these changes in -all variations. Therefore it may well be the case, that the SQL database will fail performing these changes. If +all variations. Therefore, it may well be the case, that the SQL database will fail performing these changes. If the migration strategy is set to `ALTER_REPLACE`, then Flowman will fall back to trying to replace the whole table altogether on *any* non-recoverable exception during migration. @@ -109,17 +152,18 @@ corresponding section of [relations](index.md). ## Output Modes -The `jdbc` relation supports the following output modes in a [`relation` target](../target/relation.md): - -| Output Mode | Supported | Comments | -|---------------------|-----------|-------------------------------------------------------| -| `errorIfExists` | yes | Throw an error if the JDBC table already exists | -| `ignoreIfExists` | yes | Do nothing if the JDBC table already exists | -| `overwrite` | yes | Overwrite the whole table or the specified partitions | -| `overwrite_dynamic` | no | - | -| `append` | yes | Append new records to the existing table | -| `update` | no | - | -| `merge` | no | - | +The `jdbcTable` relation supports the following output modes in a [`relation` target](../target/relation.md): + +| Output Mode | Supported | Comments | +|---------------------|-----------|--------------------------------------------------------------| +| `errorIfExists` | yes | Throw an error if the JDBC table already exists | +| `ignoreIfExists` | yes | Do nothing if the JDBC table already exists | +| `overwrite` | yes | Overwrite the whole table or the specified partitions | +| `overwrite_dynamic` | no | - | +| `append` | yes | Append new records to the existing table | +| `update` | yes | Perform upsert operations using the merge key or primary key | + +In addition, the `jdbcTable` relation also supports complex merge operations in a [`merge` target](../target/merge.md). ## Remarks diff --git a/docs/spec/relation/kafka.md b/docs/spec/relation/kafka.md index 51aa1bc3c..7193de879 100644 --- a/docs/spec/relation/kafka.md +++ b/docs/spec/relation/kafka.md @@ -61,22 +61,21 @@ List of Kafka bootstrap servers to contact. This list does not need to be exhaus ### Batch Writing The `kafa` relation supports the following output modes in a [`relation` target](../target/relation.md): -|Output Mode |Supported | Comments| ---- | --- | --- -|`errorIfExists`|yes|Throw an error if the Kafka topic already exists| -|`ignoreIfExists`|yes|Do nothing if the Kafka topic already exists| -|`overwrite`|no|-| -|`overwrite_dynamic`|no|-| -|`append`|yes|Append new records to the existing Kafka topic| -|`update`|no|-| -|`merge`|no|-| +| Output Mode | Supported | Comments | +|---------------------|-----------|--------------------------------------------------| +| `errorIfExists` | yes | Throw an error if the Kafka topic already exists | +| `ignoreIfExists` | yes | Do nothing if the Kafka topic already exists | +| `overwrite` | no | - | +| `overwrite_dynamic` | no | - | +| `append` | yes | Append new records to the existing Kafka topic | +| `update` | no | - | ### Stream Writing In addition to batch writing, the Kafka relation also supports stream writing via the [`stream` target](../target/stream.md) with the following semantics: -|Output Mode |Supported | Comments| ---- | --- | --- -|`append`|yes|Append new records from the streaming process once they don't change any more| -|`update`|yes|Append records every time they are updated| -|`complete`|no|-| +| Output Mode | Supported | Comments | +|-------------|-----------|-------------------------------------------------------------------------------| +| `append` | yes | Append new records from the streaming process once they don't change any more | +| `update` | yes | Append records every time they are updated | +| `complete` | no | - | diff --git a/docs/spec/relation/local.md b/docs/spec/relation/local.md index ed1ad8da9..0cbbfe449 100644 --- a/docs/spec/relation/local.md +++ b/docs/spec/relation/local.md @@ -62,8 +62,19 @@ whole lifecycle of the directory for you. This means that * The directory specified in `location` will be truncated or individual partitions will be dropped during `clean` phase * The directory specified in `location` tables will be removed during `destroy` phase + +## Automatic Migrations +The `local` relation does not support any automatic migration like adding/removing columns. + + ## Supported File Format +The `local` relation only supports a very limited set of file formats (currently only `CSV` files) + ### CSV + ## Partitioning + +The `local` relation also supports partitioning by storing different partitions in separate files or subdirectories. +You need to explicitly specify a *partition pattern* via the `pattern` field. diff --git a/docs/spec/relation/sqlserver.md b/docs/spec/relation/sqlserver.md index 77fb96fce..dea3cb7ed 100644 --- a/docs/spec/relation/sqlserver.md +++ b/docs/spec/relation/sqlserver.md @@ -35,6 +35,9 @@ relations: file: "${project.basedir}/schema/users.avsc" primaryKey: - user_id + indexes: + - name: "users_idx0" + columns: [user_first_name, user_last_name] ``` It is also possible to directly embed the connection as follows: ```yaml @@ -90,6 +93,9 @@ specific qualification, meaning that the default database will be used or the on The connection properties are applied first, then the relation properties. This means that a relation property can overwrite a connection property if it has the same name. + * `indexes` **(optional)** *(type: list:index)* *(default: empty)*: + Specifies a list of database indexes to be created. Each index has the properties `name`, `columns` and `unique`. + ## Automatic Migrations Flowman supports some automatic migrations, specifically with the migration strategies `ALTER`, `ALTER_REPLACE` @@ -101,9 +107,11 @@ The migration strategy `ALTER` supports the following alterations for JDBC relat * Adding new columns * Dropping columns * Changing the column type +* Adding / dropping indexes +* Changing the primary key Note that although Flowman will try to apply these changes, not all SQL databases support all of these changes in -all variations. Therefore it may well be the case, that the SQL database will fail performing these changes. If +all variations. Therefore, it may well be the case, that the SQL database will fail performing these changes. If the migration strategy is set to `ALTER_REPLACE`, then Flowman will fall back to trying to replace the whole table altogether on *any* non-recoverable exception during migration. @@ -114,7 +122,7 @@ corresponding section of [relations](index.md). ## Output Modes -The `jdbc` relation supports the following output modes in a [`relation` target](../target/relation.md): +The `sqlserver` relation supports the following output modes in a [`relation` target](../target/relation.md): | Output Mode | Supported | Comments | |---------------------|-----------|-------------------------------------------------------| @@ -123,8 +131,9 @@ The `jdbc` relation supports the following output modes in a [`relation` target] | `overwrite` | yes | Overwrite the whole table or the specified partitions | | `overwrite_dynamic` | no | - | | `append` | yes | Append new records to the existing table | -| `update` | no | - | -| `merge` | no | - | +| `update` | yes | - | + +In addition, the `sqlserver` relation also supports complex merge operations in a [`merge` target](../target/merge.md). ## Remarks diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala index ccfccea90..c55bf4428 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/JdbcRelation.scala @@ -286,7 +286,13 @@ class JdbcRelationBase( .save() } protected def doUpdate(execution: Execution, df:DataFrame): Unit = { - throw new IllegalArgumentException(s"Unsupported save mode: 'UPDATE' for generic JDBC relation.") + val mergeCondition = this.mergeCondition + val clauses = Seq( + InsertClause(), + UpdateClause() + ) + + doMerge(execution, df, mergeCondition, clauses) } /** @@ -303,30 +309,33 @@ class JdbcRelationBase( if (query.nonEmpty) throw new UnsupportedOperationException(s"Cannot write into JDBC relation '$identifier' which is defined by an SQL query") - val mergeCondition = - condition.getOrElse { - val withinPartitionKeyColumns = - if (mergeKey.nonEmpty) - mergeKey - else if (primaryKey.nonEmpty) - primaryKey - else if (schema.exists(_.primaryKey.nonEmpty)) - schema.map(_.primaryKey).get - else - throw new IllegalArgumentException(s"Merging JDBC relation '$identifier' requires primary key in schema, explicit merge key or merge condition") - (SetIgnoreCase(partitions.map(_.name)) ++ withinPartitionKeyColumns) - .toSeq - .map(c => col("source." + c) === col("target." + c)) - .reduce(_ && _) - } - - val sourceColumns = collectColumns(mergeCondition.expr, "source") ++ clauses.flatMap(c => collectColumns(df.schema, c, "source")) + val mergeCondition = condition.getOrElse(this.mergeCondition) + doMerge(execution, df, mergeCondition, clauses) + } + protected def doMerge(execution: Execution, df: DataFrame, condition:Column, clauses: Seq[MergeClause]) : Unit = { + val sourceColumns = collectColumns(condition.expr, "source") ++ clauses.flatMap(c => collectColumns(df.schema, c, "source")) val sourceDf = df.select(sourceColumns.toSeq.map(col):_*) val (url, props) = createConnectionProperties() val options = new JDBCOptions(url, tableIdentifier.unquotedString, props) val targetSchema = outputSchema(execution) - JdbcUtils.mergeTable(tableIdentifier, "target", targetSchema, sourceDf, "source", mergeCondition, clauses, options) + JdbcUtils.mergeTable(tableIdentifier, "target", targetSchema, sourceDf, "source", condition, clauses, options) + } + + protected def mergeCondition : Column = { + val withinPartitionKeyColumns = + if (mergeKey.nonEmpty) + mergeKey + else if (primaryKey.nonEmpty) + primaryKey + else if (schema.exists(_.primaryKey.nonEmpty)) + schema.map(_.primaryKey).get + else + throw new IllegalArgumentException(s"Merging JDBC relation '$identifier' requires primary key in schema, explicit merge key or merge condition") + (SetIgnoreCase(partitions.map(_.name)) ++ withinPartitionKeyColumns) + .toSeq + .map(c => col("source." + c) === col("target." + c)) + .reduce(_ && _) } /** diff --git a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala index 51d36c603..6e9484c51 100644 --- a/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala +++ b/flowman-spec/src/main/scala/com/dimajix/flowman/spec/relation/RelationSpec.scala @@ -52,6 +52,7 @@ object RelationSpec extends TypeRegistry[RelationSpec] { new JsonSubTypes.Type(name = "hiveUnionTable", value = classOf[HiveUnionTableRelationSpec]), new JsonSubTypes.Type(name = "hiveView", value = classOf[HiveViewRelationSpec]), new JsonSubTypes.Type(name = "jdbc", value = classOf[JdbcRelationSpec]), + new JsonSubTypes.Type(name = "jdbcTable", value = classOf[JdbcRelationSpec]), new JsonSubTypes.Type(name = "local", value = classOf[LocalRelationSpec]), new JsonSubTypes.Type(name = "mock", value = classOf[MockRelationSpec]), new JsonSubTypes.Type(name = "null", value = classOf[NullRelationSpec]), diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala index 9759c46b4..aa64dbe60 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/JdbcRelationTest.scala @@ -840,6 +840,105 @@ class JdbcRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession relation.loaded(execution, Map()) should be (No) } + it should "support upsert operations" in { + val db = tempDir.toPath.resolve("mydb") + val url = "jdbc:h2:" + db + val driver = "org.h2.Driver" + + val spec = + s""" + |connections: + | c0: + | kind: jdbc + | driver: $driver + | url: $url + |relations: + | t0: + | kind: jdbc + | description: "This is a test table" + | connection: c0 + | table: lala_001 + | schema: + | kind: inline + | fields: + | - name: id + | type: integer + | - name: name + | type: string + | - name: sex + | type: string + | primaryKey: ID + |""".stripMargin + val project = Module.read.string(spec).toProject("project") + + val session = Session.builder().withSparkSession(spark).build() + val execution = session.execution + val context = session.getContext(project) + + val relation = context.getRelation(RelationIdentifier("t0")) + + // == Create ================================================================================================== + relation.exists(execution) should be (No) + relation.loaded(execution, Map()) should be (No) + relation.create(execution) + relation.exists(execution) should be (Yes) + relation.read(execution).count() should be (0) + + // ===== Write Table ========================================================================================== + val tableSchema = org.apache.spark.sql.types.StructType(Seq( + StructField("id", org.apache.spark.sql.types.IntegerType), + StructField("name", org.apache.spark.sql.types.StringType), + StructField("sex", org.apache.spark.sql.types.StringType) + )) + val df0 = DataFrameBuilder.ofRows( + spark, + Seq( + Row(10, "Alice", "male"), + Row(20, "Bob", "male") + ), + tableSchema + ) + relation.write(execution, df0, mode=OutputMode.APPEND) + relation.exists(execution) should be (Yes) + relation.loaded(execution, Map()) should be (Yes) + + // ===== Read Table =========================================================================================== + val df1 = relation.read(execution) + df1.sort(col("id")).collect() should be (Seq( + Row(10, "Alice", "male"), + Row(20, "Bob", "male") + )) + + // ===== Merge Table ========================================================================================== + val updateSchema = org.apache.spark.sql.types.StructType(Seq( + StructField("id", org.apache.spark.sql.types.IntegerType), + StructField("name", org.apache.spark.sql.types.StringType), + StructField("sex", org.apache.spark.sql.types.StringType) + )) + val df2 = DataFrameBuilder.ofRows( + spark, + Seq( + Row(10, "Alice", "female"), + Row(50, "Debora", "female") + ), + updateSchema + ) + relation.write(execution, df2, mode=OutputMode.UPDATE) + + // ===== Read Table =========================================================================================== + val df3 = relation.read(execution) + df3.sort(col("id")).collect() should be (Seq( + Row(10, "Alice", "female"), + Row(20, "Bob", "male"), + Row(50, "Debora", "female") + )) + + // == Destroy ================================================================================================= + relation.destroy(execution) + relation.exists(execution) should be (No) + relation.loaded(execution, Map()) should be (No) + } + it should "support SQL queries" in { val db = tempDir.toPath.resolve("mydb") val url = "jdbc:derby:" + db + ";create=true" diff --git a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/LocalRelationTest.scala b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/LocalRelationTest.scala index 869ffdd1f..2a3839c71 100644 --- a/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/LocalRelationTest.scala +++ b/flowman-spec/src/test/scala/com/dimajix/flowman/spec/relation/LocalRelationTest.scala @@ -433,4 +433,8 @@ class LocalRelationTest extends AnyFlatSpec with Matchers with LocalSparkSession relation.loaded(execution, Map()) should be (No) relation.loaded(execution, Map("p2" -> SingleValue("2"))) should be (No) } + + it should "support using partitions without a pattern" in { + // TODO + } } From 0c244c2f08da9a75e8f20ea406735f97e2bc71e0 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Mar 2022 10:15:10 +0100 Subject: [PATCH 94/95] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af21450f0..293482aa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Version 0.22.0 +# Version 0.22.0 - 2022-03-01 * Add new `sqlserver` relation * Implement new documentation subsystem From be00e69fbfcb9ae7c69926e841a621576bcd91f5 Mon Sep 17 00:00:00 2001 From: Kaya Kupferschmidt Date: Tue, 1 Mar 2022 10:49:19 +0100 Subject: [PATCH 95/95] Update versions for release --- docker/pom.xml | 2 +- flowman-client/pom.xml | 2 +- flowman-common/pom.xml | 2 +- flowman-core/pom.xml | 2 +- flowman-dist/pom.xml | 2 +- flowman-dsl/pom.xml | 2 +- flowman-hub/pom.xml | 2 +- flowman-parent/pom.xml | 2 +- flowman-plugins/aws/pom.xml | 2 +- flowman-plugins/azure/pom.xml | 2 +- flowman-plugins/delta/pom.xml | 2 +- flowman-plugins/impala/pom.xml | 2 +- flowman-plugins/json/pom.xml | 2 +- flowman-plugins/kafka/pom.xml | 2 +- flowman-plugins/mariadb/pom.xml | 2 +- flowman-plugins/mssqlserver/pom.xml | 2 +- flowman-plugins/mysql/pom.xml | 2 +- flowman-plugins/openapi/pom.xml | 2 +- flowman-plugins/swagger/pom.xml | 2 +- flowman-scalatest-compat/pom.xml | 2 +- flowman-server-ui/pom.xml | 2 +- flowman-server/pom.xml | 2 +- flowman-spark-extensions/pom.xml | 2 +- flowman-spark-testing/pom.xml | 2 +- flowman-spec/pom.xml | 2 +- flowman-studio-ui/pom.xml | 2 +- flowman-studio/pom.xml | 2 +- flowman-testing/pom.xml | 2 +- flowman-tools/pom.xml | 2 +- pom.xml | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index 46cd47e19..edbac5d93 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-client/pom.xml b/flowman-client/pom.xml index c10464f8a..c7afe386f 100644 --- a/flowman-client/pom.xml +++ b/flowman-client/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-common/pom.xml b/flowman-common/pom.xml index 3df2b462f..91c7961c5 100644 --- a/flowman-common/pom.xml +++ b/flowman-common/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-core/pom.xml b/flowman-core/pom.xml index f2efb9630..4202ffd3a 100644 --- a/flowman-core/pom.xml +++ b/flowman-core/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-dist/pom.xml b/flowman-dist/pom.xml index 1acbc0c21..20ee89fa8 100644 --- a/flowman-dist/pom.xml +++ b/flowman-dist/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-dsl/pom.xml b/flowman-dsl/pom.xml index abf32ea0e..2c841db2b 100644 --- a/flowman-dsl/pom.xml +++ b/flowman-dsl/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-hub/pom.xml b/flowman-hub/pom.xml index 103d9cb05..0b8ff9350 100644 --- a/flowman-hub/pom.xml +++ b/flowman-hub/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-parent/pom.xml b/flowman-parent/pom.xml index 21fd7cfc5..da6aa9be0 100644 --- a/flowman-parent/pom.xml +++ b/flowman-parent/pom.xml @@ -10,7 +10,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-plugins/aws/pom.xml b/flowman-plugins/aws/pom.xml index f71183d03..776177c92 100644 --- a/flowman-plugins/aws/pom.xml +++ b/flowman-plugins/aws/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/azure/pom.xml b/flowman-plugins/azure/pom.xml index a94327f35..b473e1976 100644 --- a/flowman-plugins/azure/pom.xml +++ b/flowman-plugins/azure/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/delta/pom.xml b/flowman-plugins/delta/pom.xml index 9cac2d812..946135545 100644 --- a/flowman-plugins/delta/pom.xml +++ b/flowman-plugins/delta/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/impala/pom.xml b/flowman-plugins/impala/pom.xml index f14bc30ab..ad6ed5ff9 100644 --- a/flowman-plugins/impala/pom.xml +++ b/flowman-plugins/impala/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/json/pom.xml b/flowman-plugins/json/pom.xml index 2aa53666e..e642b9643 100644 --- a/flowman-plugins/json/pom.xml +++ b/flowman-plugins/json/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/kafka/pom.xml b/flowman-plugins/kafka/pom.xml index a1eb26ef5..ff2103606 100644 --- a/flowman-plugins/kafka/pom.xml +++ b/flowman-plugins/kafka/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/mariadb/pom.xml b/flowman-plugins/mariadb/pom.xml index 541972d99..4dd28aaf4 100644 --- a/flowman-plugins/mariadb/pom.xml +++ b/flowman-plugins/mariadb/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/mssqlserver/pom.xml b/flowman-plugins/mssqlserver/pom.xml index a82b5e09b..fb7313ff3 100644 --- a/flowman-plugins/mssqlserver/pom.xml +++ b/flowman-plugins/mssqlserver/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/mysql/pom.xml b/flowman-plugins/mysql/pom.xml index d790b4891..dd867d237 100644 --- a/flowman-plugins/mysql/pom.xml +++ b/flowman-plugins/mysql/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/openapi/pom.xml b/flowman-plugins/openapi/pom.xml index 8ced7bc7e..f17d9413f 100644 --- a/flowman-plugins/openapi/pom.xml +++ b/flowman-plugins/openapi/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-plugins/swagger/pom.xml b/flowman-plugins/swagger/pom.xml index 4a8404bd8..a4bf2afc7 100644 --- a/flowman-plugins/swagger/pom.xml +++ b/flowman-plugins/swagger/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../../pom.xml diff --git a/flowman-scalatest-compat/pom.xml b/flowman-scalatest-compat/pom.xml index 549712291..b1c03e057 100644 --- a/flowman-scalatest-compat/pom.xml +++ b/flowman-scalatest-compat/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-server-ui/pom.xml b/flowman-server-ui/pom.xml index 02a11f214..caf8dbec3 100644 --- a/flowman-server-ui/pom.xml +++ b/flowman-server-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-server/pom.xml b/flowman-server/pom.xml index a481296c4..53c141516 100644 --- a/flowman-server/pom.xml +++ b/flowman-server/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-spark-extensions/pom.xml b/flowman-spark-extensions/pom.xml index 3ae14d0c7..8c7a00250 100644 --- a/flowman-spark-extensions/pom.xml +++ b/flowman-spark-extensions/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-spark-testing/pom.xml b/flowman-spark-testing/pom.xml index 592f6c8df..307a9b514 100644 --- a/flowman-spark-testing/pom.xml +++ b/flowman-spark-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-spec/pom.xml b/flowman-spec/pom.xml index dd7a83ff7..03f1e4599 100644 --- a/flowman-spec/pom.xml +++ b/flowman-spec/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-studio-ui/pom.xml b/flowman-studio-ui/pom.xml index 77bd98025..df95bfe81 100644 --- a/flowman-studio-ui/pom.xml +++ b/flowman-studio-ui/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-studio/pom.xml b/flowman-studio/pom.xml index ce7dfd0e2..a79842ad7 100644 --- a/flowman-studio/pom.xml +++ b/flowman-studio/pom.xml @@ -9,7 +9,7 @@ flowman-root com.dimajix.flowman - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-testing/pom.xml b/flowman-testing/pom.xml index 7be01aff8..d94806b71 100644 --- a/flowman-testing/pom.xml +++ b/flowman-testing/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/flowman-tools/pom.xml b/flowman-tools/pom.xml index 59e34109e..495e992cf 100644 --- a/flowman-tools/pom.xml +++ b/flowman-tools/pom.xml @@ -9,7 +9,7 @@ com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 ../pom.xml diff --git a/pom.xml b/pom.xml index 3d8265d5f..368e356c0 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.dimajix.flowman flowman-root - 0.22.0-SNAPSHOT + 0.22.0 pom Flowman root pom A Spark based ETL tool
Schema TestQuality Check Result Remarks
${test.name}#testStatus($test)#if(${test.result})${test.result.description}#end${check.name}#testStatus($check)#if(${check.result})${check.result.description}#end