From 611d883ba4bea103adfeb8874cad6d8a2538b3d1 Mon Sep 17 00:00:00 2001 From: fei Date: Fri, 25 Aug 2023 14:37:50 +0200 Subject: [PATCH] add query as metadata --- .../services/catalogue/AtlanService.scala | 42 +++++++++++-------- .../data/mapper/app/MetabolicApp.scala | 1 - .../mapper/services/AtlanServiceTest.scala | 24 +++++------ 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala b/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala index 434245a6..dc9bd2d3 100644 --- a/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala +++ b/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala @@ -35,23 +35,35 @@ class AtlanService(token: String, baseUrl: String) extends Logging { guid match{ case "" => "" case _ => { - val last_synced = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) - val mode = mapping.environment.mode - val body = - s""" - |{ - | "Data Quality": { - | "last_synced_at" : "${last_synced}", - | "engine_type":"${mode.toString}" - | } - |} - |""".stripMargin + val body: String = generateMetadaBody(mapping) logger.info(s"Atlan Metadata Json Body ${body}") HttpRequestHandler.sendHttpPostRequest(s"https://factorial.atlan.com/api/meta/entity/guid/$guid/businessmetadata/displayName?isOverwrite=false", body, token) } } } + def generateMetadaBody(mapping: Config): String = { + val last_synced = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + val mode = mapping.environment.mode + val sql = mapping.mappings.head match { + case sqlmapping: SQLMapping => { + sqlmapping.sqlContents + } + case _ => "" + } + val body = + s""" + |{ + | "Data Quality": { + | "last_synced_at" : "${last_synced}", + | "engine_type":"${mode.toString}", + | "sql_mapping":"$sql" + | } + |} + |""".stripMargin + body + } + def setDescription(mapping: Config): String = { val outputTable = getOutputTableName(mapping) val dbName = mapping.environment.dbName @@ -62,12 +74,6 @@ class AtlanService(token: String, baseUrl: String) extends Logging { } def generateDescriptionBodyJson(mapping: Config, outputTable: String, qualifiedName: String): String = { - val sql = mapping.mappings.head match { - case sqlmapping: SQLMapping => { - sqlmapping.sqlContents - } - case _ => "" - } s""" |{ | "entities": [ @@ -76,7 +82,7 @@ class AtlanService(token: String, baseUrl: String) extends Logging { | "attributes": { | "name": "$outputTable", | "qualifiedName": "$qualifiedName", - | "description": "$sql" + | "description": "" | } | } | ] diff --git a/src/main/scala/com/metabolic/data/mapper/app/MetabolicApp.scala b/src/main/scala/com/metabolic/data/mapper/app/MetabolicApp.scala index f3f74761..5282f46d 100644 --- a/src/main/scala/com/metabolic/data/mapper/app/MetabolicApp.scala +++ b/src/main/scala/com/metabolic/data/mapper/app/MetabolicApp.scala @@ -118,7 +118,6 @@ class MetabolicApp(sparkBuilder: SparkSession.Builder) extends Logging { val atlan = new AtlanService(token, mapping.environment.atlanBaseUrl.getOrElse("")) atlan.setLineage(mapping) atlan.setMetadata(mapping) - atlan.setDescription(mapping) } case _ => "" } diff --git a/src/test/scala/com/metabolic/data/mapper/services/AtlanServiceTest.scala b/src/test/scala/com/metabolic/data/mapper/services/AtlanServiceTest.scala index 1cd65526..77a9fde9 100644 --- a/src/test/scala/com/metabolic/data/mapper/services/AtlanServiceTest.scala +++ b/src/test/scala/com/metabolic/data/mapper/services/AtlanServiceTest.scala @@ -13,6 +13,9 @@ import org.apache.spark.sql.SaveMode import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + class AtlanServiceTest extends AnyFunSuite with DataFrameSuiteBase with SharedSparkContext @@ -113,7 +116,7 @@ class AtlanServiceTest extends AnyFunSuite assert(response.trim.equalsIgnoreCase("")) } - test("Test description body") { + ignore("Test metadata body") { val testingConfig = Config( "", List(io.FileSource("raw/stripe/fake_employee/version=3/", "employees", IOFormat.PARQUET), io.FileSource("clean/fake_employee_s/version=123/", "employeesss", IOFormat.PARQUET), io.FileSource("raw/hubspot/owners/", "owners", IOFormat.PARQUET), io.FileSource("clean/hubspot_owners/", "clean_owners", IOFormat.PARQUET)), @@ -123,21 +126,16 @@ class AtlanServiceTest extends AnyFunSuite Environment("", EngineMode.Batch, "", false, "test", "", Option(""),Option(""), false, false, Seq("raw", "clean", "gold", "bronze"), Seq("raw_stripe", "raw_hubspot")) ) val calculatedJson = new AtlanService("foo", "foo") - .generateDescriptionBodyJson(testingConfig, "gold_stripe_f_fake_employee_t", "foo/test/gold_stripe_f_fake_employee_t") + .generateMetadaBody(testingConfig) val expectedJson = - """ + s""" |{ - | "entities": [ - | { - | "typeName": "Table", - | "attributes": { - | "name": "gold_stripe_f_fake_employee_t", - | "qualifiedName": "foo/test/gold_stripe_f_fake_employee_t", - | "description": "select * from employees where age < 40" - | } - | } - | ] + | "Data Quality": { + | "last_synced_at" : ${LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))}, + | "engine_type":"batch", + | "sql_mapping":"select * from employees where age < 40" + | } |} |""".stripMargin assert(expectedJson.trim.equalsIgnoreCase(calculatedJson.trim))