From ad52750864a6cc418689427047077fc43d356cc6 Mon Sep 17 00:00:00 2001 From: fazpiazu Date: Tue, 3 Dec 2024 10:42:23 +0100 Subject: [PATCH] Changed owner and files to config generator --- .../services/catalogue/AtlanService.scala | 73 ++++++++----------- .../metabolic/data/mapper/domain/Config.scala | 10 +-- .../mapper/services/ConfigParserService.scala | 28 ++++++- src/test/resources/employees.conf | 1 + .../data/services/AtlanServiceTest.scala | 13 +--- 5 files changed, 64 insertions(+), 61 deletions(-) diff --git a/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala b/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala index df34b2ae..3e2dca1e 100644 --- a/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala +++ b/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala @@ -99,57 +99,42 @@ class AtlanService(token: String, baseUrlDataLake: String, baseUrlConfluent: Str def generateOwnerBody(mapping: Config): String = { - val config = mapping.defaults.config - - if (config.hasPath("owner")) { - // TODO - Check that the owner from conf file matches the owner's name in Atlan - val owner = config.hasPath("owner") - - val body = { - s""" - |{ - | "owners": [$owner] - |} - |""".stripMargin - } - body - } else { - null + val owner = mapping.owner + + val body = { + s""" + |{ + | "owners": [$owner] + |} + |""".stripMargin } + body } def generateResourceBody(mapping: Config): String = { - val config = mapping.defaults.config - - if (config.hasPath("mapping.file")) { - // TODO - Change the hardcoded URL to a dynamic one - val urlSQL = config.getConfig("mapping").getString("file").replace("s3://factorial-metabolic/data-lake-confs/production", "https://github.com/factorialco/data-lake/tree/main") - val urlConf = urlSQL.replace(".sql", ".conf") - - val body = { - s""" - |{ - | "resources": [ - | { - | "name": "SQL File", - | "type": "LINK", - | "url": "$urlSQL" - | }, - | { - | "name": "Conf File", - | "type": "LINK", - | "url": "$urlConf" - | } - | ] - |} - |""".stripMargin - } + val urlSQL = mapping.sqlUrl + val urlConf = mapping.confUrl - body - } else { - null + val body = { + s""" + |{ + | "resources": [ + | { + | "name": "SQL File", + | "type": "LINK", + | "url": "$urlSQL" + | }, + | { + | "name": "Conf File", + | "type": "LINK", + | "url": "$urlConf" + | } + | ] + |} + |""".stripMargin } + body } def setDescription(mapping: Config): String = { diff --git a/src/main/scala/com/metabolic/data/mapper/domain/Config.scala b/src/main/scala/com/metabolic/data/mapper/domain/Config.scala index bcaa47fc..75d1632c 100644 --- a/src/main/scala/com/metabolic/data/mapper/domain/Config.scala +++ b/src/main/scala/com/metabolic/data/mapper/domain/Config.scala @@ -7,7 +7,7 @@ import com.metabolic.data.mapper.domain.ops.Mapping import com.typesafe.config.ConfigFactory class Config(val name: String, val sources: Seq[Source], val mappings: Seq[Mapping], val sink: Sink, - defaults: Defaults, environment: Environment) extends CoreConfig(defaults, environment) { + defaults: Defaults, environment: Environment, val owner: Option[String], val sqlUrl: String, val confUrl: String) extends CoreConfig(defaults, environment) { def getCanonicalName() = { //regex to remove all non-alphanumeric characters @@ -26,14 +26,14 @@ class Config(val name: String, val sources: Seq[Source], val mappings: Seq[Mappi object Config { def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink, - defaults: Defaults, platform: Environment): Config = { - new Config(name, sources, mappings, sink, defaults, platform) + defaults: Defaults, platform: Environment, owner: Option[String], sqlUrl: String, confUrl: String): Config = { + new Config(name, sources, mappings, sink, defaults, platform, owner, sqlUrl, confUrl) } - def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink): Config = { + def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink, owner: Option[String], sqlUrl: String, confUrl: String): Config = { val defaults: Defaults = Defaults(ConfigFactory.load()) val environment: Environment = Environment("", EngineMode.Batch, "", false,"test","", Regions.fromName("eu-central-1"),Option.empty, Option.empty, Option.empty) - new Config(name, sources, mappings, sink, defaults, environment) + new Config(name, sources, mappings, sink, defaults, environment, owner, sqlUrl, confUrl) } } diff --git a/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala b/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala index 3224681b..f77ac1c3 100644 --- a/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala +++ b/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala @@ -138,7 +138,13 @@ class ConfigParserService(implicit region: Regions) extends Logging { val sink = SinkConfigParserService().parseSink(config, platform) - Seq(new Config(name, sources, mappings, sink, defaults, platform)) + val owner = parseOwner(config) + + val sqlUrl = parseFileUrl(config, "sql") + + val confUrl = parseFileUrl(config, "conf") + + Seq(new Config(name, sources, mappings, sink, defaults, platform, owner, sqlUrl, confUrl)) } private def parseName(config: HoconConfig) = { @@ -230,4 +236,24 @@ class ConfigParserService(implicit region: Regions) extends Logging { Seq(mapping) } + private def parseFileUrl(config: HoconConfig, urlType: String): String = { + val fileUrl = if (config.hasPath("mappings.file")) { + config.getConfig("mappings").getString("file") + } else if (config.hasPath("mapping.file")) { + config.getConfig("mapping").getString("file") + } else { + "" + } + + // TODO - Remove hardcoded URLs and replace with a configuration + fileUrl.replace(".sql", s".$urlType").replace("s3://factorial-metabolic/data-lake-confs/production", "https://github.com/factorialco/data-lake/tree/main") + } + + private def parseOwner(config: HoconConfig): Option[String] = { + if (config.hasPathOrNull("owner")) { + Option.apply(config.getString("owner")) + } else { + Option.empty + } + } } diff --git a/src/test/resources/employees.conf b/src/test/resources/employees.conf index e30b1eb6..739ce2b6 100644 --- a/src/test/resources/employees.conf +++ b/src/test/resources/employees.conf @@ -1,5 +1,6 @@ name: "Employees Test" author: "marc@factorial.co" +owner: "Fernando Azpiazu" sources: [ { inputPath = "src/test/tmp/fake_employee" diff --git a/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala b/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala index 56dff9d3..a6ae5a66 100644 --- a/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala +++ b/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala @@ -6,22 +6,13 @@ import org.scalatest.funsuite.AnyFunSuite class AtlanServiceTest extends AnyFunSuite with RegionedTest { - test("testPreloadConfig") { + test("atlanCatalogServiceTest") { val configService = new ConfigReaderService() // Load config file from path val config = configService.getConfig("src/test/resources/employees.conf") - // When file in config.mapping - val fileUrl = if (config.hasPath("mapping.file")) { - config.getConfig("mapping").getString("file") - } else { - "" - } - - // assert variable fileUrl is not "" - assert(fileUrl != "") - } + asset(config != null) }