Changed owner and files to config generator

metabolicdata · Dec 3, 2024 · ad52750 · ad52750
1 parent c10bd49
commit ad52750
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 61 deletions.
diff --git a/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala b/src/main/scala/com/metabolic/data/core/services/catalogue/AtlanService.scala
@@ -99,57 +99,42 @@ class AtlanService(token: String, baseUrlDataLake: String, baseUrlConfluent: Str
 
   def generateOwnerBody(mapping: Config): String = {
 
-    val config = mapping.defaults.config
-
-    if (config.hasPath("owner")) {
-      // TODO - Check that the owner from conf file matches the owner's name in Atlan
-      val owner = config.hasPath("owner")
-
-      val body = {
-        s"""
-           |{
-           |  "owners": [$owner]
-           |}
-           |""".stripMargin
-      }
-      body
-    } else {
-      null
+    val owner = mapping.owner
+
+    val body = {
+      s"""
+         |{
+         |  "owners": [$owner]
+         |}
+         |""".stripMargin
     }
+    body
   }
 
   def generateResourceBody(mapping: Config): String = {
 
-    val config = mapping.defaults.config
-
-    if (config.hasPath("mapping.file")) {
-      // TODO - Change the hardcoded URL to a dynamic one
-      val urlSQL = config.getConfig("mapping").getString("file").replace("s3://factorial-metabolic/data-lake-confs/production", "https://github.com/factorialco/data-lake/tree/main")
-      val urlConf = urlSQL.replace(".sql", ".conf")
-
-      val body = {
-        s"""
-           |{
-           |  "resources": [
-           |    {
-           |      "name": "SQL File",
-           |      "type": "LINK",
-           |      "url": "$urlSQL"
-           |    },
-           |    {
-           |      "name": "Conf File",
-           |      "type": "LINK",
-           |      "url": "$urlConf"
-           |    }
-           |  ]
-           |}
-           |""".stripMargin
-      }
+    val urlSQL = mapping.sqlUrl
+    val urlConf = mapping.confUrl
 
-      body
-    } else {
-      null
+    val body = {
+      s"""
+         |{
+         |  "resources": [
+         |    {
+         |      "name": "SQL File",
+         |      "type": "LINK",
+         |      "url": "$urlSQL"
+         |    },
+         |    {
+         |      "name": "Conf File",
+         |      "type": "LINK",
+         |      "url": "$urlConf"
+         |    }
+         |  ]
+         |}
+         |""".stripMargin
     }
+    body
   }
 
   def setDescription(mapping: Config): String = {

diff --git a/src/main/scala/com/metabolic/data/mapper/domain/Config.scala b/src/main/scala/com/metabolic/data/mapper/domain/Config.scala
@@ -7,7 +7,7 @@ import com.metabolic.data.mapper.domain.ops.Mapping
 import com.typesafe.config.ConfigFactory
 
 class Config(val name: String, val sources: Seq[Source], val mappings: Seq[Mapping], val sink: Sink,
-             defaults: Defaults, environment: Environment) extends CoreConfig(defaults, environment) {
+             defaults: Defaults, environment: Environment, val owner: Option[String], val sqlUrl: String, val confUrl: String) extends CoreConfig(defaults, environment) {
 
   def getCanonicalName() = {
     //regex to remove all non-alphanumeric characters
@@ -26,14 +26,14 @@ class Config(val name: String, val sources: Seq[Source], val mappings: Seq[Mappi
 object Config {
 
   def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink,
-            defaults: Defaults, platform: Environment): Config = {
-    new Config(name, sources, mappings, sink, defaults, platform)
+            defaults: Defaults, platform: Environment, owner: Option[String], sqlUrl: String, confUrl: String): Config = {
+    new Config(name, sources, mappings, sink, defaults, platform, owner, sqlUrl, confUrl)
   }
 
-  def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink): Config = {
+  def apply(name: String, sources: Seq[Source], mappings: Seq[Mapping], sink: Sink, owner: Option[String], sqlUrl: String, confUrl: String): Config = {
     val defaults: Defaults = Defaults(ConfigFactory.load())
     val environment: Environment = Environment("", EngineMode.Batch, "", false,"test","",
       Regions.fromName("eu-central-1"),Option.empty, Option.empty, Option.empty)
-    new Config(name, sources, mappings, sink, defaults, environment)
+    new Config(name, sources, mappings, sink, defaults, environment, owner, sqlUrl, confUrl)
   }
 }
diff --git a/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala b/src/main/scala/com/metabolic/data/mapper/services/ConfigParserService.scala
@@ -138,7 +138,13 @@ class ConfigParserService(implicit region: Regions) extends Logging {
 
     val sink = SinkConfigParserService().parseSink(config, platform)
 
-    Seq(new Config(name, sources, mappings, sink, defaults, platform))
+    val owner = parseOwner(config)
+
+    val sqlUrl = parseFileUrl(config, "sql")
+
+    val confUrl = parseFileUrl(config, "conf")
+
+    Seq(new Config(name, sources, mappings, sink, defaults, platform, owner, sqlUrl, confUrl))
   }
 
   private def parseName(config: HoconConfig) = {
@@ -230,4 +236,24 @@ class ConfigParserService(implicit region: Regions) extends Logging {
     Seq(mapping)
   }
 
+  private def parseFileUrl(config: HoconConfig, urlType: String): String = {
+    val fileUrl = if (config.hasPath("mappings.file")) {
+      config.getConfig("mappings").getString("file")
+    } else if (config.hasPath("mapping.file")) {
+      config.getConfig("mapping").getString("file")
+    } else {
+      ""
+    }
+
+    // TODO - Remove hardcoded URLs and replace with a configuration
+    fileUrl.replace(".sql", s".$urlType").replace("s3://factorial-metabolic/data-lake-confs/production", "https://github.com/factorialco/data-lake/tree/main")
+  }
+
+  private def parseOwner(config: HoconConfig): Option[String] = {
+    if (config.hasPathOrNull("owner")) {
+      Option.apply(config.getString("owner"))
+    } else {
+      Option.empty
+    }
+  }
 }
diff --git a/src/test/resources/employees.conf b/src/test/resources/employees.conf
@@ -1,5 +1,6 @@
 name: "Employees Test"
 author: "[email protected]"
+owner: "Fernando Azpiazu"
 sources: [
   {
     inputPath = "src/test/tmp/fake_employee"

diff --git a/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala b/src/test/scala/com/metabolic/data/services/AtlanServiceTest.scala
@@ -6,22 +6,13 @@ import org.scalatest.funsuite.AnyFunSuite
 
 class AtlanServiceTest extends AnyFunSuite with RegionedTest  {
 
-  test("testPreloadConfig") {
+  test("atlanCatalogServiceTest") {
 
     val configService = new ConfigReaderService()
 
     // Load config file from path
     val config = configService.getConfig("src/test/resources/employees.conf")
 
-    // When file in config.mapping
-    val fileUrl = if (config.hasPath("mapping.file")) {
-      config.getConfig("mapping").getString("file")
-    } else {
-      ""
-    }
-
-    // assert variable fileUrl is not ""
-    assert(fileUrl != "")
-  }
+    asset(config != null)
 }