Merge pull request #10 from atd/feature/resolve-preloadings-on-mappings

Resolve preloadings in mappings
metabolicdata · Jul 31, 2023 · aeaad64 · aeaad64
2 parents fd4e941 + 4609019
commit aeaad64
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 5 deletions.
diff --git a/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala b/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala
@@ -84,14 +84,21 @@ class ConfigReaderService(implicit val region: Regions) {
 
   private def prepareConfig(params: Map[String, String]) = {
 
+    val preloadedConfig = preloadConfig()
+
     params
       .filter { kv =>
         kv._1.startsWith("dp.")
       }.map { kv =>
-        System.setProperty(kv._1, kv._2)
+        val value = if (preloadedConfig.contains(kv._2)) {
+          preloadedConfig(kv._2)
+        } else {
+          kv._2
+        }
+        System.setProperty(kv._1, value)
       }
 
-    preloadConfig()
+    preloadedConfig
       .map { kv =>
         System.setProperty(kv._1, kv._2)
       }

diff --git a/src/test/resources/run_map.conf b/src/test/resources/run_map.conf
@@ -1,3 +1,4 @@
 path = ${?raw_bucket}"/tmp"
 pathy = ${dp.raw_bucket}/tmp
+time = ${dp.time}
 
diff --git a/src/test/resources/run_variable.conf b/src/test/resources/run_variable.conf
@@ -1,2 +1,2 @@
 path = ${?raw_bucket}"/tmp"
-
+time = ${df.now}
diff --git a/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala b/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala
@@ -8,6 +8,8 @@ import com.metabolic.data.mapper.domain.ops.mapping.TupletIntervalMapping
 import com.metabolic.data.mapper.domain.ops.source.{DedupeSourceOp, FilterSourceOp, SQLOrder, SelectExpressionSourceOp}
 import org.scalatest.funsuite.AnyFunSuite
 
+import scala.util.matching.Regex
+
 class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   val personHOCON =
@@ -145,11 +147,20 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   }
 
+  test("ConfigReaderService resolves a preload ") {
+
+    val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_variable.conf")
+    val datePattern: String = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"""
+
+    assert(overrideConfig.getString("time").matches(datePattern))
+  }
+
   test("ConfigReaderService resolves a variable in map ") {
 
     val args: Map[String, String] = Map(
       "dp.region" -> "eu-west-4",
-      "dp.raw_bucket" -> "s3://factorial-dl-rawy"
+      "dp.raw_bucket" -> "s3://factorial-dl-rawy",
+      "dp.time" -> "2021-01-01T00:00:00"
     )
 
     val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_map.conf", args)
@@ -158,6 +169,20 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   }
 
+  test("ConfigReaderService resolves a preload in a map") {
+
+    val args: Map[String, String] = Map(
+      "dp.region" -> "eu-west-4",
+      "dp.raw_bucket" -> "s3://factorial-dl-rawy",
+      "dp.time" -> "df.now"
+    )
+
+    val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_map.conf", args)
+    val datePattern: String = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"""
+
+    assert(overrideConfig.getString("time").matches(datePattern))
+  }
+
   test("ConfigReaderService resolves mappings ") {
 
     val config = new ConfigReaderService()
@@ -254,4 +279,4 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
   }
 
 
-}
+}