From 4609019c4ca3d3bcf15229f563934e07d479f2c4 Mon Sep 17 00:00:00 2001
From: Antonio Tapiador del Dujo <antonio.tapiador@factorial.co>
Date: Thu, 27 Jul 2023 10:54:22 +0200
Subject: [PATCH] Resolve preloadings in mappings

Currently, it is not possible to set a parameter to resolve to a mapping:

```
dp.from = df.now
```

It does not get translated to a real time. It remains as "df.now"

This translation is interesting for data recovery. We can parameterize
prunning of bronze tables with default arguments, like

```
dp.from = df.start_of_today
dp.to = df.now
```

When we want to recover from a loss in some window of time,
we can manually run the glue job manually in AWS glue,
passing the boundaries of the window:

```
dp.from = "2023-07-27T09:34:00"
dp.to = "2023-07-28T10:04:00"
```
---
 .../services/util/ConfigReaderService.scala   | 11 +++++--
 src/test/resources/run_map.conf               |  1 +
 src/test/resources/run_variable.conf          |  2 +-
 .../services/ConfigReaderServiceTest.scala    | 29 +++++++++++++++++--
 4 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala b/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala
index 97aba8fa..8e762104 100644
--- a/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala
+++ b/src/main/scala/com/metabolic/data/core/services/util/ConfigReaderService.scala
@@ -72,14 +72,21 @@ class ConfigReaderService(implicit val region: Regions) {
 
   private def prepareConfig(params: Map[String, String]) = {
 
+    val preloadedConfig = preloadConfig()
+
     params
       .filter { kv =>
         kv._1.startsWith("dp.")
       }.map { kv =>
-        System.setProperty(kv._1, kv._2)
+        val value = if (preloadedConfig.contains(kv._2)) {
+          preloadedConfig(kv._2)
+        } else {
+          kv._2
+        }
+        System.setProperty(kv._1, value)
       }
 
-    preloadConfig()
+    preloadedConfig
       .map { kv =>
         System.setProperty(kv._1, kv._2)
       }
diff --git a/src/test/resources/run_map.conf b/src/test/resources/run_map.conf
index 11884fd6..7c2bf26b 100644
--- a/src/test/resources/run_map.conf
+++ b/src/test/resources/run_map.conf
@@ -1,3 +1,4 @@
 path = ${?raw_bucket}"/tmp"
 pathy = ${dp.raw_bucket}/tmp
+time = ${dp.time}
 
diff --git a/src/test/resources/run_variable.conf b/src/test/resources/run_variable.conf
index e406a820..ace6973c 100644
--- a/src/test/resources/run_variable.conf
+++ b/src/test/resources/run_variable.conf
@@ -1,2 +1,2 @@
 path = ${?raw_bucket}"/tmp"
-
+time = ${df.now}
diff --git a/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala b/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala
index c54e5675..3f7f4e61 100644
--- a/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala
+++ b/src/test/scala/com/metabolic/data/mapper/services/ConfigReaderServiceTest.scala
@@ -8,6 +8,8 @@ import com.metabolic.data.mapper.domain.ops.mapping.TupletIntervalMapping
 import com.metabolic.data.mapper.domain.ops.source.{DedupeSourceOp, FilterSourceOp, SQLOrder, SelectExpressionSourceOp}
 import org.scalatest.funsuite.AnyFunSuite
 
+import scala.util.matching.Regex
+
 class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   val personHOCON =
@@ -145,11 +147,20 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   }
 
+  test("ConfigReaderService resolves a preload ") {
+
+    val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_variable.conf")
+    val datePattern: String = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"""
+
+    assert(overrideConfig.getString("time").matches(datePattern))
+  }
+
   test("ConfigReaderService resolves a variable in map ") {
 
     val args: Map[String, String] = Map(
       "dp.region" -> "eu-west-4",
-      "dp.raw_bucket" -> "s3://factorial-dl-rawy"
+      "dp.raw_bucket" -> "s3://factorial-dl-rawy",
+      "dp.time" -> "2021-01-01T00:00:00"
     )
 
     val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_map.conf", args)
@@ -158,6 +169,20 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
 
   }
 
+  test("ConfigReaderService resolves a preload in a map") {
+
+    val args: Map[String, String] = Map(
+      "dp.region" -> "eu-west-4",
+      "dp.raw_bucket" -> "s3://factorial-dl-rawy",
+      "dp.time" -> "df.now"
+    )
+
+    val overrideConfig = new ConfigReaderService().getConfig("src/test/resources/run_map.conf", args)
+    val datePattern: String = """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"""
+
+    assert(overrideConfig.getString("time").matches(datePattern))
+  }
+
   test("ConfigReaderService resolves mappings ") {
 
     val config = new ConfigReaderService()
@@ -254,4 +279,4 @@ class ConfigReaderServiceTest extends AnyFunSuite with RegionedTest  {
   }
 
 
-}
\ No newline at end of file
+}