Added DateTime configs for run_enceladus.cmd

AbsaOSS · Mar 22, 2023 · 9098681 · 9098681
1 parent e139800
commit 9098681
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 0 deletions.
diff --git a/scripts/bash/_print_help.sh b/scripts/bash/_print_help.sh
@@ -39,6 +39,8 @@ echo "  --executor-memory MEM                    Memory per executor (e.g. 1000M
 echo "  --dra-num-executors NUM                  Same as '--num-executors' but used when DRA is enabled. Use with care! DRA won't scale below this NUM."
 echo "  --dra-executor-cores NUM                 Same as '--executor-memory' but used when DRA is enabled."
 echo "  --dra-executor-memory MEM                Same as '--executor-cores' but used when DRA is enabled."
+echo "  --parquet-datetime-read-mode"
+echo "  --parquet-datetime-write-mode"
 echo "  --master MASTER_URL                      spark://host:port, mesos://host:port, yarn, k8s://https://host:port, or local"
 echo "  --deploy-mode DEPLOY_MODE                Whether to launch the driver program locally (\"client\") or on one of the worker machines inside the cluster (\"cluster\")."
 echo "  --driver-cores NUM                       Number of cores used by the driver, only in cluster mode."

diff --git a/scripts/cmd/_run_enceladus.cmd b/scripts/cmd/_run_enceladus.cmd
@@ -29,6 +29,8 @@ SET EXECUTOR_CORES=%DEFAULT_EXECUTOR_CORES%
 SET EXECUTOR_MEMORY=%EFAULT_EXECUTOR_MEMORY%
 SET DRA_EXECUTOR_CORES=%DEFAULT_DRA_EXECUTOR_CORES%
 SET DRA_EXECUTOR_MEMORY=%DEFAULT_DRA_EXECUTOR_MEMORY%
+SET PARQUET_DATETIME_READ_MODE=%DEFAULT_PARQUET_DATETIME_READ_MODE%
+SET PARQUET_DATETIME_WRITE_MODE=%DEFAULT_PARQUET_DATETIME_WRITE_MODE%
 SET NUM_EXECUTORS=%DEFAULT_NUM_EXECUTORS%
 SET DRA_NUM_EXECUTORS=
 SET FILES=%ENCELADUS_FILES%
@@ -131,6 +133,18 @@ IF "%1"=="--dra-executor-memory" (
     SHIFT
     GOTO CmdParse
 )
+IF "%1"=="--parquet-datetime-read-mode" (
+    SET PARQUET_DATETIME_READ_MODE=%2
+    SHIFT
+    SHIFT
+    GOTO CmdParse
+)
+IF "%1"=="--parquet-datetime-write-mode" (
+    SET PARQUET_DATETIME_WRITE_MODE=%2
+    SHIFT
+    SHIFT
+    GOTO CmdParse
+)
 IF "%1"=="--master" (
     SET MASTER=%2
     SHIFT
@@ -561,6 +575,11 @@ IF %DRA_ENABLED%==true (
     IF DEFINED EXECUTOR_CORES SET CMD_LINE=%CMD_LINE% --executor-cores %EXECUTOR_CORES%
 )
 
+SET SPARK_CONF=%SPARK_CONF% --conf spark.sql.parquet.datetimeRebaseModeInRead=%PARQUET_DATETIME_READ_MODE%
+SET SPARK_CONF=%SPARK_CONF% --conf spark.sql.parquet.datetimeRebaseModeInWrite=%PARQUET_DATETIME_WRITE_MODE%
+SET SPARK_CONF=%SPARK_CONF% --conf spark.sql.parquet.int96RebaseModeInRead=%PARQUET_DATETIME_READ_MODE%
+SET SPARK_CONF=%SPARK_CONF% --conf spark.sql.parquet.int96RebaseModeInWrite=%PARQUET_DATETIME_WRITE_MODE%
+
 SET JVM_CONF=spark.driver.extraJavaOptions=-Dstandardized.hdfs.path=%STD_HDFS_PATH% -Dspline.mongodb.url=%SPLINE_MONGODB_URL% -Dspline.mongodb.name=%SPLINE_MONGODB_NAME% -Dhdp.version=%HDP_VERSION% %MT_PATTERN% %MIN_BLOCK_SIZE% %MAX_BLOCK_SIZE%
 
 SET CMD_LINE=%SPARK_SUBMIT%