From 686da0909c907508c3955f8a000f3ea8f18e44a0 Mon Sep 17 00:00:00 2001 From: Robert Hecht Date: Fri, 29 Sep 2023 11:16:51 +0200 Subject: [PATCH] adjust env name for spark presets - the env name "SPARK_CONF_DIR" is collides with a variable used by spark, see: https://github.com/apache/spark/blob/master/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala#L61 --- main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 2c679e5..f100449 100644 --- a/main.go +++ b/main.go @@ -28,12 +28,12 @@ import ( ) type mainCmd struct { - SparkHome string `required:"" default:"/opt/spark" help:"spark home directory" env:"SPARK_HOME"` - SparkConfDir string `required:"" help:"directory with spark configuration presets" env:"SPARK_CONF_DIR"` - Master string `required:"" help:"spark master address" env:"SPARK_MASTER"` - DebugSubmit bool `help:"write spark-submit output to logger" env:"DEBUG_SPARK_SUBMIT"` - DevMode bool `help:"sets the logger output to development config"` - Debug bool `help:"enables debug logs" env:"DEBUG"` + SparkHome string `required:"" default:"/opt/spark" help:"spark home directory" env:"SPARK_HOME"` + SparkPresetDir string `required:"" help:"directory with spark configuration presets" env:"SPARK_PRESET_DIR"` + Master string `required:"" help:"spark master address" env:"SPARK_MASTER"` + DebugSubmit bool `help:"write spark-submit output to logger" env:"DEBUG_SPARK_SUBMIT"` + DevMode bool `help:"sets the logger output to development config"` + Debug bool `help:"enables debug logs" env:"DEBUG"` } var CLI struct { @@ -47,7 +47,7 @@ func main() { func (cmd *mainCmd) Run() { cmd.setupLogger() - s, err := spark.New(cmd.SparkHome, cmd.SparkConfDir, cmd.Master, cmd.DebugSubmit) + s, err := spark.New(cmd.SparkHome, cmd.SparkPresetDir, cmd.Master, cmd.DebugSubmit) if err != nil { zap.L().Fatal("couldn't initialize spark dependency", zap.Error(err)) }