Support custom driver log urls

G-Research · Mar 22, 2024 · e28de27 · e28de27
1 parent 648f9b5
commit e28de27
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/...main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/...main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
@@ -102,6 +102,12 @@ private[spark] class KubernetesClusterSchedulerBackend(
     conf.getOption("spark.app.id").getOrElse(appId)
   }
 
+  override def getDriverAttributes: Option[Map[String, String]] = Some(Map(
+    "LOG_FILES" -> "log",
+    "APP_ID" -> applicationId,
+    "KUBERNETES_NAMESPACE" -> namespace
+  ))
+
   override def start(): Unit = {
     super.start()
     // Must be called before setting the executors