fix tube and spark deployment

uc-cdis · Jun 5, 2024 · ee25967 · ee25967
1 parent 3aaf628
commit ee25967
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 8 deletions.
diff --git a/kube/services/spark/spark-deploy.yaml b/kube/services/spark/spark-deploy.yaml
@@ -98,7 +98,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - resourcemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -152,7 +152,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - nodemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -206,7 +206,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-master
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -261,7 +261,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-worker
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:

diff --git a/kube/services/spark/spark-service.yaml b/kube/services/spark/spark-service.yaml
@@ -57,7 +57,7 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkmaster
+  name: spark-master
 spec:
   ports:
     - name: p7077
@@ -75,7 +75,7 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkworker
+  name: spark-worker
 spec:
   ports:
     - port: 8081

diff --git a/kube/services/tube/tube-deploy.yaml b/kube/services/tube/tube-deploy.yaml
@@ -75,17 +75,19 @@ spec:
                 name: manifest-global
                 key: dictionary_url
           - name: HADOOP_URL
-            value: hdfs://spark-service:9000
+            value: hdfs://namenode:9000
           - name: ES_URL
             value: esproxy-service
           - name: HADOOP_HOST
-            value: spark-service
+            value: namenode
           - name: HADOOP_CLIENT_OPTS
             value: -Xmx1g
           - name: SPARK_EXECUTOR_MEMORY
             value: 4g
           - name: SPARK_DRIVER_MEMORY
             value: 6g
+          - name: SPARK_MASTER
+            value: spark-master
           volumeMounts:
             - name: "creds-volume"
               readOnly: true