fix tube and spark deployment

uc-cdis · Jun 5, 2024 · 5edc884 · 5edc884
1 parent 3aaf628
commit 5edc884
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 19 deletions.
diff --git a/kube/services/spark/spark-deploy.yaml b/kube/services/spark/spark-deploy.yaml
@@ -98,7 +98,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - resourcemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -152,7 +152,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - nodemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -206,7 +206,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-master
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -261,7 +261,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-worker
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:

diff --git a/kube/services/spark/spark-service.yaml b/kube/services/spark/spark-service.yaml
@@ -4,8 +4,10 @@ metadata:
   name: namenode
 spec:
   ports:
-    - port: 9000
+    - protocol: TCP
+      port: 9000
       targetPort: 9000
+      name: hdfs
   clusterIP: None
   selector:
     app: namenode
@@ -18,8 +20,10 @@ metadata:
   name: datanode
 spec:
   ports:
-    - port: 50010
+    - protocol: TCP
+      port: 50010
       targetPort: 50010
+      name: data
   clusterIP: None
   selector:
     app: datanode
@@ -32,8 +36,10 @@ metadata:
   name: resourcemanager
 spec:
   ports:
-    - port: 8088
+    - protocol: TCP
+      port: 8088
       targetPort: 8088
+      name: resource
   clusterIP: None
   selector:
     app: resourcemanager
@@ -46,8 +52,10 @@ metadata:
   name: nodemanager
 spec:
   ports:
-    - port: 8042
+    - protocol: TCP
+      port: 8042
       targetPort: 8042
+      name: node
   clusterIP: None
   selector:
     app: nodemanager
@@ -57,15 +65,21 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkmaster
+  name: spark-master
 spec:
   ports:
-    - name: p7077
-      port: 7077
-      targetPort: 7077
-    - name: p8080
-      port: 8080
-      targetPort: 8080
+  - name: web-ui
+    protocol: TCP
+    port: 8080
+    targetPort: 8080
+  - name: master
+    protocol: TCP
+    port: 7077
+    targetPort: 7077
+  - name: master-rest
+    protocol: TCP
+    port: 6066
+    targetPort: 6066
   clusterIP: None
   selector:
     app: spark-master
@@ -75,10 +89,12 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkworker
+  name: spark-worker
 spec:
   ports:
-    - port: 8081
+    - name: web-ui
+      protocol: TCP
+      port: 8081
       targetPort: 8081
   clusterIP: None
   selector:

diff --git a/kube/services/tube/tube-deploy.yaml b/kube/services/tube/tube-deploy.yaml
@@ -75,17 +75,19 @@ spec:
                 name: manifest-global
                 key: dictionary_url
           - name: HADOOP_URL
-            value: hdfs://spark-service:9000
+            value: hdfs://namenode:9000
           - name: ES_URL
             value: esproxy-service
           - name: HADOOP_HOST
-            value: spark-service
+            value: namenode
           - name: HADOOP_CLIENT_OPTS
             value: -Xmx1g
           - name: SPARK_EXECUTOR_MEMORY
             value: 4g
           - name: SPARK_DRIVER_MEMORY
             value: 6g
+          - name: SPARK_MASTER
+            value: spark-master
           volumeMounts:
             - name: "creds-volume"
               readOnly: true