From 5edc8847154c5a1114519372764e353eb056456c Mon Sep 17 00:00:00 2001
From: Thanh Nguyen <thanhnd.fit@gmail.com>
Date: Wed, 5 Jun 2024 02:07:17 -0500
Subject: [PATCH] fix tube and spark deployment

---
 kube/services/spark/spark-deploy.yaml  |  8 ++---
 kube/services/spark/spark-service.yaml | 42 ++++++++++++++++++--------
 kube/services/tube/tube-deploy.yaml    |  6 ++--
 3 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/kube/services/spark/spark-deploy.yaml b/kube/services/spark/spark-deploy.yaml
index 7f411db50b..0ae17849ff 100644
--- a/kube/services/spark/spark-deploy.yaml
+++ b/kube/services/spark/spark-deploy.yaml
@@ -98,7 +98,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - resourcemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -152,7 +152,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - nodemanager
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -206,7 +206,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-master
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -261,7 +261,7 @@ spec:
                 - key: app
                   operator: In
                   values:
-                  - spark
+                  - spark-worker
               topologyKey: "kubernetes.io/hostname"
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
diff --git a/kube/services/spark/spark-service.yaml b/kube/services/spark/spark-service.yaml
index e28ea9c417..e666410cb0 100644
--- a/kube/services/spark/spark-service.yaml
+++ b/kube/services/spark/spark-service.yaml
@@ -4,8 +4,10 @@ metadata:
   name: namenode
 spec:
   ports:
-    - port: 9000
+    - protocol: TCP
+      port: 9000
       targetPort: 9000
+      name: hdfs
   clusterIP: None
   selector:
     app: namenode
@@ -18,8 +20,10 @@ metadata:
   name: datanode
 spec:
   ports:
-    - port: 50010
+    - protocol: TCP
+      port: 50010
       targetPort: 50010
+      name: data
   clusterIP: None
   selector:
     app: datanode
@@ -32,8 +36,10 @@ metadata:
   name: resourcemanager
 spec:
   ports:
-    - port: 8088
+    - protocol: TCP
+      port: 8088
       targetPort: 8088
+      name: resource
   clusterIP: None
   selector:
     app: resourcemanager
@@ -46,8 +52,10 @@ metadata:
   name: nodemanager
 spec:
   ports:
-    - port: 8042
+    - protocol: TCP
+      port: 8042
       targetPort: 8042
+      name: node
   clusterIP: None
   selector:
     app: nodemanager
@@ -57,15 +65,21 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkmaster
+  name: spark-master
 spec:
   ports:
-    - name: p7077
-      port: 7077
-      targetPort: 7077
-    - name: p8080
-      port: 8080
-      targetPort: 8080
+  - name: web-ui
+    protocol: TCP
+    port: 8080
+    targetPort: 8080
+  - name: master
+    protocol: TCP
+    port: 7077
+    targetPort: 7077
+  - name: master-rest
+    protocol: TCP
+    port: 6066
+    targetPort: 6066
   clusterIP: None
   selector:
     app: spark-master
@@ -75,10 +89,12 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: sparkworker
+  name: spark-worker
 spec:
   ports:
-    - port: 8081
+    - name: web-ui
+      protocol: TCP
+      port: 8081
       targetPort: 8081
   clusterIP: None
   selector:
diff --git a/kube/services/tube/tube-deploy.yaml b/kube/services/tube/tube-deploy.yaml
index 6c6feae0f3..2fc7690600 100644
--- a/kube/services/tube/tube-deploy.yaml
+++ b/kube/services/tube/tube-deploy.yaml
@@ -75,17 +75,19 @@ spec:
                 name: manifest-global
                 key: dictionary_url
           - name: HADOOP_URL
-            value: hdfs://spark-service:9000
+            value: hdfs://namenode:9000
           - name: ES_URL
             value: esproxy-service
           - name: HADOOP_HOST
-            value: spark-service
+            value: namenode
           - name: HADOOP_CLIENT_OPTS
             value: -Xmx1g
           - name: SPARK_EXECUTOR_MEMORY
             value: 4g
           - name: SPARK_DRIVER_MEMORY
             value: 6g
+          - name: SPARK_MASTER
+            value: spark-master
           volumeMounts:
             - name: "creds-volume"
               readOnly: true