multi pods spark

uc-cdis · Jun 3, 2024 · 70bdc73 · 70bdc73
1 parent 77506c8
commit 70bdc73
Show file tree

Hide file tree

Showing 2 changed files with 364 additions and 90 deletions.
diff --git a/kube/services/spark/spark-deploy.yaml b/kube/services/spark/spark-deploy.yaml
@@ -1,10 +1,122 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hadoop-spark-config
+data:
+  NAMENODE_URI: "hdfs://namenode:9000"
+  DATANODE_URI: "hdfs://datanode:50010"
+  RESOURCEMANAGER_URI: "http://resourcemanager:8088"
+  NODEMANAGER_URI: "http://nodemanager:8042"
+  HISTORYSERVER_URI: "http://historyserver:8181"
+  CORE_CONF_fs_defaultFS: "hdfs://namenode:9000"
+  YARN_CONF_yarn_resourcemanager_hostname: "resourcemanager"
+  YARN_CONF_yarn_nodemanager_aux_services: "mapreduce_shuffle"
+  YARN_CONF_yarn_nodemanager_aux_services_mapreduce_shuffle_class: "org.apache.hadoop.mapred.ShuffleHandler"
+  MAPRED_CONF_mapreduce_framework_name: "yarn"
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: namenode-deployment
+spec:
+  serviceName: "namenode"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: namenode
+  template:
+    metadata:
+      labels:
+        app: namenode
+    spec:
+      containers:
+      - name: namenode
+        image: "quay.io/cdis/namenode:3.3.0"
+        ports:
+        - containerPort: 9000
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+        volumeMounts:
+        - name: namenode-pv-storage
+          mountPath: /hadoop/dfs/name
+        command: ["/bin/bash" ]
+        args: 
+          - "-c"
+          - |
+            ssh server sudo /etc/init.d/ssh start
+            update-ca-certificates
+            hdfs namenode -format
+            hdfs dfsadmin -safemode leave
+            hdfs dfs -mkdir /result
+            hdfs dfs -mkdir /jars
+            hdfs dfs -mkdir /archive
+  volumeClaimTemplates:
+  - metadata:
+      name: namenode-pv-storage
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      storageClassName: "standard"
+      resources:
+        requests:
+          storage: 10Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: datanode-deployment
+spec:
+  serviceName: "datanode"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: datanode
+  template:
+    metadata:
+      labels:
+        app: datanode
+    spec:
+      containers:
+      - name: datanode
+        image: "quay.io/cdis/datanode:3.3.0"
+        ports:
+        - containerPort: 50010
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+        volumeMounts:
+        - name: datanode-pv-storage
+          mountPath: /hadoop/dfs/data
+        command: ["/bin/bash" ]
+        args: 
+          - "-c"
+          - |
+            ssh server sudo /etc/init.d/ssh start
+            update-ca-certificates
+            hdfs namenode -format
+            hdfs dfsadmin -safemode leave
+            hdfs dfs -mkdir /result
+            hdfs dfs -mkdir /jars
+            hdfs dfs -mkdir /archive
+  volumeClaimTemplates:
+  - metadata:
+      name: datanode-pv-storage
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      storageClassName: "standard"
+      resources:
+        requests:
+          storage: 100Gi
+
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: spark-deployment
-  annotations:
-    gen3.io/network-ingress: "tube"
+  name: resourcemanager-deployment
 spec:
+  replicas: 1
   selector:
     # Only select pods based on the 'app' label
     matchLabels:
@@ -51,54 +163,177 @@ spec:
                 values:
                 - ONDEMAND
       automountServiceAccountToken: false
-      volumes:
       containers:
-        - name: gen3-spark
-          GEN3_SPARK_IMAGE
-          ports:
-          - containerPort: 22
-          - containerPort: 9000
-          - containerPort: 8030
-          - containerPort: 8031
-          - containerPort: 8032
-          - containerPort: 7077
-          livenessProbe:
-            tcpSocket:
-              port: 9000
-            initialDelaySeconds: 10
-            periodSeconds: 30
-          env:
-          - name: DICTIONARY_URL
-            valueFrom:
-              configMapKeyRef:
-                name: manifest-global
-                key: dictionary_url
-          - name: HADOOP_URL
-            value: hdfs://0.0.0.0:9000
-          - name: HADOOP_HOST
-            value: 0.0.0.0
-          volumeMounts:
-          imagePullPolicy: Always
-          resources:
-            requests:
-              cpu: 3
-              memory: 4Gi
-          command: ["/bin/bash" ]
-          args: 
-            - "-c"
-            - |
-              # get /usr/local/share/ca-certificates/cdis-ca.crt into system bundle
-              ssh server sudo /etc/init.d/ssh start
-              update-ca-certificates
-              python run_config.py
-              hdfs namenode -format
-              hdfs --daemon start namenode
-              hdfs --daemon start datanode
-              yarn --daemon start resourcemanager
-              yarn --daemon start nodemanager
-              hdfs dfsadmin -safemode leave
-              hdfs dfs -mkdir /result
-              hdfs dfs -mkdir /jars
-              hdfs dfs -mkdir /archive
-              /spark/sbin/start-all.sh
-              while true; do sleep 5; done
+      - name: resourcemanager
+        image: "quay.io/cdis/resourcemanager:3.3.0"
+        ports:
+        - containerPort: 8088
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nodemanager-deployment
+spec:
+  replicas: 2  # Adjust based on your needs
+  selector:
+    matchLabels:
+      app: nodemanager
+  template:
+    metadata:
+      labels:
+        app: nodemanager
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 25
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - spark
+              topologyKey: "kubernetes.io/hostname"
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      automountServiceAccountToken: false
+      containers:
+      - name: nodemanager
+        image: "quay.io/cdis/nodemanager:3.3.0"
+        ports:
+        - containerPort: 8042
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-master-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: spark-master
+  template:
+    metadata:
+      labels:
+        app: spark-master
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 25
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - spark
+              topologyKey: "kubernetes.io/hostname"
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      automountServiceAccountToken: false
+      containers:
+      - name: spark-master
+        image: "quay.io/cdis/spark-master:3.3.0-hadoop3.3"
+        ports:
+        - containerPort: 7077
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-worker-deployment
+spec:
+  replicas: 2  # Adjust based on your needs
+  selector:
+    matchLabels:
+      app: spark-worker
+  template:
+    metadata:
+      labels:
+        app: spark-worker
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 25
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - spark
+              topologyKey: "kubernetes.io/hostname"
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      automountServiceAccountToken: false
+      containers:
+      - name: spark-worker
+        image: quay.io/cdis/spark-worker:3.3.0-hadoop3.3
+        ports:
+        - containerPort: 8081
+        envFrom:
+        - configMapRef:
+            name: hadoop-spark-config
+        command: ["/bin/bash" ]
+        args: 
+          - "-c"
+          - |
+            ssh server sudo /etc/init.d/ssh start
+            update-ca-certificates