KEP-2170: Deploy JobSet in kubeflow-system namespace (#2388)

* KEP-2170: Deploy JobSet in kubeflow-system namespace Signed-off-by: Andrey Velichkevich <[email protected]> * Remove namespace from base Signed-off-by: Andrey Velichkevich <[email protected]> * Remove label from namespace Signed-off-by: Andrey Velichkevich <[email protected]> * Create third-party dir for JobSet Signed-off-by: Andrey Velichkevich <[email protected]> * Bump JobSet to v0.7.3 Signed-off-by: Andrey Velichkevich <[email protected]> * Drop namespace from JobSet config Signed-off-by: Andrey Velichkevich <[email protected]> --------- Signed-off-by: Andrey Velichkevich <[email protected]>
kubeflow · Jan 27, 2025 · ee1b691 · ee1b691
1 parent 6eeb1fb
commit ee1b691
Show file tree

Hide file tree

Showing 13 changed files with 55 additions and 30 deletions.
diff --git a/manifests/v2/base/manager/kustomization.yaml b/manifests/v2/base/manager/kustomization.yaml
@@ -1,4 +1,2 @@
 resources:
   - manager.yaml
-# TODO (andreyvelich): Move it to overlays once we copy the JobSet manifests.
-namespace: kubeflow-system
diff --git a/manifests/v2/base/rbac/kustomization.yaml b/manifests/v2/base/rbac/kustomization.yaml
@@ -2,5 +2,3 @@ resources:
   - role.yaml
   - role_binding.yaml
   - service_account.yaml
-# TODO (andreyvelich): Move it to overlays once we copy the JobSet manifests.
-namespace: kubeflow-system
diff --git a/.../runtimes/pre-training/kustomization.yaml → ...e/runtimes/pretraining/kustomization.yaml b/.../runtimes/pre-training/kustomization.yaml → ...e/runtimes/pretraining/kustomization.yaml
@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - torch-distributed.yaml
+  - torch_distributed.yaml
diff --git a/...times/pre-training/torch-distributed.yaml → ...ntimes/pretraining/torch_distributed.yaml b/...times/pre-training/torch-distributed.yaml → ...ntimes/pretraining/torch_distributed.yaml
diff --git a/manifests/v2/base/webhook/kustomization.yaml b/manifests/v2/base/webhook/kustomization.yaml
@@ -10,5 +10,3 @@ patches:
       kind: ValidatingWebhookConfiguration
 configurations:
   - kustomizeconfig.yaml
-# TODO (andreyvelich): Move it to overlays once we copy the JobSet manifests.
-namespace: kubeflow-system
diff --git a/.../overlays/only-manager/kustomization.yaml → ...ts/v2/overlays/manager/kustomization.yaml b/.../overlays/only-manager/kustomization.yaml → ...ts/v2/overlays/manager/kustomization.yaml
@@ -1,16 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+
+# Namespace where all resources are deployed.
+namespace: kubeflow-system
+
 resources:
   - namespace.yaml
   - ../../base/crds
   - ../../base/manager
   - ../../base/rbac
   - ../../base/webhook
-  # TODO (andreyvelich): JobSet should support kubeflow-system namespace.
-  - https://github.com/kubernetes-sigs/jobset/releases/download/v0.6.0/manifests.yaml
+  - ../../third-party/jobset # Comment this line if JobSet is installed on the Kubernetes cluster.
+
+# Update the Kubeflow Training manager image tag.
 images:
   - name: kubeflow/training-operator-v2
     newTag: latest
+
+# Secret for the Kubeflow Training webhook.
 secretGenerator:
   - name: training-operator-v2-webhook-cert
     namespace: kubeflow-system

diff --git a/...s/v2/overlays/only-manager/namespace.yaml → manifests/v2/overlays/manager/namespace.yaml b/...s/v2/overlays/only-manager/namespace.yaml → manifests/v2/overlays/manager/namespace.yaml
diff --git a/...overlays/only-runtimes/kustomization.yaml → ...s/v2/overlays/runtimes/kustomization.yaml b/...overlays/only-runtimes/kustomization.yaml → ...s/v2/overlays/runtimes/kustomization.yaml
@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../base/runtimes/pre-training
+  - ../../base/runtimes/pretraining
diff --git a/manifests/v2/overlays/standalone/kustomization.yaml b/manifests/v2/overlays/standalone/kustomization.yaml
diff --git a/manifests/v2/third-party/jobset/jobset_manager_config.yaml b/manifests/v2/third-party/jobset/jobset_manager_config.yaml
@@ -0,0 +1,2 @@
+apiVersion: config.jobset.x-k8s.io/v1alpha1
+kind: Configuration
diff --git a/manifests/v2/third-party/jobset/kustomization.yaml b/manifests/v2/third-party/jobset/kustomization.yaml
@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://github.com/kubernetes-sigs/jobset/releases/download/v0.7.3/manifests.yaml
+
+# Config for the JobSet manager.
+configMapGenerator:
+  - name: jobset-manager-config
+    files:
+      - jobset_manager_config.yaml
+    options:
+      disableNameSuffixHash: true
+
+# Add required patches.
+patchesStrategicMerge:
+  - patches/jobset_remove_namespace.yaml # Remove namespace from the JobSet release manifests.
+  - patches/jobset_config_patch.yaml # Add custom manager config to the JobSet.
diff --git a/manifests/v2/third-party/jobset/patches/jobset_config_patch.yaml b/manifests/v2/third-party/jobset/patches/jobset_config_patch.yaml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jobset-controller-manager
+  namespace: jobset-system
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          args:
+            - "--config=/jobset_manager_config.yaml"
+          volumeMounts:
+            - name: jobset-manager-config
+              mountPath: /jobset_manager_config.yaml
+              subPath: jobset_manager_config.yaml
+              readOnly: true
+      volumes:
+        - name: jobset-manager-config
+          configMap:
+            name: jobset-manager-config
diff --git a/...sts/v2/overlays/standalone/namespace.yaml → ...bset/patches/jobset_remove_namespace.yaml b/...sts/v2/overlays/standalone/namespace.yaml → ...bset/patches/jobset_remove_namespace.yaml
@@ -1,4 +1,6 @@
+---
+$patch: delete
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: kubeflow-system
+  name: jobset-system
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		apiVersion: config.jobset.x-k8s.io/v1alpha1
		kind: Configuration