Added topology and node selector

eigr · Sep 28, 2024 · 0b8d854 · 0b8d854
1 parent 4724e0d
commit 0b8d854
Show file tree

Hide file tree

Showing 5 changed files with 269 additions and 17 deletions.
diff --git a/examples/topology/host.yaml b/examples/topology/host.yaml
@@ -8,24 +8,50 @@ metadata:
     # Mandatory. Name of the ActorSystem declared in ActorSystem CRD
     spawn-eigr.io/actor-system: spawn-system
 spec:
+  topology:
+    # affinity:
+    #   podAffinity:
+    #     preferredDuringSchedulingIgnoredDuringExecution:
+    #       - weight: 50
+    #         podAffinityTerm:
+    #           labelSelector:
+    #             matchExpressions:
+    #               - key: actor-system
+    #                 operator: In
+    #                 values:
+    #                   - system
+    #           topologyKey: kubernetes.io/hostname
+
+    #   podAntiAffinity:
+    #     preferredDuringSchedulingIgnoredDuringExecution:
+    #       - weight: 100
+    #         podAffinityTerm:
+    #           labelSelector:
+    #             matchExpressions:
+    #               - key: app
+    #                 operator: In
+    #                 values:
+    #                   - app_name
+    #           topologyKey: kubernetes.io/hostname
+    nodeSelector:
+      gpu: "false"
+    tolerations:
+      - key: "cpu-machines"
+        operator: "Exists"
+        effect: "NoExecute"
   host:
     image: eigr/spawn-springboot-examples:0.5.5 # Mandatory
-    topology:
-      nodeSelector:
-        gpu: "false"
-      tolerations:
-        - key: "cpu-machines"
-          operator: "Exists"
-          effect: "NoExecute"
-      # this configure podTemplate for Task Actors
-      taskActors:
-        - parentName: Jose
+    # this configure podTemplate for Task Actors
+    taskActors:
+      - parentName: Jose
+        topology:
           nodeSelector:
             gpu: "true"
           tolerations:
             - key: "gpu-machines"
               operator: "Exists"
               effect: "NoExecute"
-        - parentName: Franchesco
+      - parentName: Franchesco
+        topology:
           nodeSelector:
             beam: "true"
diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex
@@ -70,6 +70,9 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
          } = _resource
        ) do
     host_params = Map.get(params, "host")
+    task_actors_config = Map.get(host_params, "taskActors", %{})
+    topology = Map.get(params, "topology", %{})
+
     replicas = max(1, Map.get(params, "replicas", @default_actor_host_function_replicas))
     embedded = Map.get(host_params, "embedded", false)
 
@@ -109,8 +112,16 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
           },
           "spec" =>
             %{
-              "affinity" => Map.get(host_params, "affinity", build_affinity(system, name)),
-              "containers" => get_containers(embedded, system, name, host_params, annotations),
+              "affinity" => Map.get(topology, "affinity", build_affinity(system, name)),
+              "containers" =>
+                get_containers(
+                  embedded,
+                  system,
+                  name,
+                  host_params,
+                  annotations,
+                  task_actors_config
+                ),
               "initContainers" => [
                 %{
                   "name" => "init-certificates",
@@ -131,6 +142,8 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
               ],
               "serviceAccountName" => "#{system}-sa"
             }
+            |> maybe_put_node_selector(topology)
+            |> maybe_put_node_tolerations(topology)
             |> maybe_put_volumes(params)
             |> maybe_set_termination_period(params)
         }
@@ -185,10 +198,28 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
     }
   end
 
-  defp get_containers(true, system, name, host_params, annotations) do
+  defp build_task_env(task_actors_config) do
+    value =
+      task_actors_config
+      |> Jason.encode!()
+      |> Base.encode32()
+
+    [
+      %{"name" => "SPAWN_PROXY_TASK_CONFIG", "value" => value}
+    ]
+  end
+
+  defp get_containers(true, system, name, host_params, annotations, task_actors_config) do
     actor_host_function_image = Map.get(host_params, "image")
 
-    actor_host_function_envs = Map.get(host_params, "env", []) ++ @default_actor_host_function_env
+    actor_host_function_envs =
+      if length(Map.values(task_actors_config)) == 0 do
+        Map.get(host_params, "env", []) ++ @default_actor_host_function_env
+      else
+        Map.get(host_params, "env", []) ++
+          @default_actor_host_function_env ++
+          build_task_env(task_actors_config)
+      end
 
     proxy_http_port = String.to_integer(annotations.proxy_http_port)
 
@@ -230,7 +261,7 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
     ]
   end
 
-  defp get_containers(false, system, name, host_params, annotations) do
+  defp get_containers(false, system, name, host_params, annotations, task_actors_config) do
     actor_host_function_image = Map.get(host_params, "image")
 
     actor_host_function_envs =
@@ -247,12 +278,19 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
       %{"containerPort" => proxy_http_port, "name" => "proxy-http"}
     ]
 
+    envs =
+      if length(Map.values(task_actors_config)) == 0 do
+        @default_actor_host_function_env
+      else
+        @default_actor_host_function_env ++ build_task_env(task_actors_config)
+      end
+
     proxy_container =
       %{
         "name" => "sidecar",
         "image" => "#{annotations.proxy_image_tag}",
         "imagePullPolicy" => "Always",
-        "env" => @default_actor_host_function_env,
+        "env" => envs,
         "ports" => proxy_actor_host_function_ports,
         "livenessProbe" => %{
           "httpGet" => %{
@@ -310,6 +348,18 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do
     ]
   end
 
+  defp maybe_put_node_selector(spec, %{"nodeSelector" => selectors} = _topology) do
+    Map.merge(spec, %{"nodeSelector" => selectors})
+  end
+
+  defp maybe_put_node_selector(spec, _), do: spec
+
+  defp maybe_put_node_tolerations(spec, %{"tolerations" => tolerations} = _topology) do
+    Map.merge(spec, %{"tolerations" => tolerations})
+  end
+
+  defp maybe_put_node_tolerations(spec, _), do: spec
+
   defp maybe_put_ports_to_host_container(spec, %{"ports" => ports}) do
     Map.put(spec, "ports", ports)
   end

diff --git a/spawn_operator/spawn_operator/mix.lock b/spawn_operator/spawn_operator/mix.lock
@@ -56,7 +56,9 @@
   "nkeys": {:hex, :nkeys, "0.2.2", "b1ab3324ed4f3a2c9658d7e80feeef86b4d15fbfd12ca5c8cf068289f582fcfa", [:mix], [{:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}], "hexpm", "3578802427b8d1d11ea6dd785c2ab774f527e2c3e449e67bd34612ab71ca471d"},
   "opentelemetry": {:hex, :opentelemetry, "1.4.0", "f928923ed80adb5eb7894bac22e9a198478e6a8f04020ae1d6f289fdcad0b498", [:rebar3], [{:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "50b32ce127413e5d87b092b4d210a3449ea80cd8224090fe68d73d576a3faa15"},
   "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"},
+  "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"},
   "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.7.0", "dec4e90c0667cf11a3642f7fe71982dbc0c6bfbb8725a0b13766830718cf0d98", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.4.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "d0f25f6439ec43f2561537c3fabbe177b38547cddaa3a692cbb8f4770dbefc1e"},
+  "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"},
   "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"},
   "owl": {:hex, :owl, "0.8.0", "0ef925cb784311093d4e3734822960cbdbdb13b095d748bb5bc82abcd5b56732", [:mix], [], "hexpm", "0a5586ceb1a12f4bbda90e330c20e6ea034552335d09466c10e4218c98977529"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},

diff --git a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs
@@ -11,13 +11,15 @@ defmodule DeploymentTest do
     simple_host_with_ports = build_simple_actor_host_with_ports()
     simple_actor_host_with_volume_mounts = build_simple_actor_host_with_volume_mounts()
     embedded_actor_host = build_embedded_actor_host()
+    embedded_actor_host_with_node_selector = build_embedded_actor_host_with_node_selector()
     embedded_actor_host_with_volume_mounts = build_embedded_actor_host_with_volume_mounts()
 
     %{
       simple_host: simple_host,
       simple_host_with_ports: simple_host_with_ports,
       simple_actor_host_with_volume_mounts: simple_actor_host_with_volume_mounts,
       embedded_actor_host: embedded_actor_host,
+      embedded_actor_host_with_node_selector: embedded_actor_host_with_node_selector,
       embedded_actor_host_with_volume_mounts: embedded_actor_host_with_volume_mounts
     }
   end
@@ -170,6 +172,154 @@ defmodule DeploymentTest do
              } == build_host_deploy(embedded_actor_host)
     end
 
+    test "generate embedded deployment with defaults and node selector", ctx do
+      %{
+        embedded_actor_host_with_node_selector: embedded_actor_host_with_node_selector
+      } = ctx
+
+      assert %{
+               "apiVersion" => "apps/v1",
+               "kind" => "Deployment",
+               "metadata" => %{
+                 "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"},
+                 "name" => "spawn-test",
+                 "namespace" => "default"
+               },
+               "spec" => %{
+                 "replicas" => 1,
+                 "selector" => %{
+                   "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}
+                 },
+                 "strategy" => %{
+                   "rollingUpdate" => %{"maxSurge" => "50%", "maxUnavailable" => 0},
+                   "type" => "RollingUpdate"
+                 },
+                 "template" => %{
+                   "metadata" => %{
+                     "annotations" => %{
+                       "prometheus.io/path" => "/metrics",
+                       "prometheus.io/port" => "9001",
+                       "prometheus.io/scrape" => "true"
+                     },
+                     "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}
+                   },
+                   "spec" => %{
+                     "affinity" => %{
+                       "podAffinity" => %{
+                         "preferredDuringSchedulingIgnoredDuringExecution" => [
+                           %{
+                             "weight" => 50,
+                             "podAffinityTerm" => %{
+                               "labelSelector" => %{
+                                 "matchExpressions" => [
+                                   %{
+                                     "key" => "actor-system",
+                                     "operator" => "In",
+                                     "values" => [
+                                       "spawn-system"
+                                     ]
+                                   }
+                                 ]
+                               },
+                               "topologyKey" => "kubernetes.io/hostname"
+                             }
+                           }
+                         ]
+                       },
+                       "podAntiAffinity" => %{
+                         "preferredDuringSchedulingIgnoredDuringExecution" => [
+                           %{
+                             "weight" => 100,
+                             "podAffinityTerm" => %{
+                               "labelSelector" => %{
+                                 "matchExpressions" => [
+                                   %{
+                                     "key" => "app",
+                                     "operator" => "In",
+                                     "values" => [
+                                       "spawn-test"
+                                     ]
+                                   }
+                                 ]
+                               },
+                               "topologyKey" => "kubernetes.io/hostname"
+                             }
+                           }
+                         ]
+                       }
+                     },
+                     "containers" => [
+                       %{
+                         "env" => [
+                           %{"name" => "RELEASE_NAME", "value" => "spawn"},
+                           %{
+                             "name" => "NAMESPACE",
+                             "valueFrom" => %{
+                               "fieldRef" => %{"fieldPath" => "metadata.namespace"}
+                             }
+                           },
+                           %{
+                             "name" => "POD_IP",
+                             "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}}
+                           },
+                           %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"},
+                           %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"},
+                           %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"},
+                           %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"}
+                         ],
+                         "envFrom" => [
+                           %{"configMapRef" => %{"name" => "spawn-test-sidecar-cm"}},
+                           %{"secretRef" => %{"name" => "spawn-system-secret"}}
+                         ],
+                         "image" => "eigr/spawn-test:latest",
+                         "name" => "actorhost",
+                         "ports" => [
+                           %{"containerPort" => 4369, "name" => "epmd"},
+                           %{"containerPort" => 9001, "name" => "proxy-http"}
+                         ],
+                         "resources" => %{
+                           "requests" => %{
+                             "cpu" => "100m",
+                             "ephemeral-storage" => "1M",
+                             "memory" => "80Mi"
+                           }
+                         },
+                         "volumeMounts" => [%{"mountPath" => "/app/certs", "name" => "certs"}]
+                       }
+                     ],
+                     "terminationGracePeriodSeconds" => 405,
+                     "initContainers" => [
+                       %{
+                         "args" => [
+                           "--environment",
+                           :prod,
+                           "--secret",
+                           "tls-certs",
+                           "--namespace",
+                           "default",
+                           "--service",
+                           "spawn-system",
+                           "--to",
+                           "default"
+                         ],
+                         "image" => "ghcr.io/eigr/spawn-initializer:1.4.3",
+                         "name" => "init-certificates"
+                       }
+                     ],
+                     "serviceAccountName" => "spawn-system-sa",
+                     "volumes" => [
+                       %{
+                         "name" => "certs",
+                         "secret" => %{"optional" => true, "secretName" => "tls-certs"}
+                       }
+                     ],
+                     "nodeSelector" => %{"gpu" => "false"}
+                   }
+                 }
+               }
+             } == build_host_deploy(embedded_actor_host_with_node_selector)
+    end
+
     test "generate embedded deployment with volumeMount", ctx do
       %{
         embedded_actor_host_with_volume_mounts: embedded_actor_host_with_volume_mounts

diff --git a/spawn_operator/spawn_operator/test/support/factory.ex b/spawn_operator/spawn_operator/test/support/factory.ex
@@ -20,6 +20,30 @@ defmodule SpawnOperator.FactoryTest do
     }
   end
 
+  def build_embedded_actor_host_with_node_selector(attrs \\ []) do
+    %{
+      "apiVersion" => "spawn-eigr.io/v1",
+      "kind" => "ActorHost",
+      "metadata" => %{
+        "name" => attrs[:name] || "spawn-test",
+        "system" => "spawn-system",
+        "namespace" => "default",
+        "generation" => 1
+      },
+      "spec" => %{
+        "topology" => %{
+          "nodeSelector" => %{
+            "gpu" => "false"
+          }
+        },
+        "host" => %{
+          "embedded" => true,
+          "image" => attrs[:host_image] || "eigr/spawn-test:latest"
+        }
+      }
+    }
+  end
+
   def build_embedded_actor_host_with_volume_mounts(attrs \\ []) do
     %{
       "apiVersion" => "spawn-eigr.io/v1",