From 0b8d8545bc99fa76fb381d13d253c98980431dca Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Sat, 28 Sep 2024 15:51:59 -0300 Subject: [PATCH] Added topology and node selector --- examples/topology/host.yaml | 48 ++++-- .../spawn_operator/k8s/proxy/deployment.ex | 62 +++++++- spawn_operator/spawn_operator/mix.lock | 2 + .../resources/actorhost/deployment_test.exs | 150 ++++++++++++++++++ .../spawn_operator/test/support/factory.ex | 24 +++ 5 files changed, 269 insertions(+), 17 deletions(-) diff --git a/examples/topology/host.yaml b/examples/topology/host.yaml index 8d1a08f6..2fe3b6a1 100644 --- a/examples/topology/host.yaml +++ b/examples/topology/host.yaml @@ -8,24 +8,50 @@ metadata: # Mandatory. Name of the ActorSystem declared in ActorSystem CRD spawn-eigr.io/actor-system: spawn-system spec: + topology: + # affinity: + # podAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 50 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: actor-system + # operator: In + # values: + # - system + # topologyKey: kubernetes.io/hostname + + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - app_name + # topologyKey: kubernetes.io/hostname + nodeSelector: + gpu: "false" + tolerations: + - key: "cpu-machines" + operator: "Exists" + effect: "NoExecute" host: image: eigr/spawn-springboot-examples:0.5.5 # Mandatory - topology: - nodeSelector: - gpu: "false" - tolerations: - - key: "cpu-machines" - operator: "Exists" - effect: "NoExecute" - # this configure podTemplate for Task Actors - taskActors: - - parentName: Jose + # this configure podTemplate for Task Actors + taskActors: + - parentName: Jose + topology: nodeSelector: gpu: "true" tolerations: - key: "gpu-machines" operator: "Exists" effect: "NoExecute" - - parentName: Franchesco + - parentName: Franchesco + topology: nodeSelector: beam: "true" diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex index e9c342b5..c2cca071 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex @@ -70,6 +70,9 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do } = _resource ) do host_params = Map.get(params, "host") + task_actors_config = Map.get(host_params, "taskActors", %{}) + topology = Map.get(params, "topology", %{}) + replicas = max(1, Map.get(params, "replicas", @default_actor_host_function_replicas)) embedded = Map.get(host_params, "embedded", false) @@ -109,8 +112,16 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do }, "spec" => %{ - "affinity" => Map.get(host_params, "affinity", build_affinity(system, name)), - "containers" => get_containers(embedded, system, name, host_params, annotations), + "affinity" => Map.get(topology, "affinity", build_affinity(system, name)), + "containers" => + get_containers( + embedded, + system, + name, + host_params, + annotations, + task_actors_config + ), "initContainers" => [ %{ "name" => "init-certificates", @@ -131,6 +142,8 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do ], "serviceAccountName" => "#{system}-sa" } + |> maybe_put_node_selector(topology) + |> maybe_put_node_tolerations(topology) |> maybe_put_volumes(params) |> maybe_set_termination_period(params) } @@ -185,10 +198,28 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do } end - defp get_containers(true, system, name, host_params, annotations) do + defp build_task_env(task_actors_config) do + value = + task_actors_config + |> Jason.encode!() + |> Base.encode32() + + [ + %{"name" => "SPAWN_PROXY_TASK_CONFIG", "value" => value} + ] + end + + defp get_containers(true, system, name, host_params, annotations, task_actors_config) do actor_host_function_image = Map.get(host_params, "image") - actor_host_function_envs = Map.get(host_params, "env", []) ++ @default_actor_host_function_env + actor_host_function_envs = + if length(Map.values(task_actors_config)) == 0 do + Map.get(host_params, "env", []) ++ @default_actor_host_function_env + else + Map.get(host_params, "env", []) ++ + @default_actor_host_function_env ++ + build_task_env(task_actors_config) + end proxy_http_port = String.to_integer(annotations.proxy_http_port) @@ -230,7 +261,7 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do ] end - defp get_containers(false, system, name, host_params, annotations) do + defp get_containers(false, system, name, host_params, annotations, task_actors_config) do actor_host_function_image = Map.get(host_params, "image") actor_host_function_envs = @@ -247,12 +278,19 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do %{"containerPort" => proxy_http_port, "name" => "proxy-http"} ] + envs = + if length(Map.values(task_actors_config)) == 0 do + @default_actor_host_function_env + else + @default_actor_host_function_env ++ build_task_env(task_actors_config) + end + proxy_container = %{ "name" => "sidecar", "image" => "#{annotations.proxy_image_tag}", "imagePullPolicy" => "Always", - "env" => @default_actor_host_function_env, + "env" => envs, "ports" => proxy_actor_host_function_ports, "livenessProbe" => %{ "httpGet" => %{ @@ -310,6 +348,18 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do ] end + defp maybe_put_node_selector(spec, %{"nodeSelector" => selectors} = _topology) do + Map.merge(spec, %{"nodeSelector" => selectors}) + end + + defp maybe_put_node_selector(spec, _), do: spec + + defp maybe_put_node_tolerations(spec, %{"tolerations" => tolerations} = _topology) do + Map.merge(spec, %{"tolerations" => tolerations}) + end + + defp maybe_put_node_tolerations(spec, _), do: spec + defp maybe_put_ports_to_host_container(spec, %{"ports" => ports}) do Map.put(spec, "ports", ports) end diff --git a/spawn_operator/spawn_operator/mix.lock b/spawn_operator/spawn_operator/mix.lock index 21e31005..cb1e4ae1 100644 --- a/spawn_operator/spawn_operator/mix.lock +++ b/spawn_operator/spawn_operator/mix.lock @@ -56,7 +56,9 @@ "nkeys": {:hex, :nkeys, "0.2.2", "b1ab3324ed4f3a2c9658d7e80feeef86b4d15fbfd12ca5c8cf068289f582fcfa", [:mix], [{:ed25519, "~> 1.3", [hex: :ed25519, repo: "hexpm", optional: false]}], "hexpm", "3578802427b8d1d11ea6dd785c2ab774f527e2c3e449e67bd34612ab71ca471d"}, "opentelemetry": {:hex, :opentelemetry, "1.4.0", "f928923ed80adb5eb7894bac22e9a198478e6a8f04020ae1d6f289fdcad0b498", [:rebar3], [{:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "50b32ce127413e5d87b092b4d210a3449ea80cd8224090fe68d73d576a3faa15"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.0", "03e2177f28dd8d11aaa88e8522c81c2f6a788170fe52f7a65262340961e663f9", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "b9e5ff775fd064fa098dba3c398490b77649a352b40b0b730a6b7dc0bdd68858"}, + "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.7.0", "dec4e90c0667cf11a3642f7fe71982dbc0c6bfbb8725a0b13766830718cf0d98", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.4.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "d0f25f6439ec43f2561537c3fabbe177b38547cddaa3a692cbb8f4770dbefc1e"}, + "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"}, "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, "owl": {:hex, :owl, "0.8.0", "0ef925cb784311093d4e3734822960cbdbdb13b095d748bb5bc82abcd5b56732", [:mix], [], "hexpm", "0a5586ceb1a12f4bbda90e330c20e6ea034552335d09466c10e4218c98977529"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, diff --git a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs index 8b3b7ad5..f8641ad8 100644 --- a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs +++ b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs @@ -11,6 +11,7 @@ defmodule DeploymentTest do simple_host_with_ports = build_simple_actor_host_with_ports() simple_actor_host_with_volume_mounts = build_simple_actor_host_with_volume_mounts() embedded_actor_host = build_embedded_actor_host() + embedded_actor_host_with_node_selector = build_embedded_actor_host_with_node_selector() embedded_actor_host_with_volume_mounts = build_embedded_actor_host_with_volume_mounts() %{ @@ -18,6 +19,7 @@ defmodule DeploymentTest do simple_host_with_ports: simple_host_with_ports, simple_actor_host_with_volume_mounts: simple_actor_host_with_volume_mounts, embedded_actor_host: embedded_actor_host, + embedded_actor_host_with_node_selector: embedded_actor_host_with_node_selector, embedded_actor_host_with_volume_mounts: embedded_actor_host_with_volume_mounts } end @@ -170,6 +172,154 @@ defmodule DeploymentTest do } == build_host_deploy(embedded_actor_host) end + test "generate embedded deployment with defaults and node selector", ctx do + %{ + embedded_actor_host_with_node_selector: embedded_actor_host_with_node_selector + } = ctx + + assert %{ + "apiVersion" => "apps/v1", + "kind" => "Deployment", + "metadata" => %{ + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}, + "name" => "spawn-test", + "namespace" => "default" + }, + "spec" => %{ + "replicas" => 1, + "selector" => %{ + "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} + }, + "strategy" => %{ + "rollingUpdate" => %{"maxSurge" => "50%", "maxUnavailable" => 0}, + "type" => "RollingUpdate" + }, + "template" => %{ + "metadata" => %{ + "annotations" => %{ + "prometheus.io/path" => "/metrics", + "prometheus.io/port" => "9001", + "prometheus.io/scrape" => "true" + }, + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} + }, + "spec" => %{ + "affinity" => %{ + "podAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "weight" => 50, + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "actor-system", + "operator" => "In", + "values" => [ + "spawn-system" + ] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + } + } + ] + }, + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "weight" => 100, + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "app", + "operator" => "In", + "values" => [ + "spawn-test" + ] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + } + } + ] + } + }, + "containers" => [ + %{ + "env" => [ + %{"name" => "RELEASE_NAME", "value" => "spawn"}, + %{ + "name" => "NAMESPACE", + "valueFrom" => %{ + "fieldRef" => %{"fieldPath" => "metadata.namespace"} + } + }, + %{ + "name" => "POD_IP", + "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} + }, + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} + ], + "envFrom" => [ + %{"configMapRef" => %{"name" => "spawn-test-sidecar-cm"}}, + %{"secretRef" => %{"name" => "spawn-system-secret"}} + ], + "image" => "eigr/spawn-test:latest", + "name" => "actorhost", + "ports" => [ + %{"containerPort" => 4369, "name" => "epmd"}, + %{"containerPort" => 9001, "name" => "proxy-http"} + ], + "resources" => %{ + "requests" => %{ + "cpu" => "100m", + "ephemeral-storage" => "1M", + "memory" => "80Mi" + } + }, + "volumeMounts" => [%{"mountPath" => "/app/certs", "name" => "certs"}] + } + ], + "terminationGracePeriodSeconds" => 405, + "initContainers" => [ + %{ + "args" => [ + "--environment", + :prod, + "--secret", + "tls-certs", + "--namespace", + "default", + "--service", + "spawn-system", + "--to", + "default" + ], + "image" => "ghcr.io/eigr/spawn-initializer:1.4.3", + "name" => "init-certificates" + } + ], + "serviceAccountName" => "spawn-system-sa", + "volumes" => [ + %{ + "name" => "certs", + "secret" => %{"optional" => true, "secretName" => "tls-certs"} + } + ], + "nodeSelector" => %{"gpu" => "false"} + } + } + } + } == build_host_deploy(embedded_actor_host_with_node_selector) + end + test "generate embedded deployment with volumeMount", ctx do %{ embedded_actor_host_with_volume_mounts: embedded_actor_host_with_volume_mounts diff --git a/spawn_operator/spawn_operator/test/support/factory.ex b/spawn_operator/spawn_operator/test/support/factory.ex index 221f9cb4..ece0423e 100644 --- a/spawn_operator/spawn_operator/test/support/factory.ex +++ b/spawn_operator/spawn_operator/test/support/factory.ex @@ -20,6 +20,30 @@ defmodule SpawnOperator.FactoryTest do } end + def build_embedded_actor_host_with_node_selector(attrs \\ []) do + %{ + "apiVersion" => "spawn-eigr.io/v1", + "kind" => "ActorHost", + "metadata" => %{ + "name" => attrs[:name] || "spawn-test", + "system" => "spawn-system", + "namespace" => "default", + "generation" => 1 + }, + "spec" => %{ + "topology" => %{ + "nodeSelector" => %{ + "gpu" => "false" + } + }, + "host" => %{ + "embedded" => true, + "image" => attrs[:host_image] || "eigr/spawn-test:latest" + } + } + } + end + def build_embedded_actor_host_with_volume_mounts(attrs \\ []) do %{ "apiVersion" => "spawn-eigr.io/v1",