diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index c6234297b3e07..612bca0ddd3ec 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -640,6 +640,17 @@ spec: logFormat: json ``` +### Environment Variables +```yaml +spec: + kubeAPIServer: + env: + - name: GOMEMLIMIT + value: "2750MiB" + - name: GOGC + value: 50 +``` + ## externalDns This block contains configuration options for your `external-DNS` provider. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 87ce65120896d..0728c24d749bf 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1885,6 +1885,129 @@ spec: description: EncryptionProviderConfig enables encryption at rest for secrets. type: string + env: + description: |- + Env allows users to pass in env variables to the apiserver container. + This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver + This also allows the flexibility for adding any other variables for future use cases + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array etcdCaFile: description: EtcdCAFile is the path to a ca certificate type: string diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 8058e7f85a393..f1636a44b4674 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -699,7 +699,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops container := &v1.Container{ Name: "kube-apiserver", Image: image, - Env: proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy), + Env: append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...), LivenessProbe: livenessProbe, ReadinessProbe: readinessProbe, StartupProbe: startupProbe, diff --git a/nodeup/pkg/model/kube_apiserver_test.go b/nodeup/pkg/model/kube_apiserver_test.go index 4937e0287439e..f39e44c984703 100644 --- a/nodeup/pkg/model/kube_apiserver_test.go +++ b/nodeup/pkg/model/kube_apiserver_test.go @@ -204,3 +204,10 @@ func TestKubeAPIServerBuilderARM64(t *testing.T) { return builder.Build(target) }) } + +func TestKubeAPIServerEnvBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/envvars", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.NodeupModelBuilderContext) error { + builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} diff --git a/nodeup/pkg/model/tests/golden/envvars/cluster.yaml b/nodeup/pkg/model/tests/golden/envvars/cluster.yaml new file mode 100644 index 0000000000000..7a8113f7898a6 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/cluster.yaml @@ -0,0 +1,72 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: main + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: events + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events + iam: {} + kubeAPIServer: + env: + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: GOGC + value: "50" + kubelet: + anonymousAuth: false + kubernetesVersion: v1.28.0 + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: ami-1234 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kops-controller.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kops-controller.yaml new file mode 100644 index 0000000000000..922fc937161df --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kops-controller.yaml @@ -0,0 +1,112 @@ +mode: "0755" +path: /etc/kubernetes/kops-controller +type: directory +--- +contents: | + kubernetes-ca: "3" + service-account: "2" +mode: "0600" +owner: kops-controller +path: /etc/kubernetes/kops-controller/keypair-ids.yaml +type: file +--- +contents: + task: + Name: kops-controller + alternateNames: + - kops-controller.internal.minimal.example.com + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kops-controller + type: server +mode: "0644" +owner: kops-controller +path: /etc/kubernetes/kops-controller/kops-controller.crt +type: file +--- +contents: + task: + Name: kops-controller + alternateNames: + - kops-controller.internal.minimal.example.com + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kops-controller + type: server +mode: "0600" +owner: kops-controller +path: /etc/kubernetes/kops-controller/kops-controller.key +type: file +--- +contents: | + -----BEGIN CERTIFICATE----- + MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw + FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy + MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA + A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd + XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY + OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1 + YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC + An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA + NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r + hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO + HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe + rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt + O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy + VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh + 8yGeRx9AbknHh4Ia + -----END CERTIFICATE----- +mode: "0600" +owner: kops-controller +path: /etc/kubernetes/kops-controller/kubernetes-ca.crt +type: file +--- +contents: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH + AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA + gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF + GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/ + P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx + 9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI + Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9 + /Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM + 0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t + vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd + cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G + 8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj + 22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1 + AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV + 99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs + z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg + ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK + sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7 + KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i + Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T + t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q + dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z + worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu + BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq + Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw== + -----END RSA PRIVATE KEY----- +mode: "0600" +owner: kops-controller +path: /etc/kubernetes/kops-controller/kubernetes-ca.key +type: file +--- +Name: kops-controller +alternateNames: +- kops-controller.internal.minimal.example.com +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kops-controller +type: server +--- +Name: kops-controller +home: "" +shell: /sbin/nologin +uid: 10011 diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-apiserver.yaml new file mode 100644 index 0000000000000..c8c80dc738b93 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-apiserver.yaml @@ -0,0 +1,376 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + dns.alpha.kubernetes.io/external: api.minimal.example.com + dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com + kubectl.kubernetes.io/default-container: kube-apiserver + creationTimestamp: null + labels: + k8s-app: kube-apiserver + name: kube-apiserver + namespace: kube-system + spec: + containers: + - args: + - --log-file=/var/log/kube-apiserver.log + - --also-stdout + - /usr/local/bin/kube-apiserver + - --allow-privileged=true + - --anonymous-auth=false + - --api-audiences=kubernetes.svc.default + - --apiserver-count=1 + - --authorization-mode=AlwaysAllow + - --bind-address=0.0.0.0 + - --client-ca-file=/srv/kubernetes/ca.crt + - --cloud-config=/etc/kubernetes/in-tree-cloud.config + - --cloud-provider=external + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota + - --etcd-cafile=/srv/kubernetes/kube-apiserver/etcd-ca.crt + - --etcd-certfile=/srv/kubernetes/kube-apiserver/etcd-client.crt + - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key + - --etcd-servers-overrides=/events#https://127.0.0.1:4002 + - --etcd-servers=https://127.0.0.1:4001 + - --feature-gates=InTreePluginAWSUnregister=true + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key + - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP + - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt + - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key + - --requestheader-allowed-names=aggregator + - --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --secure-port=443 + - --service-account-issuer=https://api.internal.minimal.example.com + - --service-account-jwks-uri=https://api.internal.minimal.example.com/openid/v1/jwks + - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub + - --service-account-signing-key-file=/srv/kubernetes/kube-apiserver/service-account.key + - --service-cluster-ip-range=100.64.0.0/13 + - --storage-backend=etcd3 + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key + - --v=2 + command: + - /go-runner + env: + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + divisor: "1" + resource: limits.memory + - name: GOGC + value: "50" + image: registry.k8s.io/kube-apiserver:v1.28.0 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 45 + timeoutSeconds: 15 + name: kube-apiserver + ports: + - containerPort: 443 + hostPort: 443 + name: https + resources: + requests: + cpu: 150m + volumeMounts: + - mountPath: /var/log/kube-apiserver.log + name: logfile + - mountPath: /etc/ssl + name: etcssl + readOnly: true + - mountPath: /etc/pki/tls + name: etcpkitls + readOnly: true + - mountPath: /etc/pki/ca-trust + name: etcpkica-trust + readOnly: true + - mountPath: /usr/share/ssl + name: usrsharessl + readOnly: true + - mountPath: /usr/ssl + name: usrssl + readOnly: true + - mountPath: /usr/lib/ssl + name: usrlibssl + readOnly: true + - mountPath: /usr/local/openssl + name: usrlocalopenssl + readOnly: true + - mountPath: /var/ssl + name: varssl + readOnly: true + - mountPath: /etc/openssl + name: etcopenssl + readOnly: true + - mountPath: /etc/kubernetes/in-tree-cloud.config + name: cloudconfig + readOnly: true + - mountPath: /srv/kubernetes/ca.crt + name: kubernetesca + readOnly: true + - mountPath: /srv/kubernetes/kube-apiserver + name: srvkapi + readOnly: true + - mountPath: /srv/sshproxy + name: srvsshproxy + readOnly: true + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/kube-apiserver.log + name: logfile + - hostPath: + path: /etc/ssl + name: etcssl + - hostPath: + path: /etc/pki/tls + name: etcpkitls + - hostPath: + path: /etc/pki/ca-trust + name: etcpkica-trust + - hostPath: + path: /usr/share/ssl + name: usrsharessl + - hostPath: + path: /usr/ssl + name: usrssl + - hostPath: + path: /usr/lib/ssl + name: usrlibssl + - hostPath: + path: /usr/local/openssl + name: usrlocalopenssl + - hostPath: + path: /var/ssl + name: varssl + - hostPath: + path: /etc/openssl + name: etcopenssl + - hostPath: + path: /etc/kubernetes/in-tree-cloud.config + name: cloudconfig + - hostPath: + path: /srv/kubernetes/ca.crt + name: kubernetesca + - hostPath: + path: /srv/kubernetes/kube-apiserver + name: srvkapi + - hostPath: + path: /srv/sshproxy + name: srvsshproxy + status: {} +path: /etc/kubernetes/manifests/kube-apiserver.manifest +type: file +--- +mode: "0755" +path: /srv/kubernetes/kube-apiserver +type: directory +--- +contents: "" +mode: "0644" +path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt +type: file +--- +contents: + task: + Name: apiserver-aggregator + keypairID: "" + signer: apiserver-aggregator-ca + subject: + CommonName: aggregator + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt +type: file +--- +contents: + task: + Name: apiserver-aggregator + keypairID: "" + signer: apiserver-aggregator-ca + subject: + CommonName: aggregator + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key +type: file +--- +contents: "" +mode: "0644" +path: /srv/kubernetes/kube-apiserver/etcd-ca.crt +type: file +--- +contents: + task: + Name: etcd-client + keypairID: "" + signer: etcd-clients-ca + subject: + CommonName: kube-apiserver + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/etcd-client.crt +type: file +--- +contents: + task: + Name: etcd-client + keypairID: "" + signer: etcd-clients-ca + subject: + CommonName: kube-apiserver + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/etcd-client.key +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- +contents: | + -----BEGIN RSA PRIVATE KEY----- + MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4 + 9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R + 2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo + xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+ + ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr + Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh + AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY + -----END RSA PRIVATE KEY----- +mode: "0600" +path: /srv/kubernetes/kube-apiserver/service-account.key +type: file +--- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/kube-apiserver/service-account.pub +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-apiserver.log +type: file +--- +Name: apiserver-aggregator +keypairID: "" +signer: apiserver-aggregator-ca +subject: + CommonName: aggregator +type: client +--- +Name: etcd-client +keypairID: "" +signer: etcd-clients-ca +subject: + CommonName: kube-apiserver +type: client +--- +Name: kubelet-api +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubelet-api +type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-controller-manager.yaml new file mode 100644 index 0000000000000..afd6cb3402305 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-controller-manager.yaml @@ -0,0 +1,331 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + k8s-app: kube-controller-manager + name: kube-controller-manager + namespace: kube-system + spec: + containers: + - args: + - --log-file=/var/log/kube-controller-manager.log + - --also-stdout + - /usr/local/bin/kube-controller-manager + - --allocate-node-cidrs=true + - --attach-detach-reconcile-sync-period=1m0s + - --authentication-kubeconfig=/var/lib/kube-controller-manager/kubeconfig + - --authorization-kubeconfig=/var/lib/kube-controller-manager/kubeconfig + - --cloud-config=/etc/kubernetes/in-tree-cloud.config + - --cloud-provider=external + - --cluster-cidr=100.96.0.0/11 + - --cluster-name=minimal.example.com + - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt + - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key + - --configure-cloud-routes=true + - --feature-gates=InTreePluginAWSUnregister=true + - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/ + - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig + - --leader-elect=true + - --root-ca-file=/srv/kubernetes/ca.crt + - --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key + - --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key + - --use-service-account-credentials=true + - --v=2 + command: + - /go-runner + image: registry.k8s.io/kube-controller-manager:v1.28.0 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10257 + scheme: HTTPS + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: kube-controller-manager + resources: + requests: + cpu: 100m + volumeMounts: + - mountPath: /var/log/kube-controller-manager.log + name: logfile + - mountPath: /etc/ssl + name: etcssl + readOnly: true + - mountPath: /etc/pki/tls + name: etcpkitls + readOnly: true + - mountPath: /etc/pki/ca-trust + name: etcpkica-trust + readOnly: true + - mountPath: /usr/share/ssl + name: usrsharessl + readOnly: true + - mountPath: /usr/ssl + name: usrssl + readOnly: true + - mountPath: /usr/lib/ssl + name: usrlibssl + readOnly: true + - mountPath: /usr/local/openssl + name: usrlocalopenssl + readOnly: true + - mountPath: /var/ssl + name: varssl + readOnly: true + - mountPath: /etc/openssl + name: etcopenssl + readOnly: true + - mountPath: /etc/kubernetes/in-tree-cloud.config + name: cloudconfig + readOnly: true + - mountPath: /srv/kubernetes/ca.crt + name: cabundle + readOnly: true + - mountPath: /srv/kubernetes/kube-controller-manager + name: srvkcm + readOnly: true + - mountPath: /var/lib/kube-controller-manager + name: varlibkcm + readOnly: true + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ + name: volplugins + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/kube-controller-manager.log + name: logfile + - hostPath: + path: /etc/ssl + name: etcssl + - hostPath: + path: /etc/pki/tls + name: etcpkitls + - hostPath: + path: /etc/pki/ca-trust + name: etcpkica-trust + - hostPath: + path: /usr/share/ssl + name: usrsharessl + - hostPath: + path: /usr/ssl + name: usrssl + - hostPath: + path: /usr/lib/ssl + name: usrlibssl + - hostPath: + path: /usr/local/openssl + name: usrlocalopenssl + - hostPath: + path: /var/ssl + name: varssl + - hostPath: + path: /etc/openssl + name: etcopenssl + - hostPath: + path: /etc/kubernetes/in-tree-cloud.config + name: cloudconfig + - hostPath: + path: /srv/kubernetes/ca.crt + name: cabundle + - hostPath: + path: /srv/kubernetes/kube-controller-manager + name: srvkcm + - hostPath: + path: /var/lib/kube-controller-manager + name: varlibkcm + - hostPath: + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/ + name: volplugins + status: {} +path: /etc/kubernetes/manifests/kube-controller-manager.manifest +type: file +--- +mode: "0755" +path: /srv/kubernetes/kube-controller-manager +type: directory +--- +contents: | + -----BEGIN CERTIFICATE----- + MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw + FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy + MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA + A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd + XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY + OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1 + YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC + An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA + NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r + hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO + HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe + rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt + O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy + VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh + 8yGeRx9AbknHh4Ia + -----END CERTIFICATE----- +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/ca.crt +type: file +--- +contents: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH + AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA + gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF + GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/ + P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx + 9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI + Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9 + /Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM + 0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t + vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd + cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G + 8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj + 22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1 + AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV + 99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs + z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg + ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK + sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7 + KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i + Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T + t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q + dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z + worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu + BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq + Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw== + -----END RSA PRIVATE KEY----- +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/ca.key +type: file +--- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0644" +path: /srv/kubernetes/kube-controller-manager/server.crt +type: file +--- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/server.key +type: file +--- +contents: | + -----BEGIN RSA PRIVATE KEY----- + MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4 + 9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R + 2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo + xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+ + ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr + Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh + AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY + -----END RSA PRIVATE KEY----- +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/service-account.key +type: file +--- +contents: + task: + CA: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client + Cert: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client + Key: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client + Name: kube-controller-manager + ServerURL: https://127.0.0.1 +mode: "0400" +path: /var/lib/kube-controller-manager/kubeconfig +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-controller-manager.log +type: file +--- +Name: kube-controller-manager +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: system:kube-controller-manager +type: client +--- +Name: kube-controller-manager-server +alternateNames: +- kube-controller-manager.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-controller-manager +type: server +--- +CA: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client +Cert: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client +Key: + task: + Name: kube-controller-manager + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-controller-manager + type: client +Name: kube-controller-manager +ServerURL: https://127.0.0.1 diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-proxy.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-proxy.yaml new file mode 100644 index 0000000000000..53173b7e58c2f --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-proxy.yaml @@ -0,0 +1,145 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + k8s-app: kube-proxy + kubernetes.io/managed-by: nodeup + tier: node + name: kube-proxy + namespace: kube-system + spec: + containers: + - args: + - --log-file=/var/log/kube-proxy.log + - --also-stdout + - /usr/local/bin/kube-proxy + - --cluster-cidr=100.96.0.0/11 + - --conntrack-max-per-core=131072 + - --kubeconfig=/var/lib/kube-proxy/kubeconfig + - --master=https://127.0.0.1 + - --oom-score-adj=-998 + - --v=2 + command: + - /go-runner + image: registry.k8s.io/kube-proxy:v1.28.0 + name: kube-proxy + resources: + requests: + cpu: 100m + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/log/kube-proxy.log + name: logfile + - mountPath: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + readOnly: true + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-hosts + readOnly: true + - mountPath: /run/xtables.lock + name: iptableslock + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/log/kube-proxy.log + name: logfile + - hostPath: + path: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + - hostPath: + path: /lib/modules + name: modules + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-hosts + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + status: {} +path: /etc/kubernetes/manifests/kube-proxy.manifest +type: file +--- +beforeServices: +- kubelet.service +contents: + task: + CA: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client + Cert: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client + Key: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client + Name: kube-proxy + ServerURL: https://127.0.0.1 +mode: "0400" +path: /var/lib/kube-proxy/kubeconfig +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-proxy.log +type: file +--- +Name: kube-proxy +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: system:kube-proxy +type: client +--- +CA: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client +Cert: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client +Key: + task: + Name: kube-proxy + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-proxy + type: client +Name: kube-proxy +ServerURL: https://127.0.0.1 diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-scheduler.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-scheduler.yaml new file mode 100644 index 0000000000000..932291a39f898 --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-scheduler.yaml @@ -0,0 +1,187 @@ +contents: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + labels: + k8s-app: kube-scheduler + name: kube-scheduler + namespace: kube-system + spec: + containers: + - args: + - --log-file=/var/log/kube-scheduler.log + - --also-stdout + - /usr/local/bin/kube-scheduler + - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig + - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig + - --config=/var/lib/kube-scheduler/config.yaml + - --feature-gates=InTreePluginAWSUnregister=true + - --leader-elect=true + - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key + - --v=2 + command: + - /go-runner + image: registry.k8s.io/kube-scheduler:v1.28.0 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: kube-scheduler + resources: + requests: + cpu: 100m + volumeMounts: + - mountPath: /var/lib/kube-scheduler + name: varlibkubescheduler + readOnly: true + - mountPath: /srv/kubernetes/kube-scheduler + name: srvscheduler + readOnly: true + - mountPath: /var/log/kube-scheduler.log + name: logfile + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: /var/lib/kube-scheduler + name: varlibkubescheduler + - hostPath: + path: /srv/kubernetes/kube-scheduler + name: srvscheduler + - hostPath: + path: /var/log/kube-scheduler.log + name: logfile + status: {} +path: /etc/kubernetes/manifests/kube-scheduler.manifest +type: file +--- +mode: "0755" +path: /srv/kubernetes/kube-scheduler +type: directory +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0644" +path: /srv/kubernetes/kube-scheduler/server.crt +type: file +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0600" +path: /srv/kubernetes/kube-scheduler/server.key +type: file +--- +contents: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration +mode: "0400" +path: /var/lib/kube-scheduler/config.yaml +type: file +--- +contents: + task: + CA: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client + Cert: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client + Key: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client + Name: kube-scheduler + ServerURL: https://127.0.0.1 +mode: "0400" +path: /var/lib/kube-scheduler/kubeconfig +type: file +--- +contents: "" +ifNotExists: true +mode: "0400" +path: /var/log/kube-scheduler.log +type: file +--- +Name: kube-scheduler +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: system:kube-scheduler +type: client +--- +Name: kube-scheduler-server +alternateNames: +- kube-scheduler.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-scheduler +type: server +--- +CA: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client +Cert: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client +Key: + task: + Name: kube-scheduler + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: system:kube-scheduler + type: client +Name: kube-scheduler +ServerURL: https://127.0.0.1 diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-kubectl.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-kubectl.yaml new file mode 100644 index 0000000000000..4f247b7032dae --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kubectl.yaml @@ -0,0 +1,87 @@ +contents: + Asset: + AssetPath: /path/to/kubectl/asset + Key: kubectl +mode: "0755" +path: /opt/kops/bin/kubectl +type: file +--- +contents: + task: + CA: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client + Cert: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client + Key: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client + Name: kubecfg + ServerURL: https://127.0.0.1 +mode: "0400" +path: /var/lib/kubectl/kubeconfig +type: file +--- +Name: kubecfg +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubecfg + Organization: + - system:masters +type: client +--- +CA: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client +Cert: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client +Key: + task: + Name: kubecfg + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubecfg + Organization: + - system:masters + type: client +Name: kubecfg +ServerURL: https://127.0.0.1 diff --git a/nodeup/pkg/model/tests/golden/envvars/tasks-secret.yaml b/nodeup/pkg/model/tests/golden/envvars/tasks-secret.yaml new file mode 100644 index 0000000000000..3bcafb654de2d --- /dev/null +++ b/nodeup/pkg/model/tests/golden/envvars/tasks-secret.yaml @@ -0,0 +1,32 @@ +contents: | + -----BEGIN CERTIFICATE----- + MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw + FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy + MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA + A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd + XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY + OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1 + YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC + An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA + NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB + /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r + hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO + HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe + rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt + O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy + VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh + 8yGeRx9AbknHh4Ia + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2 + aWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw + FgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA + o4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD + q4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T + AQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI + hvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J + A1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA + -----END CERTIFICATE----- +mode: "0600" +path: /srv/kubernetes/ca.crt +type: file diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index f1ec517829c62..ca849a52ff0c6 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -17,6 +17,7 @@ limitations under the License. package kops import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -548,6 +549,11 @@ type KubeAPIServerConfig struct { DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"` // DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration. DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"` + + // Env allows users to pass in env variables to the apiserver container. + // This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver + // This also allows the flexibility for adding any other variables for future use cases + Env []corev1.EnvVar `json:"env,omitempty"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 7f1a584a135d1..9dbf072ba77b9 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha2 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -555,6 +556,11 @@ type KubeAPIServerConfig struct { DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"` // DefaultUnreachableTolerationSeconds DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"` + + // Env allows users to pass in env variables to the apiserver container. + // This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver + // This also allows the flexibility for adding any other variables for future use cases + Env []corev1.EnvVar `json:"env,omitempty"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 11035647427aa..a7f082b2b2b35 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4973,6 +4973,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.CorsAllowedOrigins = in.CorsAllowedOrigins out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds + out.Env = in.Env return nil } @@ -5089,6 +5090,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.CorsAllowedOrigins = in.CorsAllowedOrigins out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds + out.Env = in.Env return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index fc07374edeacf..ddc1e80c4184a 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -3425,6 +3425,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(int64) **out = **in } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.defaults.go b/pkg/apis/kops/v1alpha2/zz_generated.defaults.go index 84efc75b28452..fcc8862296d40 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.defaults.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.defaults.go @@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_Cluster(in *Cluster) { SetDefaults_ClusterSpec(&in.Spec) + if in.Spec.KubeAPIServer != nil { + for i := range in.Spec.KubeAPIServer.Env { + a := &in.Spec.KubeAPIServer.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.ConfigMapKeyRef != nil { + if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" { + a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = "" + } + } + if a.ValueFrom.SecretKeyRef != nil { + if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" { + a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = "" + } + } + } + } + } } func SetObjectDefaults_ClusterList(in *ClusterList) { diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index 91001c11fe4fd..621d68b9e7257 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -546,6 +547,11 @@ type KubeAPIServerConfig struct { DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"` // DefaultUnreachableTolerationSeconds DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"` + + // Env allows users to pass in env variables to the apiserver container. + // This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver + // This also allows the flexibility for adding any other variables for future use cases + Env []corev1.EnvVar `json:"env,omitempty"` } // KubeControllerManagerConfig is the configuration for the controller diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 2f617ac479889..23e8effbe456b 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -5369,6 +5369,7 @@ func autoConvert_v1alpha3_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.CorsAllowedOrigins = in.CorsAllowedOrigins out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds + out.Env = in.Env return nil } @@ -5485,6 +5486,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha3_KubeAPIServerConfig(in *ko out.CorsAllowedOrigins = in.CorsAllowedOrigins out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds + out.Env = in.Env return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 8788745fc9cfe..cf796dbdc43fd 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -3399,6 +3399,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(int64) **out = **in } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.defaults.go b/pkg/apis/kops/v1alpha3/zz_generated.defaults.go index c34ffad064802..54953b47906e6 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.defaults.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.defaults.go @@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error { func SetObjectDefaults_Cluster(in *Cluster) { SetDefaults_ClusterSpec(&in.Spec) + if in.Spec.KubeAPIServer != nil { + for i := range in.Spec.KubeAPIServer.Env { + a := &in.Spec.KubeAPIServer.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.ConfigMapKeyRef != nil { + if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" { + a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = "" + } + } + if a.ValueFrom.SecretKeyRef != nil { + if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" { + a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = "" + } + } + } + } + } } func SetObjectDefaults_ClusterList(in *ClusterList) { diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index aca28f0979e3d..2ecc1e9fdf47a 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -3502,6 +3502,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { *out = new(int64) **out = **in } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return }