diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 7fd8eff78..a550bf6fe 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index d94aa85a6..dd380aa5a 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 1c7a1af7c..7e6e8d709 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index deb91e618..defbcca80 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index e7a60d1b2..3db39d632 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -14,7 +14,8 @@ presubmits: - name: pull-machine-controller-e2e-aws - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -46,7 +47,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata - always_run: true + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -60,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -142,7 +144,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance - always_run: true + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -156,7 +159,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +190,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +221,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +252,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index f06fa7dd0..1cb8c860b 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 0a2ee8929..bed60b3bd 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 986bb7faf..059766bcc 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 5c85b2939..5f3721c08 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-gce - run_if_changed: "(pkg/cloudprovider/provider/gce/|pkg/userdata)" + always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -44,3 +44,37 @@ presubmits: cpu: 2 limits: memory: 7Gi + + - name: pull-machine-controller-e2e-gce-legacy-userdata + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-gce: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestGCEProvisioningE2E" + env: + - name: OPERATING_SYSTEM_MANAGER + value: "false" + - name: CLOUD_PROVIDER + value: gce + securityContext: + privileged: true + resources: + requests: + memory: 7Gi + cpu: 2 + limits: + memory: 7Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 7205e1492..b8655e8f5 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index ad712a605..adb808ae5 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index def0cb7de..12a0711ed 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 4c05da09b..31e53cf54 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 8213afc44..22ff68180 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index c9a33e4fc..0fa091d99 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index bc09746d8..8be8ec733 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 71ea8ab3e..a75d1f864 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -140,3 +140,35 @@ presubmits: cpu: 2 limits: memory: 7Gi + + - name: pull-machine-controller-e2e-vsphere-anti-affinity + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-vsphere: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestVsphereAntiAffinityProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere + securityContext: + privileged: true + resources: + requests: + memory: 7Gi + cpu: 2 + limits: + memory: 7Gi diff --git a/.prow/verify.yaml b/.prow/verify.yaml index edee2d747..0ba98100b 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: diff --git a/Dockerfile b/Dockerfile index 5651e7673..68bd62d93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.0 +ARG GO_VERSION=1.22.1 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 0ebd201ce..f9e88fc8b 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.0 +GO_VERSION ?= 1.22.1 GOOS ?= $(shell go env GOOS) diff --git a/README.md b/README.md index 470eb79aa..f0e41d67f 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ Currently supported K8S versions are: - 1.29 - 1.28 - 1.27 -- 1.26 ### Community Providers diff --git a/go.sum b/go.sum index 4b957bea7..004b2e405 100644 --- a/go.sum +++ b/go.sum @@ -447,8 +447,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -1310,6 +1308,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 8657d5ab6..5200ec615 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.22.0 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.1 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 3646eec8f..5465f0aef 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 522e77ff9..18cef6a7b 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -103,12 +103,10 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, if err := clonedVMTask.Wait(ctx); err != nil { return nil, fmt.Errorf("error when waiting for result of clone task: %w", err) } - virtualMachine, err := session.Finder.VirtualMachine(ctx, vmName) if err != nil { return nil, fmt.Errorf("failed to get virtual machine object after cloning: %w", err) } - vmDevices, err := virtualMachine.Device(ctx) if err != nil { return nil, fmt.Errorf("failed to list devices of template VM: %w", err) @@ -138,7 +136,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, guestInfoUserData = "guestinfo.ignition.config.data" guestInfoUserDataEncoding = "guestinfo.ignition.config.data.encoding" - for _, item := range mvm.Config.VAppConfig.GetVmConfigInfo().Property { switch item.Id { case guestInfoUserData: @@ -170,7 +167,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, } diskUUIDEnabled := true - var deviceSpecs []types.BaseVirtualDeviceConfigSpec if config.DiskSizeGB != nil { disks, err := getDisksFromVM(ctx, virtualMachine) @@ -221,7 +217,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, if err := removeFloppyDevice(ctx, virtualMachine); err != nil { return nil, fmt.Errorf("failed to remove floppy device: %w", err) } - return virtualMachine, nil } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 1156ef2f2..3166ac24f 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -23,7 +23,6 @@ import ( "net/url" "os" "strings" - "sync" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" @@ -49,7 +48,6 @@ import ( type provider struct { configVarResolver *providerconfig.ConfigVarResolver - mutex sync.Mutex } // New returns a VSphere provider. @@ -388,8 +386,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * } if config.VMAntiAffinity { - machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] - if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { + if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) } } @@ -456,6 +453,12 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to delete tags: %w", err) } + if config.VMAntiAffinity { + if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { + return false, fmt.Errorf("failed to update VMs in anti-affinity rule: %w", err) + } + } + powerState, err := virtualMachine.PowerState(ctx) if err != nil { return false, fmt.Errorf("failed to get virtual machine power state: %w", err) @@ -511,13 +514,6 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } - if config.VMAntiAffinity { - machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] - if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { - return false, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) - } - } - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { filemanager := datastore.NewFileManager(session.Datacenter, false) diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index e4dcf2e8b..4fea8e276 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -20,64 +20,75 @@ import ( "context" "errors" "fmt" - "reflect" "strings" - "time" + "sync" - "github.com/aws/smithy-go/ptr" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + + "k8s.io/utils/ptr" ) +var lock sync.Mutex + // createOrUpdateVMAntiAffinityRule creates or updates an anti affinity rule with the name in the given cluster. // VMs are attached to the rule based on their folder path and name prefix in vsphere. // A minimum of two VMs is required. -func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session *Session, name string, config *Config) error { - p.mutex.Lock() - defer p.mutex.Unlock() - +func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error { + lock.Lock() + defer lock.Unlock() cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster) if err != nil { return err } + machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/")) if err != nil { if errors.Is(err, &find.NotFoundError{}) { - return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) } return err } var ruleVMRef []types.ManagedObjectReference for _, vm := range vmsInFolder { - if strings.HasPrefix(vm.Name(), name) { + // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted + if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { ruleVMRef = append(ruleVMRef, vm.Reference()) } } - // minimum of two vms required - if len(ruleVMRef) < 2 { - return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + if len(ruleVMRef) == 0 { + log.Debugf("No VMs in folder %s with name prefix %s found", config.Folder, machineSetName) + return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) + } else if len(ruleVMRef) < 2 { + // DRS rule must have at least two virtual machine members + log.Debugf("Not enough VMs in folder %s to create anti-affinity rule", config.Folder) + return nil } - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, name) + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, machineSetName) if err != nil { return err } + log.Debugf("Creating or updating anti-affinity rule for VMs %v in cluster %s", ruleVMRef, config.Cluster) operation := types.ArrayUpdateOperationEdit //create new rule if info == nil { info = &types.ClusterAntiAffinityRuleSpec{ ClusterRuleInfo: types.ClusterRuleInfo{ - Enabled: ptr.Bool(true), - Mandatory: ptr.Bool(false), - Name: name, - UserCreated: ptr.Bool(true), + Enabled: ptr.To(true), + Mandatory: ptr.To(false), + Name: machineSetName, + UserCreated: ptr.To(true), }, } operation = types.ArrayUpdateOperationAdd @@ -95,49 +106,22 @@ func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session }, } + log.Debugf("Performing %q for anti-affinity rule for VMs %v in cluster %s", operation, ruleVMRef, config.Cluster) task, err := cluster.Reconfigure(ctx, spec, true) if err != nil { return err } - err = task.Wait(ctx) + taskResult, err := task.WaitForResult(ctx) if err != nil { - return err + return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) } - - return waitForRule(ctx, cluster, info) -} - -// waitForRule checks periodically the vsphere api for the ClusterAntiAffinityRule and returns error if the rule was not found after a timeout. -func waitForRule(ctx context.Context, cluster *object.ClusterComputeResource, rule *types.ClusterAntiAffinityRuleSpec) error { - timeout := time.NewTimer(10 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) - defer timeout.Stop() - defer ticker.Stop() - - for { - select { - case <-timeout.C: - - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) - if err != nil { - return err - } - - if !reflect.DeepEqual(rule, info) { - return fmt.Errorf("expected anti affinity changes not found in vsphere") - } - case <-ticker.C: - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) - if err != nil { - return err - } - - if reflect.DeepEqual(rule, info) { - return nil - } - } + if taskResult.State != types.TaskInfoStateSuccess { + return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) } + log.Debugf("Successfully created/updated anti-affinity rule for machineset %v against machine %v", machineSetName, machine.Name) + + return nil } // removeVMAntiAffinityRule removes an anti affinity rule with the name in the given cluster. @@ -172,7 +156,15 @@ func removeVMAntiAffinityRule(ctx context.Context, session *Session, clusterPath if err != nil { return err } - return task.Wait(ctx) + + taskResult, err := task.WaitForResult(ctx) + if err != nil { + return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) + } + if taskResult.State != types.TaskInfoStateSuccess { + return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) + } + return nil } func findClusterAntiAffinityRuleByName(ctx context.Context, cluster *object.ClusterComputeResource, name string) (*types.ClusterAntiAffinityRuleSpec, error) { diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index 06afef450..a298b487a 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -42,7 +42,7 @@ func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine if machineSetName != "" { machineSet := &clusterv1alpha1.MachineSet{} - if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { + if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: machine.Namespace}, machineSet); err != nil { return "", "", err } diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index f4b2f0455..6d20a7b22 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -102,40 +102,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -145,11 +145,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -158,15 +158,6 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, { name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml deleted file mode 100644 index d78822c7f..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ /dev/null @@ -1,454 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml index dc0b4c19a..4a158e5f4 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml index 38144310b..cd96ceb4c 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index cdf312425..f0c0ea4aa 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 913600e2b..b0717e211 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml index 6efd50a5e..028a63809 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index e6bd23c03..d93b87a2f 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -102,50 +102,50 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-nutanix", + name: "kubelet-v1.29.2-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -155,11 +155,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml deleted file mode 100644 index 598e6a672..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ /dev/null @@ -1,460 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml index 8037bbf2c..5ba93232e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml similarity index 100% rename from pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml index 240ae56bd..963494efe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index c30cc8c89..1061299b4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 2749fdc6f..93fe63ce9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml index 23b83aa16..3a8004604 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index f05ecb0e6..91d64a2d7 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -188,7 +188,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.28.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -204,7 +204,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.28.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -219,7 +219,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -235,7 +235,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -250,7 +250,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -266,7 +266,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -343,7 +343,38 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.26.6", + name: "cloud-init_v1.28.5", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "anexia", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: pointer.String("ssh-rsa AAABBB"), + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.28.5", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "anexia", + config: "{anexia-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: CloudInit, + }, + }, + { + name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -359,7 +390,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -374,7 +405,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.26.6", + name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -390,7 +421,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -417,7 +448,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml index a75f6a7b9..30c4bfa86 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml @@ -201,7 +201,7 @@ write_files: kubeReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 300Mi + memory: 200Mi logging: flushFrequency: 0 options: @@ -223,7 +223,7 @@ write_files: systemReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 500Mi + memory: 200Mi tlsCipherSuites: - TLS_AES_128_GCM_SHA256 - TLS_AES_256_GCM_SHA384 @@ -432,7 +432,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index a984766a3..ede5308e3 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -411,7 +411,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json index a55371087..b14200180 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json @@ -228,4 +228,4 @@ } ] } -} \ No newline at end of file +} diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json index 43276fd70..e66d02ce7 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json index 8c586d86c..1de594c31 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json new file mode 100644 index 000000000..87202d61f --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index db213784b..71c2857f2 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,9 +26,8 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } ) diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index 786e153ac..870469c37 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -42,9 +42,9 @@ func TestDownloadBinariesScript(t *testing.T) { } func TestSafeDownloadBinariesScript(t *testing.T) { - name := "safe_download_binaries_v1.26.6" + name := "safe_download_binaries_v1.29.2" t.Run(name, func(t *testing.T) { - script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.26.6") + script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.29.2") if err != nil { t.Error(err) } diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index 0b71d10d1..be2dc79b2 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -66,7 +66,7 @@ func TestKubeletSystemdUnit(t *testing.T) { tests = append(tests, []kubeletFlagTestCase{ { name: "multiple-dns-servers", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{ net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), @@ -76,14 +76,14 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "cloud-provider-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", }, { name: "pause-image-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", @@ -91,7 +91,7 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "taints-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden new file mode 100644 index 000000000..4ae9bd7b5 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden @@ -0,0 +1,21 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then +<<<<<<<< HEAD:pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.26.12/bin/linux/amd64/kubelet +======== + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.27.11/bin/linux/amd64/kubelet +>>>>>>>> upstream/release/v1.59:pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden similarity index 88% rename from pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden index fe50c1a8f..103682b02 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.26.12/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.28.7/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden new file mode 100644 index 000000000..214208182 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.29.2/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden similarity index 98% rename from pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden rename to pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden index 7a61bdb36..eb0c7e5bc 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden @@ -40,7 +40,7 @@ tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - -KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" +KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 263611a1a..e972a47e5 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -120,40 +120,40 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -163,11 +163,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -176,25 +176,6 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, - { - name: "kubelet-v1.26-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - externalCloudProvider: true, - }, { name: "kubelet-v1.28-nutanix", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml deleted file mode 100644 index c69138b97..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml +++ /dev/null @@ -1,532 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml deleted file mode 100644 index 4974e179a..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml +++ /dev/null @@ -1,532 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: -- path: "/opt/restart-kubelet.sh" - permissions: "0755" - content: | - #!/bin/bash - - while true; do - output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") - if [[ $? != 0 ]]; then - echo "Error not found in logs" - elif [[ $output ]]; then - echo "Restart kubelet" - systemctl restart kubelet - fi - - # Runs every 10 minutes - sleep 600 - done - -- path: "/etc/systemd/system/kubelet-restart.service" - permissions: "0644" - content: | - [Unit] - Description=Restarts kubelet on use of closed network connection error - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/kubelet-restart.sh - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml index 38fcae567..5f992d1e1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml @@ -174,7 +174,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml index eecad163b..d57d39d41 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml @@ -174,7 +174,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index 5627dad7b..d3f74c179 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -188,7 +188,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index b082116b2..9fe688cf1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -188,7 +188,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml index 50c896437..2221a9b2b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml @@ -180,7 +180,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index 3a9e7d596..f5c883cb1 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -111,49 +111,40 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "kubelet-v1.29-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", - }, - }, - }, - { - name: "kubelet-v1.26.6-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -163,11 +154,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -177,11 +168,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-nutanix", + name: "kubelet-v1.29.2-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("nutanix"), diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml index d38b03963..11d4239a6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml index 17b9ce869..10ed4d353 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -340,7 +340,7 @@ write_files: kubeReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 300Mi + memory: 200Mi logging: flushFrequency: 0 options: @@ -362,7 +362,7 @@ write_files: systemReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 500Mi + memory: 200Mi tlsCipherSuites: - TLS_AES_128_GCM_SHA256 - TLS_AES_256_GCM_SHA384 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml index f820413a6..9e35bb4f3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index 8356f482c..ae8c9d41c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index f948cbc12..88f42cd35 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml index 46cabdb91..65fe363ec 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index ae172d5a7..492c9fcd4 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,10 +129,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } var tests []userDataTestCase @@ -603,7 +602,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.12", + Kubelet: "1.29.0", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index efcfe55d1..10cb29335 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.1.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.11.yaml index daf0c4dcc..6175e4a64 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.1}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.11}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -397,6 +397,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml similarity index 98% rename from pkg/userdata/ubuntu/testdata/version-1.24.9.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.7.yaml index bb429ce7d..2b99ca5a9 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -229,7 +229,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -387,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -398,12 +398,13 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 300Mi + memory: 200Mi logging: flushFrequency: 0 options: @@ -425,7 +426,7 @@ write_files: systemReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 500Mi + memory: 200Mi tlsCipherSuites: - TLS_AES_128_GCM_SHA256 - TLS_AES_256_GCM_SHA384 diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.2.yaml similarity index 98% rename from pkg/userdata/ubuntu/testdata/version-1.25.5.yaml rename to pkg/userdata/ubuntu/testdata/version-1.29.2.yaml index 8a172811e..3831bb0e0 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.29.2.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -229,7 +229,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -387,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -398,12 +398,13 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 300Mi + memory: 200Mi logging: flushFrequency: 0 options: @@ -425,7 +426,7 @@ write_files: systemReserved: cpu: 200m ephemeral-storage: 1Gi - memory: 500Mi + memory: 200Mi tlsCipherSuites: - TLS_AES_128_GCM_SHA256 - TLS_AES_256_GCM_SHA384 diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c0f75ad41..4d205b01b 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -66,6 +66,7 @@ const ( LinodeManifest = "./testdata/machinedeployment-linode.yaml" VMwareCloudDirectorManifest = "./testdata/machinedeployment-vmware-cloud-director.yaml" VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" + VSPhereAntiAffinityManifest = "./testdata/machinedeployment-vsphere-anti-affinity.yaml" VSPhereMultipleNICManifest = "./testdata/machinedeployment-vsphere-multiple-nic.yaml" VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" VSPhereResourcePoolManifest = "./testdata/machinedeployment-vsphere-resource-pool.yaml" @@ -84,7 +85,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.28.5" + defaultKubernetesVersion = "1.28.7" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -344,7 +345,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.12", "1.27.9", "1.28.5", "1.29.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +425,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.9", "1.28.5", "1.29.0")) + selector := Not(VersionSelector("1.27.11", "1.28.7", "1.29.2")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +479,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +501,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -685,7 +686,7 @@ func TestGCEProvisioningE2E(t *testing.T) { } // Act. GCE does not support CentOS. - selector := OsSelector("ubuntu", "flatcar") + selector := And(OsSelector("ubuntu", "flatcar"), Not(VersionSelector("1.29.2"))) params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } @@ -855,6 +856,23 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) } +// TestVsphereAntiAffinityProvisioningE2E - is the same as the TestVsphereProvisioning suit but has anti-affinity rules applied to the VMs. +func TestVsphereAntiAffinityProvisioningE2E(t *testing.T) { + t.Parallel() + + params := getVSphereTestParams(t) + + scenario := scenario{ + name: "VSphere Anti-Affinity provisioning", + osName: "ubuntu", + containerRuntime: defaultContainerRuntime, + kubernetesVersion: defaultKubernetesVersion, + executor: verifyCreateAndDelete, + } + + testScenario(t, scenario, *testRunIdentifier, params, VSPhereAntiAffinityManifest, false) +} + // TestVsphereDatastoreClusterProvisioning - is the same as the TestVsphereProvisioning suite but specifies a DatastoreCluster // instead of the Datastore in the provider specs. func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 6783aaf80..aff05972f 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml new file mode 100644 index 000000000..8f74c8465 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml @@ -0,0 +1,54 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vsphere" + cloudProviderSpec: + templateVMName: '<< OS_Image_Template >>' + username: '<< VSPHERE_USERNAME >>' + vsphereURL: '<< VSPHERE_ADDRESS >>' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' + password: << VSPHERE_PASSWORD >> + # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically + cluster: Kubermatic + vmAntiAffinity: true + datastore: vsan + cpus: 2 + MemoryMB: 4096 + diskSizeGB: << DISK_SIZE >> + allowInsecure: true + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + attachSubscription: false + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>"