diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index feca2ff70..0021f1e92 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -15,7 +15,7 @@ jobs: name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.20.5 + go-version-file: go.mod cache: true - name: Run the tests diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index fe59ab740..7fd8eff78 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 68d162289..d94aa85a6 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index d3462ad51..1c7a1af7c 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 2c6896ceb..deb91e618 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 9adfb1063..e7a60d1b2 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index a055a5b3f..f06fa7dd0 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 393ef95fd..0a2ee8929 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 0e4888a7a..986bb7faf 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 592af7d08..5c85b2939 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 204cfba35..7205e1492 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index a6f1204ce..ad712a605 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index dc900ef89..def0cb7de 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index bf40ac5a2..4c05da09b 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index c32083ad7..8213afc44 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 42eb04cd7..c9a33e4fc 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index e0abf8433..bc09746d8 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 8af222d88..71ea8ab3e 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 261ba1c32..edee2d747 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.54.1 + - image: golangci/golangci-lint:v1.56.0 command: - make args: @@ -83,11 +83,11 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/yamllint:0.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-3 command: - - "sh" - - "-c" - - "yamllint -c .yamllint.conf ." + - make + args: + - yamllint resources: requests: memory: 32Mi @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: diff --git a/Dockerfile b/Dockerfile index eb4168adb..5651e7673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.21.5 +ARG GO_VERSION=1.22.0 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . RUN make all -FROM alpine:3.17 +FROM alpine:3.19 RUN apk add --no-cache ca-certificates cdrkit diff --git a/Makefile b/Makefile index 001e51135..0ebd201ce 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.21.5 +GO_VERSION ?= 1.22.0 GOOS ?= $(shell go env GOOS) @@ -67,6 +67,9 @@ clean: lint: golangci-lint run -v +yamllint: + yamllint -c .yamllint.conf . + .PHONY: docker-image docker-image: docker build --build-arg GO_VERSION=$(GO_VERSION) -t $(IMAGE_NAME) . diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 21494f318..dc87eeb2a 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -10,6 +10,3 @@ aliases: - moelsayed - xmudrii - xrstf - sig-virtualization: - - cnvergence - - wozniakjan diff --git a/README.md b/README.md index 4407629df..470eb79aa 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.29 - 1.28 - 1.27 - 1.26 diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 3eff631ab..0931c26f8 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -58,7 +58,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) var ( @@ -339,7 +339,7 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana namespace = defaultLeaderElectionNamespace } - extraHandlers := make(map[string]http.Handler) + metricsOptions := metricsserver.Options{BindAddress: metricsAddress} if profiling { m := http.NewServeMux() m.HandleFunc("/", pprof.Index) @@ -347,22 +347,21 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana m.HandleFunc("/profile", pprof.Profile) m.HandleFunc("/symbol", pprof.Symbol) m.HandleFunc("/trace", pprof.Trace) - - extraHandlers["/debug/pprof/"] = m + metricsOptions.ExtraHandlers = map[string]http.Handler{ + "/debug/pprof/": m, + } } mgr, err := manager.New(options.cfg, manager.Options{ Cache: cache.Options{ - SyncPeriod: &syncPeriod, + DefaultNamespaces: map[string]cache.Config{}, + SyncPeriod: &syncPeriod, }, LeaderElection: enableLeaderElection, LeaderElectionID: defaultLeaderElectionID, LeaderElectionNamespace: namespace, HealthProbeBindAddress: healthProbeAddress, - Metrics: server.Options{ - BindAddress: metricsAddress, - ExtraHandlers: extraHandlers, - }, + Metrics: metricsOptions, }) if err != nil { return nil, fmt.Errorf("failed to build ctrlruntime manager: %w", err) @@ -379,7 +378,6 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana if err := mgr.AddHealthzCheck("apiserver-connection", health.ApiserverReachable(options.kubeClient)); err != nil { return nil, fmt.Errorf("failed to add health check: %w", err) } - if err := mgr.Add(&controllerBootstrap{ mgr: mgr, opt: options, diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index d501bafaf..606e2664c 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/tools/clientcmd" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) type options struct { @@ -143,6 +144,13 @@ func main() { NodeFlags: nodeFlags, Namespace: opt.namespace, VersionConstraints: constraint, + + // we could change this to get the CertDir from the configured CertName + // and KeyName, but doing so does not bring us any benefits but would + // technically break compatibility. + CertDir: "/", + CertName: opt.admissionTLSCertPath, + KeyName: opt.admissionTLSKeyPath, }.Build() if err != nil { log.Fatalw("Failed to create admission hook", zap.Error(err)) @@ -150,12 +158,8 @@ func main() { log.Infow("Listening", "address", opt.admissionListenAddress) - if err := srv.ListenAndServeTLS(opt.admissionTLSCertPath, opt.admissionTLSKeyPath); err != nil { + serverContext := signals.SetupSignalHandler() + if err := srv.Start(serverContext); err != nil { log.Fatalw("Failed to start server", zap.Error(err)) } - defer func() { - if err := srv.Close(); err != nil { - log.Fatalw("Failed to shutdown server", zap.Error(err)) - } - }() } diff --git a/docs/operating-system.md b/docs/operating-system.md index b223f42b1..fd6994ece 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -10,7 +10,7 @@ | Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | | Digitalocean | ✓ | ✓ | x | x | x | ✓ | | Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | -| Google Cloud Platform | ✓ | x | x | x | x | x | +| Google Cloud Platform | ✓ | x | ✓ | x | x | x | | Hetzner | ✓ | x | x | x | x | ✓ | | KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | | Nutanix | ✓ | ✓ | x | x | x | x | diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index e25a10b2b..154b6196c 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index acafdaf75..7e9745903 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -54,4 +54,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 8f56c6892..b82c2b505 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 6612809f7..755a1e5a7 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index bdf1db222..804c8c3b4 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 20e4733b4..dbce45cce 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index dca6466f8..7c598af34 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 07a7324fe..1d3adb228 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index d510c9490..c96d47350 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index a08d60acf..f00928ac4 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 862bb15bb..bb7e863a6 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index 768135a99..ab4309da7 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index ccffe0096..ddfa3c030 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 4f6ef9943..9a0116400 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -4,8 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: operatingsystemconfigs.operatingsystemmanager.k8c.io spec: group: operatingsystemmanager.k8c.io @@ -14,367 +13,409 @@ spec: listKind: OperatingSystemConfigList plural: operatingsystemconfigs shortNames: - - osc + - osc singular: operatingsystemconfig scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemConfig is the object that represents the OperatingSystemConfig - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: OperatingSystemConfigSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemConfig is the object that represents the OperatingSystemConfig + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemConfigSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' - type: string - yum_repos: - additionalProperties: + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: additionalProperties: type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - description: YumRepos adds yum repository configuration to - the system. - type: object - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - cloudProvider: - description: CloudProvider represent the cloud provider that support - the given operating system version - properties: - name: - description: Name represents the name of the supported cloud provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - - opennebula - type: string - spec: - description: Spec represents the os/image reference in the supported - cloud provider - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - osName: - description: 'OSType represent the operating system name e.g: ubuntu' - enum: - - flatcar - - rhel - - centos - - ubuntu - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. properties: - data: - description: Data is the file's data. + content: + description: Content is the content of the drop-in. type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). + name: + description: Name is the name of the drop-in. type: string required: - - data + - content + - name type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: type: string - yum_repos: - additionalProperties: - additionalProperties: + type: array + type: object + cloudProvider: + description: + CloudProvider represent the cloud provider that support + the given operating system version + properties: + name: + description: Name represents the name of the supported cloud provider + enum: + - aws + - azure + - digitalocean + - edge + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + - opennebula + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. type: string - type: object - description: YumRepos adds yum repository configuration to - the system. + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - provisioningUtility: - default: cloud-init - description: ProvisioningUtility used for configuring the worker node. - Defaults to cloud-init. - enum: - - cloud-init - - ignition - type: string - required: - - bootstrapConfig - - cloudProvider - - osName - - osVersion - - provisioningConfig - type: object - required: - - spec - type: object - served: true - storage: true + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + provisioningUtility: + default: cloud-init + description: + ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string + required: + - bootstrapConfig + - cloudProvider + - osName + - osVersion + - provisioningConfig + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: operatingsystemprofiles.operatingsystemmanager.k8c.io spec: group: operatingsystemmanager.k8c.io @@ -383,504 +424,569 @@ spec: listKind: OperatingSystemProfileList plural: operatingsystemprofiles shortNames: - - osp + - osp singular: operatingsystemprofile scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemProfile is the object that represents the OperatingSystemProfile - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: OperatingSystemProfileSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org - type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' - type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: YumRepos adds yum repository configuration to - the system. - type: object - type: object - supportedContainerRuntimes: - description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: Files to add to the main files list when the - containerRuntime is selected - items: - description: File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemProfile is the object that represents the OperatingSystemProfile + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemProfileSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. properties: - content: - description: Content describe the file's content. + inline: + description: + Inline is a struct that contains information + about the inlined data. properties: - inline: - description: Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data type: object - path: - description: Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: + rh_subscription: additionalProperties: type: string - description: Templates to add to the available templates - when the containerRuntime is selected + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - type: object - osName: - description: 'OSType represent the operating system name e.g: ubuntu' - enum: - - flatcar - - rhel - - centos - - ubuntu - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS. Docker has been deprecated + and is no-op. + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer required: - - data + - content + - path type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' + type: array + templates: + additionalProperties: type: string - yum_repos: - additionalProperties: - additionalProperties: + description: Templates to be included in units and files + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. type: string - type: object - description: YumRepos adds yum repository configuration to - the system. + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name type: object - type: object - supportedContainerRuntimes: - description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: Files to add to the main files list when the - containerRuntime is selected - items: - description: File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + type: array + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. properties: - content: - description: Content describe the file's content. + inline: + description: + Inline is a struct that contains information + about the inlined data. properties: - inline: - description: Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data type: object - path: - description: Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: + rh_subscription: additionalProperties: type: string - description: Templates to add to the available templates - when the containerRuntime is selected + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - type: object - provisioningUtility: - default: cloud-init - description: ProvisioningUtility used for configuring the worker node. - Defaults to cloud-init. - enum: - - cloud-init - - ignition - type: string - supportedCloudProviders: - description: SupportedCloudProviders represent the cloud providers - that support the given operating system version - items: - description: CloudProviderSpec contains the os/image reference for - a specific supported cloud provider - properties: - name: - description: Name represents the name of the supported cloud - provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - - opennebula - type: string - spec: - description: Spec represents the os/image reference in the supported - cloud provider + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS. Docker has been deprecated + and is no-op. + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array type: object - type: array - version: - description: Version is the version of the operating System Profile - pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - type: string - required: - - bootstrapConfig - - osName - - osVersion - - provisioningConfig - - supportedCloudProviders - - version - type: object - required: - - spec - type: object - served: true - storage: true + provisioningUtility: + default: cloud-init + description: + ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string + supportedCloudProviders: + description: + SupportedCloudProviders represent the cloud providers + that support the given operating system version + items: + description: + CloudProviderSpec contains the os/image reference for + a specific supported cloud provider + properties: + name: + description: + Name represents the name of the supported cloud + provider + enum: + - aws + - azure + - digitalocean + - edge + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + - opennebula + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + version: + description: Version is the version of the operating System Profile + pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + type: string + required: + - bootstrapConfig + - osName + - osVersion + - provisioningConfig + - supportedCloudProviders + - version + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: cert-manager.io/v1 kind: Issuer @@ -995,8 +1101,8 @@ spec: name: webhook command: - /usr/local/bin/webhook - - -logtostderr - - -v=4 + - -log-debug=false + - -log-format=json - -namespace=kube-system volumeMounts: - name: operating-system-manager-admission-cert @@ -1323,8 +1429,8 @@ spec: name: operating-system-manager command: - /usr/local/bin/osm-controller - - -logtostderr - - -v=4 + - -log-debug=false + - -log-format=json - -worker-count=5 - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 9a14eb3cc..b996f3f0c 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index eb342dceb..40d076a8e 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 58d93c5ef..1bae0b162 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 368f1f16a..4e1bb6cef 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 89ecbb0c9..03e2b7f07 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -50,13 +50,13 @@ spec: plan: "vhf-8c-32gb" # This takes precedence over enableVPC vpcId: - - + - # For more reference, see # https://www.vultr.com/api/#tag/instances/operation/create-instance enableVPC: false enableVPC2: true vpc2Id: - - + - # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. # This takes precedence over operatingSystem osId: 215 @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/go.mod b/go.mod index d41523ab3..23feb6dbd 100644 --- a/go.mod +++ b/go.mod @@ -2,77 +2,79 @@ module github.com/kubermatic/machine-controller go 1.21 +toolchain go1.21.5 + require ( - cloud.google.com/go/logging v1.8.1 - cloud.google.com/go/monitoring v1.15.1 + cloud.google.com/go/logging v1.9.0 + cloud.google.com/go/monitoring v1.17.0 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 - github.com/aws/aws-sdk-go-v2 v1.20.1 - github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.32 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 - github.com/aws/smithy-go v1.14.1 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 + github.com/aws/aws-sdk-go-v2 v1.24.1 + github.com/aws/aws-sdk-go-v2/config v1.26.3 + github.com/aws/aws-sdk-go-v2/credentials v1.16.14 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 + github.com/aws/smithy-go v1.19.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.102.0 + github.com/digitalocean/godo v1.107.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.0.8 - github.com/google/uuid v1.3.0 - github.com/gophercloud/gophercloud v1.5.0 + github.com/google/uuid v1.5.0 + github.com/gophercloud/gophercloud v1.8.0 github.com/hashicorp/go-multierror v1.1.1 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go v1.39.0 - github.com/linode/linodego v1.20.1 + github.com/linode/linodego v1.26.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 - github.com/packethost/packngo v0.30.0 + github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.18.0 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.21.0 - github.com/vmware/govmomi v0.30.7 - github.com/vultr/govultr/v3 v3.3.1 - go.anx.io/go-anxcloud v0.5.3 + github.com/vmware/govmomi v0.34.2 + github.com/vultr/govultr/v3 v3.6.0 + go.anx.io/go-anxcloud v0.6.2 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.16.0 - golang.org/x/oauth2 v0.12.0 + golang.org/x/crypto v0.17.0 + golang.org/x/oauth2 v0.15.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.137.0 - google.golang.org/grpc v1.58.3 + google.golang.org/api v0.155.0 + google.golang.org/grpc v1.60.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.0 - k8s.io/apiextensions-apiserver v0.29.0 - k8s.io/apimachinery v0.29.0 - k8s.io/client-go v0.29.0 - k8s.io/cloud-provider v0.28.0 + k8s.io/api v0.29.1 + k8s.io/apiextensions-apiserver v0.29.1 + k8s.io/apimachinery v0.29.1 + k8s.io/client-go v0.29.1 + k8s.io/cloud-provider v0.29.1 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.28.0 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - kubevirt.io/api v1.0.0 - kubevirt.io/containerized-data-importer-api v1.57.0 + k8s.io/kubelet v0.29.1 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e + kubevirt.io/api v1.1.1 + kubevirt.io/containerized-data-importer-api v1.58.0 sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.110.7 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go v0.111.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.5.1 // indirect + cloud.google.com/go/longrunning v0.5.4 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect @@ -85,32 +87,35 @@ require ( github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect - github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.1 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-resty/resty/v2 v2.7.0 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect + github.com/go-openapi/swag v0.22.7 // indirect + github.com/go-resty/resty/v2 v2.11.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -121,12 +126,12 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/s2a-go v0.1.5 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -148,7 +153,7 @@ require ( github.com/onsi/ginkgo/v2 v2.14.0 // indirect github.com/onsi/gomega v1.30.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 // indirect + github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect @@ -156,40 +161,41 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect - go.opentelemetry.io/otel v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/net v0.19.0 // indirect - golang.org/x/sync v0.5.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.16.1 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.29.0 // indirect + k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 0aa5ac5ce..4b957bea7 100644 --- a/go.sum +++ b/go.sum @@ -18,28 +18,28 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= +cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= -cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= -cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= -cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= +cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= +cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= +cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -87,6 +87,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= @@ -96,8 +97,8 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 h1:hzczEtHROO5bxA9QWva112sguv3REVxKWq5oTY5biOE= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 h1:kWb9OISprBC94fTeagHWzz+TQOx5IrwQOY88JyEVNjc= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -112,16 +113,17 @@ github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVb github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 h1:1r0pVpVs5XrgibXeBW7SwD9kU9ceTEq+LHjPEwxkKU0= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.512/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 h1:UpBbuyd0eqDkIfiuRmBGqdjXWd4Q7YwD9entykxwlnI= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.654/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -132,32 +134,34 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= -github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= -github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 h1:8I4NQ9BfrQATHzXKtBuu+jBdOVd2mBANqhbMOXfSIdA= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0/go.mod h1:Ie0Kp61cLk223argiS+t8vO29SpbFIphzlPflIvYcv0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 h1:OJELEgyaT2kmaBGZ+myyZbTTLobfe3ox3FSh5eYK9Qs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= -github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= -github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= +github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 h1:tTAfm9YsKlmlv6ORgco838e0ZeAcGVRkgevseiYO0gU= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1/go.mod h1:hIsHE0PaWAQakLCshKS7VKWMGXaqrAFp4m95s2W9E6c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -196,7 +200,7 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pq github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= @@ -206,21 +210,23 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.102.0 h1:iXkLEHmrBi5n9TAbRcKM0182NwXYrp1xCj//7vMRFs4= -github.com/digitalocean/godo v1.102.0/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg= +github.com/digitalocean/godo v1.107.0 h1:P72IbmGFQvKOvyjVLyT59bmHxilA4E5hWi40rF4zNQc= +github.com/digitalocean/godo v1.107.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.1 h1:S+9bSbua1z3FgCnV0KKOSSZ3mDthb5NyEPL5gEpCvyk= +github.com/emicklei/go-restful/v3 v3.11.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -232,18 +238,21 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -283,20 +292,18 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= +github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -307,8 +314,8 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -326,6 +333,7 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -406,23 +414,24 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= -github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.8.0 h1:TM3Jawprb2NrdOnvcHhWJalmKmAmOGgfZElM/3oBYCk= +github.com/gophercloud/gophercloud v1.8.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -440,8 +449,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -484,6 +493,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= @@ -521,8 +531,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.20.1 h1:IW3SrZjRzrclYZnzFd80f8lSkTYAM7gVTJW0t7HnFKQ= -github.com/linode/linodego v1.20.1/go.mod h1:ggoWnJXssx9wPWNnR3x7WaOpOBOEhsPB/HO7iflF5qY= +github.com/linode/linodego v1.26.0 h1:2tOZ3Wxn4YvGBRgZi3Vz6dab+L16XUntJ9sJxh3ZBio= +github.com/linode/linodego v1.26.0/go.mod h1:kD7Bf1piWg/AXb9TA0ThAVwzR+GPf6r2PvbTbVk7PMA= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -593,15 +603,15 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 h1:PY2JBJdSkzTxathfsYMa/Mb2Xcw9YphTZ6IcvURayKk= -github.com/openshift/api v0.0.0-20230815201604-a2362cf53230/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= +github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= +github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/packethost/packngo v0.30.0 h1:JVeTwbXXETsLTDQncUbYwIFpkOp/xevXrffM2HrFECI= -github.com/packethost/packngo v0.30.0/go.mod h1:BT/XcdwLVmeMtGPbovnxCpnI1s9ylSE1cs/7pq007NE= +github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= +github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFpECX6WF+yUWS2Bd1Nc1W+AczzqbY= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -652,8 +662,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -661,8 +671,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 h1:a9hSJdJcd16e0HoMsnFvaHvxB3pxSD+SC7+CISp7xY0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 h1:wJrcTdddKOI8TFxs8cemnhKP2EmKy3yfUKHj3ZdfzYo= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= @@ -681,10 +691,10 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -727,12 +737,12 @@ github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8A github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.21.0 h1:zIONrJpM+Fj+rDyXmsRfMAn1sP5WAP87USL0T9GS4DY= github.com/vmware/go-vcloud-director/v2 v2.21.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.30.7 h1:YO8CcDpLJzmq6PK5/CBQbXyV21iCMh8SbdXt+xNkXp8= -github.com/vmware/govmomi v0.30.7/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg= +github.com/vmware/govmomi v0.34.2 h1:o6ydkTVITOkpQU6HAf6tP5GvHFCNJlNUNlMsvFK77X4= +github.com/vmware/govmomi v0.34.2/go.mod h1:qWWT6n9mdCr/T9vySsoUqcI04sSEj4CqHXxtk/Y+Los= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/vultr/govultr/v3 v3.3.1 h1:gn46pSL0A3pxBKZklzwtYgSW2Iq7dW1euDRBlOsIzTo= -github.com/vultr/govultr/v3 v3.3.1/go.mod h1:7NjuHeQv5vgUWR2H1sPc9D+xffrT5ql+kNi6R3yuwzo= +github.com/vultr/govultr/v3 v3.6.0 h1:WCXQwgdiZnGxG4CI+TTohE14V3jV6ikg/64fhDVdbIs= +github.com/vultr/govultr/v3 v3.6.0/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= @@ -745,8 +755,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.5.3 h1:ZKdHC2QaVvN6VdzT1O/aBxkabE/OoNJNLNpOyRzLqcw= -go.anx.io/go-anxcloud v0.5.3/go.mod h1:XTIg137l88FfE42X0/aOgvSm/fcXxWVp8k4+x9//zaQ= +go.anx.io/go-anxcloud v0.6.2 h1:4FV9xtjilRny/TEBeqsqOPxm1i9UPlPjHRDK86fhFjc= +go.anx.io/go-anxcloud v0.6.2/go.mod h1:TW0KcKa1hlYEwCQ2YAFec07xtfX60psI/dmjJqRdmjY= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -761,16 +771,20 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -811,15 +825,18 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -829,8 +846,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -858,6 +876,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -904,7 +923,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -913,6 +931,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -928,8 +948,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -942,8 +962,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1017,6 +1038,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1024,8 +1047,10 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1038,6 +1063,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1045,11 +1072,14 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1108,6 +1138,7 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1116,6 +1147,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1137,16 +1172,17 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.137.0 h1:QrKX6uNvzJLr0Fd3vWVqcyrcmFoYi036VUAsZbiF4+s= -google.golang.org/api v0.137.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= +google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= +google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1191,12 +1227,12 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 h1:Iveh6tGCJkHAjJgEqUQYGDGgbwmhjoAOz8kO/ajxefY= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 h1:WGq4lvB/mlicysM/dUT3SBvijH4D3sm/Ny1A4wmt2CI= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= +google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1219,9 +1255,8 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1237,8 +1272,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1272,6 +1307,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1283,23 +1319,23 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= -k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= +k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= +k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= +k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= -k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= +k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= -k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/cloud-provider v0.28.0 h1:BTIW7b757T+VXB5yqJeajPXsNOmeooopUgfzQueiWvk= -k8s.io/cloud-provider v0.28.0/go.mod h1:u0MGqdlutkTmCJyNrCzIMJ+OhrwQE9x5X8mBTN0R7us= +k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= +k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= +k8s.io/cloud-provider v0.29.1 h1:bDLpOSpysWrtU2PCkvyP2sUTwRBa6MGCmxt68CRRW/8= +k8s.io/cloud-provider v0.29.1/go.mod h1:u50Drm6AbuoKpsVbAstNiFHGgbSVHuJV4TWN5imdM2w= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= -k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= +k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1315,23 +1351,24 @@ k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U= -k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= +k8s.io/kubelet v0.29.1 h1:cso8Dk8dymkj8q+EvW/aCbIYU2aOkH27gho48tYza/8= +k8s.io/kubelet v0.29.1/go.mod h1:hTl/naFcCVG1Ku17fMgj/krbheBwBkf3gnFhaboMx7E= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.0.0 h1:RBdXP5CDhE0v5qL2OUQdrYyRrHe/F68Z91GWqBDF6nw= -kubevirt.io/api v1.0.0/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= -kubevirt.io/containerized-data-importer-api v1.57.0 h1:IpRCUyDS0x7BaVa5q5MCzuWRAfvXT54GpEnNJke5hSE= -kubevirt.io/containerized-data-importer-api v1.57.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.1.1 h1:vt5bOpACArNFIudx1bcE1VeejQdh5wCd7Oz/uFBIkH8= +kubevirt.io/api v1.1.1/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= +kubevirt.io/containerized-data-importer-api v1.58.0 h1:l6bH2SrCUi14QAi1Mv1vzcrqZI0XYzrV1KLK6hiC0QI= +kubevirt.io/containerized-data-importer-api v1.58.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 0afcaa361..8657d5ab6 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.21.5 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.0 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index cb13ab003..3646eec8f 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/image-builder/download_kubernetes.sh b/image-builder/download_kubernetes.sh index 14fc74e7a..0b5cdd042 100755 --- a/image-builder/download_kubernetes.sh +++ b/image-builder/download_kubernetes.sh @@ -46,7 +46,7 @@ else echo " * Using version $K8S_RELEASE" fi -wget --quiet https://storage.googleapis.com/kubernetes-release/release/$K8S_RELEASE/bin/linux/amd64/{kubeadm,kubelet,kubectl}.sha1 -P "$TEMPDIR" +wget --quiet https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/{kubeadm,kubelet,kubectl}.sha1 -P "$TEMPDIR" for util in kubeadm kubelet kubectl; do echo " * $util" @@ -59,7 +59,7 @@ for util in kubeadm kubelet kubectl; do exit 1 fi else - wget "https://storage.googleapis.com/kubernetes-release/release/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" + wget "https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" CALCULATED_SHA1="$(sha1sum "$TEMPDIR/$util" | cut -f1 -d ' ')" EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index bf563bb6e..b10d76871 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -22,9 +22,10 @@ import ( "errors" "fmt" "io" + "net" "net/http" "reflect" - "time" + "strconv" "github.com/Masterminds/semver/v3" "go.uber.org/zap" @@ -39,6 +40,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) type admissionData struct { @@ -64,10 +67,13 @@ type Builder struct { NodeFlags *node.Flags Namespace string VersionConstraints *semver.Constraints + + CertDir string + CertName string + KeyName string } -func (build Builder) Build() (*http.Server, error) { - mux := http.NewServeMux() +func (build Builder) Build() (webhook.Server, error) { ad := &admissionData{ log: build.Log, client: build.Client, @@ -82,18 +88,43 @@ func (build Builder) Build() (*http.Server, error) { return nil, fmt.Errorf("error updating nodeSettings, %w", err) } - mux.HandleFunc("/machinedeployments", handleFuncFactory(build.Log, ad.mutateMachineDeployments)) - mux.HandleFunc("/machines", handleFuncFactory(build.Log, ad.mutateMachines)) - mux.HandleFunc("/healthz", healthZHandler) + options := webhook.Options{ + CertDir: build.CertDir, + CertName: build.CertName, + KeyName: build.KeyName, + } - return &http.Server{ - Addr: build.ListenAddress, - Handler: http.TimeoutHandler(mux, 25*time.Second, "timeout"), - }, nil -} + if build.ListenAddress != "" { + host, port, err := net.SplitHostPort(build.ListenAddress) + if err != nil { + return nil, fmt.Errorf("error parsing ListenAddress: %w", err) + } + + options.Host = host + + if port != "" { + port, err := strconv.ParseInt(port, 10, 16) + if err != nil { + return nil, fmt.Errorf("error parsing port from ListenAddress: %w", err) + } + + options.Port = int(port) + } + } + + server := webhook.NewServer(options) + + server.Register("/machinedeployments", handleFuncFactory(build.Log, ad.mutateMachineDeployments)) + server.Register("/machines", handleFuncFactory(build.Log, ad.mutateMachines)) + + checkers := healthz.Handler{ + Checks: map[string]healthz.Checker{ + "ping": healthz.Ping, + }, + } + server.Register("/healthz/", http.StripPrefix("/healthz/", &checkers)) -func healthZHandler(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) + return server, nil } func newJSONPatch(original, current runtime.Object) ([]jsonpatch.JsonPatchOperation, error) { @@ -136,7 +167,7 @@ func createAdmissionResponse(log *zap.SugaredLogger, original, mutated runtime.O type mutator func(context.Context, admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) -func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) func(http.ResponseWriter, *http.Request) { +func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { review, err := readReview(r) if err != nil { diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 28161700c..4cd794157 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -26,6 +26,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/edge" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/fake" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce" @@ -85,6 +86,9 @@ var ( providerconfigtypes.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) }, + providerconfigtypes.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return edge.New(cvr) + }, providerconfigtypes.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return kubevirt.New(cvr) }, diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 4897254e4..23b59da7f 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -233,7 +233,7 @@ func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient. status := reconcileContext.Status // only use IP if it is still unbound - if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { + if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound && (!status.IPProvisioningExpires.IsZero() && status.IPProvisioningExpires.After(time.Now())) { log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) return status.ReservedIP, nil } @@ -259,6 +259,7 @@ func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient. ip := res.Data[0].Address status.ReservedIP = ip status.IPState = anxtypes.IPStateUnbound + status.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) return ip, nil } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 9bf2fc6ef..f89d470d8 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -303,6 +303,7 @@ func TestAnexiaProvider(t *testing.T) { expectedIP := "8.8.8.8" providerStatus.ReservedIP = expectedIP providerStatus.IPState = anxtypes.IPStateUnbound + providerStatus.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) reservedIP, err := getIPAddress(ctx, log, client) testhelper.AssertNoErr(t, err) testhelper.AssertEquals(t, expectedIP, reservedIP) diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 26dfb752f..bc692439f 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -34,8 +34,9 @@ const ( GetRequestTimeout = 1 * time.Minute DeleteRequestTimeout = 1 * time.Minute - IPStateBound = "Bound" - IPStateUnbound = "Unbound" + IPStateBound = "Bound" + IPStateUnbound = "Unbound" + IPProvisioningExpires = 1800 * time.Second VmxNet3NIC = "vmxnet3" MachinePoweredOn = "poweredOn" @@ -72,12 +73,13 @@ type RawConfig struct { } type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - DeprovisioningID string `json:"deprovisioningID"` - ReservedIP string `json:"reservedIP"` - IPState string `json:"ipState"` - Conditions []v1.Condition `json:"conditions,omitempty"` + InstanceID string `json:"instanceID"` + ProvisioningID string `json:"provisioningID"` + DeprovisioningID string `json:"deprovisioningID"` + ReservedIP string `json:"reservedIP"` + IPState string `json:"ipState"` + IPProvisioningExpires time.Time `json:"ipProvisioningExpires"` + Conditions []v1.Condition `json:"conditions,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go new file mode 100644 index 000000000..e29af0931 --- /dev/null +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -0,0 +1,104 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edge + +import ( + "context" + + "go.uber.org/zap" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type provider struct{} + +type CloudProviderSpec struct{} + +type CloudProviderInstance struct{} + +func (f CloudProviderInstance) Name() string { + return "" +} + +func (f CloudProviderInstance) ID() string { + return "" +} + +func (f CloudProviderInstance) ProviderID() string { + return "" +} + +func (f CloudProviderInstance) HostID() string { + return "" +} + +func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { + return nil +} + +func (f CloudProviderInstance) Status() instance.Status { + return instance.StatusUnknown +} + +// New returns a edge cloud provider. +func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{} +} + +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +// Validate returns success or failure based according to its EdgeCloudProviderSpec. +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, _ clusterv1alpha1.MachineSpec) error { + return nil +} + +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { + return "", "", nil +} + +// Create creates a cloud instance according to the given machine. +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + return true, nil +} + +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { + return nil +} + +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { + return map[string]string{}, nil +} + +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { + return nil +} diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 4a6e777d6..2beb82f2a 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -45,12 +45,14 @@ const ( // imageProjects maps the OS to the Google Cloud image projects. var imageProjects = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", + providerconfigtypes.OperatingSystemFlatcar: "kinvolk-public", } // imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", + providerconfigtypes.OperatingSystemFlatcar: "flatcar-stable", } // diskTypes are the disk types of the Google Cloud. Map is used for diff --git a/pkg/cloudprovider/provider/kubevirt/OWNERS b/pkg/cloudprovider/provider/kubevirt/OWNERS deleted file mode 100644 index ac28b34e2..000000000 --- a/pkg/cloudprovider/provider/kubevirt/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md - -approvers: - - sig-virtualization - -reviewers: - - sig-virtualization - -labels: - - sig/virtualization - -options: - no_parent_owners: true diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index e9a3add29..27da10688 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -99,6 +99,7 @@ type Config struct { Namespace string OSImageSource *cdiv1beta1.DataVolumeSource StorageClassName string + StorageAccessType corev1.PersistentVolumeAccessMode PVCSize resource.Quantity Instancetype *kubevirtv1.InstancetypeMatcher Preference *kubevirtv1.PreferenceMatcher @@ -144,9 +145,10 @@ type NodeAffinityPreset struct { } type SecondaryDisks struct { - Name string - Size resource.Quantity - StorageClassName string + Name string + Size resource.Quantity + StorageClassName string + StorageAccessType corev1.PersistentVolumeAccessMode } type kubeVirtServer struct { @@ -261,14 +263,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } - pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) + pvcSize, err := p.configVarResolver.GetConfigVarStringValue(&rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) } if config.PVCSize, err = resource.ParseQuantity(pvcSize); err != nil { return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %w`, err) } - config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) + config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(&rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } @@ -292,7 +294,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) for i, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { - sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) + sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(&sd.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) } @@ -301,16 +303,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) } - scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) + scString, err := p.configVarResolver.GetConfigVarStringValue(&sd.StorageClassName) if err != nil { return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ - Name: fmt.Sprintf("secondarydisk%d", i), - Size: pvc, - StorageClassName: scString, + Name: fmt.Sprintf("secondarydisk%d", i), + Size: pvc, + StorageClassName: scString, + StorageAccessType: p.getStorageAccessType(sd.StorageAccessType), }) } + config.StorageAccessType = p.getStorageAccessType(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageAccessType) config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { @@ -324,6 +328,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &config, pconfig, nil } +func (p *provider) getStorageAccessType(accessType providerconfigtypes.ConfigVarString) corev1.PersistentVolumeAccessMode { + at, _ := p.configVarResolver.GetConfigVarStringValue(&accessType) + if at == "" { + return corev1.ReadWriteOnce + } + return corev1.PersistentVolumeAccessMode(at) +} + func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { nodeAffinity := NodeAffinityPreset{} var err error @@ -848,7 +860,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ - "ReadWriteOnce", + config.StorageAccessType, }, Resources: corev1.VolumeResourceRequirements{ Requests: pvcRequest, @@ -867,7 +879,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ - "ReadWriteOnce", + config.StorageAccessType, }, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 6f47f45de..6b3434b2a 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -90,8 +90,9 @@ type SecondaryDisks struct { // Disk. type Disk struct { - Size *providerconfigtypes.ConfigVarString `json:"size,omitempty"` - StorageClassName *providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` + Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` + StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` + StorageAccessType providerconfigtypes.ConfigVarString `json:"storageAccessType,omitempty"` } // Affinity. diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 9680c4d80..522e77ff9 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -282,9 +282,8 @@ func resolveDatastoreRef(ctx context.Context, log *zap.SugaredLogger, config *Co return nil, fmt.Errorf("failed to get datastore: %w", err) } return types.NewReference(datastore.Reference()), nil - } else { - return nil, fmt.Errorf("please provide either a datastore or a datastore cluster") } + return nil, fmt.Errorf("please provide either a datastore or a datastore cluster") } func uploadAndAttachISO(ctx context.Context, log *zap.SugaredLogger, session *Session, vmRef *object.VirtualMachine, localIsoFilePath string) error { diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 7b9d3469a..2b5250658 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -96,7 +96,7 @@ cat <"$kube_sum_file" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml index f9e64df34..dc0b4c19a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml index 8417a3866..d78822c7f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 7bd2ea203..cdf312425 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -338,6 +338,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 662e14cfe..913600e2b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -338,6 +338,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml index 1fa12dc24..6efd50a5e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml @@ -150,7 +150,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -329,6 +329,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml index 6542d0c01..4441180c2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml index 9f7c667c4..ba8347f97 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml @@ -143,9 +143,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..651915b94 --- /dev/null +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,454 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 57a94030c..e6bd23c03 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -173,7 +173,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.5", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml index 1659cb5bb..0ca4f01cc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml @@ -1,458 +1,448 @@ #cloud-config - ssh_pwauth: false write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 300Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 500Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target runcmd: -- systemctl enable --now setup.service + - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml index 30823b9b4..8037bbf2c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml index 840ff9aae..598e6a672 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml index a727dbc38..240ae56bd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -335,6 +335,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index c621045ad..c30cc8c89 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -164,7 +164,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -344,6 +344,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 030c26904..2749fdc6f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -164,7 +164,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -344,6 +344,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml index fa34bf9bf..23b83aa16 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml @@ -156,7 +156,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -335,6 +335,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 3e038586a..8a5bb7a7b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index 36b302f1d..d57407477 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -149,9 +149,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..29b7cb2bb --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,460 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index df40a62c0..f05ecb0e6 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -23,10 +23,11 @@ package flatcar import ( "encoding/json" "flag" - "k8s.io/utils/pointer" "net" "testing" + "k8s.io/utils/pointer" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -125,7 +126,7 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "ignition_v1.28.0", + name: "ignition_v1.29.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -141,7 +142,38 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.29.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "vsphere", + config: "{vsphere-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: Ignition, + }, + }, + { + name: "ignition_v1.28.5", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "vsphere", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: pointer.String("ssh-rsa AAABBB"), + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.28.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -249,7 +281,38 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.28.0", + name: "cloud-init_v1.29.0", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "anexia", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: pointer.String("ssh-rsa AAABBB"), + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "anexia", + config: "{anexia-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: CloudInit, + }, + }, + { + name: "cloud-init_v1.28.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -265,7 +328,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml index 74721d941..a75f6a7b9 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml @@ -195,6 +195,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: @@ -433,7 +434,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -463,7 +464,7 @@ write_files: Restart=always Environment=CONTAINERD_CONFIG=/etc/containerd/config.toml ExecStart= - ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} \${TORCX_BINDIR}/containerd --config \${CONTAINERD_CONFIG} + ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} containerd --config \${CONTAINERD_CONFIG} EOF systemctl daemon-reload diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml index 4cba5caf4..027f2b4a8 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml @@ -1,520 +1,509 @@ #cloud-config users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - + - name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD coreos: units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start content: | [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target + [Install] + WantedBy=multi-user.target - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: resolv.conf + - name: nodeip.service + enable: true + command: start content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - name: 40-download.conf + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | [Unit] - Requires=download-script.service - After=download-script.service - - name: 50-rpc-statd.conf + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start content: | [Unit] - Wants=rpc-statd.service + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: /etc/sysctl.d/k8s.conf + permissions: "0644" content: | - [Unit] - After=containerd.service - Requires=containerd.service + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ + - path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + - path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: /etc/hostname + permissions: "0600" + content: "node1" + + - path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + + - path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" @@ -442,7 +443,7 @@ write_files: Restart=always Environment=CONTAINERD_CONFIG=/etc/containerd/config.toml ExecStart= - ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} \${TORCX_BINDIR}/containerd --config \${CONTAINERD_CONFIG} + ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} containerd --config \${CONTAINERD_CONFIG} EOF systemctl daemon-reload diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json index ff16bc8ac..647b7a3f9 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json index 496c3f005..b14200180 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json @@ -1 +1,231 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{ + "ignition": { + "config": {}, + "security": { "tls": {} }, + "timeouts": {}, + "version": "2.3.0" + }, + "networkd": { + "units": [ + { + "contents": "[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n", + "name": "static-nic.network" + } + ] + }, + "passwd": { + "users": [ + { + "name": "core", + "sshAuthorizedKeys": ["ssh-rsa AAABBB", "ssh-rsa CCCDDD"] + } + ] + }, + "storage": { + "files": [ + { + "filesystem": "root", + "path": "/etc/systemd/journald.conf.d/max_disk_use.conf", + "contents": { + "source": "data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/kubelet.conf", + "contents": { + "source": "data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/opt/load-kernel-modules.sh", + "contents": { + "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A", + "verification": {} + }, + "mode": 493 + }, + { + "filesystem": "root", + "path": "/etc/sysctl.d/k8s.conf", + "contents": { + "source": "data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/proc/sys/kernel/panic_on_oops", + "contents": { "source": "data:,1%0A", "verification": {} }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/proc/sys/kernel/panic", + "contents": { "source": "data:,10%0A", "verification": {} }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/proc/sys/vm/overcommit_memory", + "contents": { "source": "data:,1%0A", "verification": {} }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/opt/bin/setup_net_env.sh", + "contents": { + "source": "data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A", + "verification": {} + }, + "mode": 493 + }, + { + "filesystem": "root", + "path": "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf", + "contents": { + "source": "data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A", + "verification": {} + }, + "mode": 493 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/bootstrap-kubelet.conf", + "contents": { + "source": "data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A", + "verification": {} + }, + "mode": 256 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/cloud-config", + "contents": { + "source": "data:,%7Bvsphere-config%3Atrue%7D%0A", + "verification": {} + }, + "mode": 256 + }, + { + "filesystem": "root", + "path": "/etc/kubernetes/pki/ca.crt", + "contents": { + "source": "data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/hostname", + "contents": { "source": "data:,node1", "verification": {} }, + "mode": 384 + }, + { + "filesystem": "root", + "group": { "id": 0 }, + "path": "/etc/ssh/sshd_config", + "user": { "id": 0 }, + "contents": { + "source": "data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A", + "verification": {} + }, + "mode": 384 + }, + { + "filesystem": "root", + "path": "/opt/bin/setup.sh", + "contents": { + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A", + "verification": {} + }, + "mode": 493 + }, + { + "filesystem": "root", + "path": "/opt/bin/download.sh", + "contents": { + "source": "data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A", + "verification": {} + }, + "mode": 493 + }, + { + "filesystem": "root", + "path": "/etc/containerd/config.toml", + "contents": { + "source": "data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A", + "verification": {} + }, + "mode": 420 + }, + { + "filesystem": "root", + "path": "/etc/crictl.yaml", + "contents": { + "source": "data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A", + "verification": {} + }, + "mode": 420 + } + ] + }, + "systemd": { + "units": [ + { "mask": true, "name": "update-engine.service" }, + { "mask": true, "name": "locksmithd.service" }, + { + "contents": "[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n", + "enabled": true, + "name": "setup.service" + }, + { + "contents": "[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "download-script.service" + }, + { + "contents": "[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Unit]\nRequires=download-script.service\nAfter=download-script.service\n", + "name": "40-download.conf" + } + ], + "enabled": true, + "name": "kubelet-healthcheck.service" + }, + { + "contents": "[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n", + "enabled": true, + "name": "nodeip.service" + }, + { + "contents": "[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n", + "dropins": [ + { + "contents": "[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n", + "name": "10-nodeip.conf" + }, + { + "contents": "[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n", + "name": "resolv.conf" + }, + { + "contents": "[Unit]\nRequires=download-script.service\nAfter=download-script.service\n", + "name": "40-download.conf" + } + ], + "enabled": true, + "name": "kubelet.service" + } + ] + } +} diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json new file mode 100644 index 000000000..43276fd70 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.5.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json new file mode 100644 index 000000000..8c586d86c --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.0.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20300Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20500Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0A%0A%23%20sys11%3A%20we%20use%20the%20user-ssh-keys-agent%20to%20deploy%20ssh%20keys%0Asystemctl%20stop%20coreos-metadata-sshkeys%40core.service%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/disable-download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Asystemctl%20stop%20download-script.service%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"mask":true,"name":"coreos-metadata-sshkeys@core.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nAfter=download-script.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/disable-download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"disable-download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nAfter=disable-download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 4f4bb710c..db213784b 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,8 +26,9 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.26.9"), - semver.MustParse("v1.27.6"), - semver.MustParse("v1.28.2"), + semver.MustParse("v1.26.12"), + semver.MustParse("v1.27.9"), + semver.MustParse("v1.28.5"), + semver.MustParse("v1.29.0"), } ) diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 8495c450b..6a2ec7a86 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -98,7 +98,7 @@ cd - {{- /* kubelet */}} KUBE_VERSION="${KUBE_VERSION:-{{ .KubeVersion }}}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" -kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" +kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" {{- /* create versioned kube dir */}} @@ -146,7 +146,7 @@ fi {{- if .DownloadKubelet }} {{- /* kubelet */}} if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi {{- end }} diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index ec71332b1..786e153ac 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -25,6 +25,8 @@ import ( "github.com/kubermatic/machine-controller/pkg/test" ) +const goldenExtension = ".golden" + func TestDownloadBinariesScript(t *testing.T) { for _, version := range versions { name := fmt.Sprintf("download_binaries_%s", version.Original()) @@ -33,7 +35,7 @@ func TestDownloadBinariesScript(t *testing.T) { if err != nil { t.Error(err) } - goldenName := name + ".golden" + goldenName := name + goldenExtension test.CompareOutput(t, goldenName, script, *update) }) } @@ -46,7 +48,7 @@ func TestSafeDownloadBinariesScript(t *testing.T) { if err != nil { t.Error(err) } - goldenName := name + ".golden" + goldenName := name + goldenExtension test.CompareOutput(t, goldenName, script, *update) }) } diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden similarity index 84% rename from pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden index 2982cbae4..fe50c1a8f 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.9/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.26.12/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.9.golden similarity index 84% rename from pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.27.9.golden index 87010dae6..973ac2eee 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.9.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.27.9/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.5.golden similarity index 84% rename from pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.28.5.golden index 4add0f3ee..2d481358a 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.5.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.28.2/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.28.5/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden new file mode 100644 index 000000000..fe412fdad --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.29.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.9-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.9-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.9.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.9.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.5-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.5-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.5.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.5.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden index fe3d167a5..7a61bdb36 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden @@ -42,7 +42,7 @@ ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipp cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" -kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" +kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index f4e8b338c..263611a1a 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -106,7 +106,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", }, }, }, @@ -191,7 +200,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", }, }, cloudProviderName: stringPtr("nutanix"), @@ -201,11 +210,21 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.29.0", }, }, cloudProviderName: stringPtr("azure"), }, + { + name: "kubelet-v1.29-nutanix", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", + }, + }, + cloudProviderName: stringPtr("nutanix"), + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml index 46132fba2..c69138b97 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml @@ -176,7 +176,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -365,6 +365,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml index 3ca8ad4d8..eecad163b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml @@ -176,7 +176,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -365,6 +365,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml index 448ff7b92..38fcae567 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml @@ -176,7 +176,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -365,6 +365,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml index 106a975db..4974e179a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml @@ -176,7 +176,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -365,6 +365,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 401f7eded..5627dad7b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -190,7 +190,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -383,6 +383,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 295949774..b082116b2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -190,7 +190,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -383,6 +383,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml index 47bd995b4..50c896437 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml @@ -182,7 +182,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -374,6 +374,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml index eac98c6a5..c3cc9851c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml @@ -174,9 +174,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -365,6 +365,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml index 06f9ed3d3..b9fc1d821 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml @@ -182,9 +182,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -374,6 +374,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..b2b01ef42 --- /dev/null +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,532 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: +- path: "/opt/restart-kubelet.sh" + permissions: "0755" + content: | + #!/bin/bash + + while true; do + output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") + if [[ $? != 0 ]]; then + echo "Error not found in logs" + elif [[ $output ]]; then + echo "Restart kubelet" + systemctl restart kubelet + fi + + # Runs every 10 minutes + sleep 600 + done + +- path: "/etc/systemd/system/kubelet-restart.service" + permissions: "0644" + content: | + [Unit] + Description=Restarts kubelet on use of closed network connection error + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/kubelet-restart.sh + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml new file mode 100644 index 000000000..224e419cf --- /dev/null +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml @@ -0,0 +1,541 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 +fqdn: node1 + + +ssh_pwauth: false + +write_files: +- path: "/opt/restart-kubelet.sh" + permissions: "0755" + content: | + #!/bin/bash + + while true; do + output=$(journalctl -u kubelet -n 1 | grep "use of closed network connection") + if [[ $? != 0 ]]; then + echo "Error not found in logs" + elif [[ $output ]]; then + echo "Restart kubelet" + systemctl restart kubelet + fi + + # Runs every 10 minutes + sleep 600 + done + +- path: "/etc/systemd/system/kubelet-restart.service" + permissions: "0644" + content: | + [Unit] + Description=Restarts kubelet on use of closed network connection error + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/kubelet-restart.sh + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + hostnamectl set-hostname node1 + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index e6dc92042..cc0db60c7 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -179,9 +179,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -371,6 +371,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index b306379b4..3a9e7d596 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -106,7 +106,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", }, }, }, diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml index bb06443a0..d38b03963 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml @@ -146,7 +146,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml index 6a4c1f0cf..17b9ce869 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml @@ -146,7 +146,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml index ca5970ceb..f820413a6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml @@ -153,7 +153,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -342,6 +342,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index f9eabaa5f..8356f482c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -351,6 +351,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index f294472ae..f948cbc12 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -351,6 +351,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml index 9736280f6..46cabdb91 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -342,6 +342,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml index 388786a13..837fd1b23 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml @@ -144,9 +144,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..696a7c258 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,467 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index e0a976144..ae172d5a7 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,9 +129,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.26.9"), - semver.MustParse("v1.27.6"), - semver.MustParse("v1.28.2"), + semver.MustParse("v1.26.12"), + semver.MustParse("v1.27.9"), + semver.MustParse("v1.28.5"), + semver.MustParse("v1.29.0"), } var tests []userDataTestCase @@ -602,7 +603,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.26.12", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 7d4b020c1..b5c8fdd7d 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -410,6 +410,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index cee82d4c0..8ced434e6 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 16dc0099c..8fbbf463a 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 280d95296..845bcb1aa 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 7481c4706..f9522c18d 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -405,6 +405,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index d06f099cb..13e6bc3d0 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 5d6deadcb..9e2ca0e6f 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 1182a362c..fba22fa8b 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index d97f1b46d..efcfe55d1 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -153,9 +153,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -409,6 +409,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index ea009fa7f..23934963f 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 988ad8188..4e4c0077a 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 047f69066..f8038cb0d 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -406,6 +406,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index d3903b550..3447226fc 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -406,6 +406,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.9.yaml rename to pkg/userdata/ubuntu/testdata/version-1.26.12.yaml index adc27a1e1..3c2cbf051 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.6.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.9.yaml index d9f0f467d..e49e0fc66 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.28.2.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.5.yaml index 61cb3a74a..83c720e94 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml @@ -150,9 +150,9 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml new file mode 100644 index 000000000..0db63c62d --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml @@ -0,0 +1,459 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 300Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 500Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index a57a73092..e64e6f726 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -417,6 +417,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index f6c6faeb3..cf2ca8b6f 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -424,6 +424,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 0e6d7f089..880c25728 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -153,7 +153,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" @@ -407,6 +407,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 995b9fe89..c0f75ad41 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -84,8 +84,8 @@ const ( ) const ( - defaultKubernetesVersion = "1.27.6" - awsDefaultKubernetesVersion = "1.26.9" + defaultKubernetesVersion = "1.28.5" + awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -344,7 +344,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.9", "1.27.6", "1.28.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.12", "1.27.9", "1.28.5", "1.29.0"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.6", "1.28.2")) + selector := Not(VersionSelector("1.27.9", "1.28.5", "1.29.0")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -685,7 +685,7 @@ func TestGCEProvisioningE2E(t *testing.T) { } // Act. GCE does not support CentOS. - selector := OsSelector("ubuntu") + selector := OsSelector("ubuntu", "flatcar") params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 5ba7817e4..6783aaf80 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.26.9"), - semver.MustParse("v1.27.6"), - semver.MustParse("v1.28.2"), + semver.MustParse("v1.26.12"), + semver.MustParse("v1.27.9"), + semver.MustParse("v1.28.5"), + semver.MustParse("v1.29.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ @@ -238,7 +239,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.02")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.023")) } if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index db1768715..d92a6a034 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.26.6 + kubelet: 1.28.5