diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 91595d7b..c87487f3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,8 @@ ## General +* Added support for IPv6 addresses in the Kubernetes 'apiVIP' field + ## API ### Image Definition Changes diff --git a/docs/building-images.md b/docs/building-images.md index 23690ea1..71600f56 100644 --- a/docs/building-images.md +++ b/docs/building-images.md @@ -261,7 +261,7 @@ kubernetes: * `network` - Required for multi-node clusters, optional for single-node clusters; Defines the network configuration for bootstrapping a cluster. * `apiVIP` - Required for multi-node clusters, optional for single-node clusters; Specifies the IP address which - will serve as the cluster LoadBalancer, backed by MetalLB. + will serve as the cluster LoadBalancer, backed by MetalLB. Supports IPv4 and IPv6 addresses. * `apiHost` - Optional; Specifies the domain address for accessing the cluster. * `nodes` - Required for multi-node clusters; Defines a list of all nodes that form the cluster. * `hostname` - Required; Indicates the fully qualified domain name (FQDN) to identify the particular node on which diff --git a/pkg/combustion/kubernetes.go b/pkg/combustion/kubernetes.go index e8308506..da48f045 100644 --- a/pkg/combustion/kubernetes.go +++ b/pkg/combustion/kubernetes.go @@ -3,6 +3,7 @@ package combustion import ( _ "embed" "fmt" + "net/netip" "os" "path/filepath" "strings" @@ -316,11 +317,18 @@ func (c *Combustion) downloadRKE2Artefacts(ctx *image.Context, cluster *kubernet } func kubernetesVIPManifest(k *image.Kubernetes) (string, error) { + ip, err := netip.ParseAddr(k.Network.APIVIP) + if err != nil { + return "", fmt.Errorf("parsing kubernetes apiVIP address: %w", err) + } + manifest := struct { APIAddress string + IsIPV4 bool RKE2 bool }{ APIAddress: k.Network.APIVIP, + IsIPV4: ip.Is4(), RKE2: strings.Contains(k.Version, image.KubernetesDistroRKE2), } diff --git a/pkg/combustion/kubernetes_test.go b/pkg/combustion/kubernetes_test.go index 700d4f73..f0412642 100644 --- a/pkg/combustion/kubernetes_test.go +++ b/pkg/combustion/kubernetes_test.go @@ -821,3 +821,46 @@ func TestConfigureKubernetes_SuccessfulRKE2ServerWithManifests(t *testing.T) { assert.Contains(t, contents, "name: my-nginx") assert.Contains(t, contents, "image: nginx:1.14.2") } + +func TestKubernetesVIPManifestValidIPV4(t *testing.T) { + k8s := &image.Kubernetes{ + Version: "v1.30.3+rke2r1", + Network: image.Network{ + APIVIP: "192.168.1.1", + }, + } + + manifest, err := kubernetesVIPManifest(k8s) + require.NoError(t, err) + + assert.Contains(t, manifest, "- 192.168.1.1/32") + assert.Contains(t, manifest, "- name: rke2-api") +} + +func TestKubernetesVIPManifestValidIPV6(t *testing.T) { + k8s := &image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIVIP: "fd12:3456:789a::21", + }, + } + + manifest, err := kubernetesVIPManifest(k8s) + require.NoError(t, err) + + assert.Contains(t, manifest, "- fd12:3456:789a::21/128") + assert.Contains(t, manifest, "- name: k8s-api") + assert.NotContains(t, manifest, "rke2") +} + +func TestKubernetesVIPManifestInvalidIP(t *testing.T) { + k8s := &image.Kubernetes{ + Version: "v1.30.3+k3s1", + Network: image.Network{ + APIVIP: "1111", + }, + } + + _, err := kubernetesVIPManifest(k8s) + require.ErrorContains(t, err, "parsing kubernetes apiVIP address: ParseAddr(\"1111\"): unable to parse IP") +} diff --git a/pkg/combustion/templates/k8s-vip.yaml.tpl b/pkg/combustion/templates/k8s-vip.yaml.tpl index dc75546f..28c8484c 100644 --- a/pkg/combustion/templates/k8s-vip.yaml.tpl +++ b/pkg/combustion/templates/k8s-vip.yaml.tpl @@ -6,7 +6,11 @@ metadata: namespace: metallb-system spec: addresses: - - {{ .APIAddress }}/32 + {{- if .IsIPV4 }} + - {{ .APIAddress }}/32 + {{- else }} + - {{ .APIAddress }}/128 + {{- end }} avoidBuggyIPs: true serviceAllocation: namespaces: diff --git a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl index 91a29d97..0efec761 100644 --- a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl +++ b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl @@ -91,7 +91,7 @@ systemctl enable kubernetes-resources-install.service {{- end }} fi -{{- if .apiHost }} +{{- if and .apiVIP .apiHost }} echo "{{ .apiVIP }} {{ .apiHost }}" >> /etc/hosts {{- end }} diff --git a/pkg/image/validation/kubernetes.go b/pkg/image/validation/kubernetes.go index 647a6326..8a01b5b4 100644 --- a/pkg/image/validation/kubernetes.go +++ b/pkg/image/validation/kubernetes.go @@ -3,6 +3,7 @@ package validation import ( "errors" "fmt" + "net/netip" "net/url" "os" "path/filepath" @@ -32,6 +33,7 @@ func validateKubernetes(ctx *image.Context) []FailedValidation { return failures } + failures = append(failures, validateNetwork(&def.Kubernetes)...) failures = append(failures, validateNodes(&def.Kubernetes)...) failures = append(failures, validateManifestURLs(&def.Kubernetes)...) failures = append(failures, validateHelm(&def.Kubernetes, combustion.HelmValuesPath(ctx), combustion.HelmCertsPath(ctx))...) @@ -52,12 +54,6 @@ func validateNodes(k8s *image.Kubernetes) []FailedValidation { return failures } - if k8s.Network.APIVIP == "" { - failures = append(failures, FailedValidation{ - UserMessage: "The 'apiVIP' field is required in the 'network' section when defining entries under 'nodes'.", - }) - } - var nodeTypes []string var nodeNames []string var initialisers []*image.Node @@ -117,6 +113,47 @@ func validateNodes(k8s *image.Kubernetes) []FailedValidation { return failures } +func validateNetwork(k8s *image.Kubernetes) []FailedValidation { + var failures []FailedValidation + + if k8s.Network.APIVIP == "" { + if len(k8s.Nodes) > 1 { + failures = append(failures, FailedValidation{ + UserMessage: "The 'apiVIP' field is required in the 'network' section for multi node clusters.", + }) + } + + return failures + } + + parsedIP, err := netip.ParseAddr(k8s.Network.APIVIP) + if err != nil { + failures = append(failures, FailedValidation{ + UserMessage: fmt.Sprintf("Invalid address value %q for field 'apiVIP'.", k8s.Network.APIVIP), + Error: err, + }) + + return failures + } + + if !parsedIP.Is4() && !parsedIP.Is6() { + failures = append(failures, FailedValidation{ + UserMessage: "Only IPv4 and IPv6 addresses are valid values for field 'apiVIP'.", + }) + + return failures + } + + if !parsedIP.IsGlobalUnicast() { + msg := fmt.Sprintf("Invalid non-unicast cluster API address (%s) for field 'apiVIP'.", k8s.Network.APIVIP) + failures = append(failures, FailedValidation{ + UserMessage: msg, + }) + } + + return failures +} + func validateManifestURLs(k8s *image.Kubernetes) []FailedValidation { var failures []FailedValidation diff --git a/pkg/image/validation/kubernetes_test.go b/pkg/image/validation/kubernetes_test.go index 82406ddb..b2e0a691 100644 --- a/pkg/image/validation/kubernetes_test.go +++ b/pkg/image/validation/kubernetes_test.go @@ -15,7 +15,7 @@ import ( var validNetwork = image.Network{ APIHost: "host.com", - APIVIP: "127.0.0.1", + APIVIP: "192.168.1.1", } func TestValidateKubernetes(t *testing.T) { @@ -78,7 +78,10 @@ func TestValidateKubernetes(t *testing.T) { `failures all sections`: { K8s: image.Kubernetes{ Version: "v1.30.3", - Network: validNetwork, + Network: image.Network{ + APIHost: "host.com", + APIVIP: "127.0.0.1", + }, Nodes: []image.Node{ { Type: image.KubernetesNodeTypeServer, @@ -116,6 +119,7 @@ func TestValidateKubernetes(t *testing.T) { "Helm chart 'name' field must be defined.", "Helm repository 'name' field for \"apache-repo\" must match the 'repositoryName' field in at least one defined Helm chart.", "Helm chart 'repositoryName' \"another-apache-repo\" for Helm chart \"\" does not match the name of any defined repository.", + "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", }, }, } @@ -185,24 +189,6 @@ func TestValidateNodes(t *testing.T) { Nodes: []image.Node{}, }, }, - `with nodes - no network config`: { - K8s: image.Kubernetes{ - Network: image.Network{}, - Nodes: []image.Node{ - { - Hostname: "host1", - Type: image.KubernetesNodeTypeServer, - }, - { - Hostname: "host2", - Type: image.KubernetesNodeTypeAgent, - }, - }, - }, - ExpectedFailedMessages: []string{ - "The 'apiVIP' field is required in the 'network' section when defining entries under 'nodes'.", - }, - }, `no hostname`: { K8s: image.Kubernetes{ Network: validNetwork, @@ -1079,3 +1065,108 @@ func TestValidateAdditionalArtifacts(t *testing.T) { }) } } + +func TestValidateNetwork(t *testing.T) { + tests := map[string]struct { + K8s image.Kubernetes + ExpectedFailedMessages []string + }{ + `no network defined, no nodes defined`: { + K8s: image.Kubernetes{ + Network: image.Network{}, + }, + }, + `no network defined, nodes defined`: { + K8s: image.Kubernetes{ + Network: image.Network{}, + Nodes: []image.Node{ + { + Hostname: "node1", + Type: "server", + Initialiser: false, + }, + { + Hostname: "node2", + Type: "server", + Initialiser: false, + }, + }, + }, + ExpectedFailedMessages: []string{ + "The 'apiVIP' field is required in the 'network' section for multi node clusters.", + }, + }, + `valid ipv4`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "192.168.1.1", + }, + }, + }, + `valid ipv6`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "fd12:3456:789a::21", + }, + }, + }, + `invalid ipv4`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "500.168.1.1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid address value \"500.168.1.1\" for field 'apiVIP'.", + }, + }, + `non-unicast ipv4`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "127.0.0.1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (127.0.0.1) for field 'apiVIP'.", + }, + }, + `invalid ipv6`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "xxxx:3456:789a::21", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid address value \"xxxx:3456:789a::21\" for field 'apiVIP'.", + }, + }, + `non-unicast ipv6`: { + K8s: image.Kubernetes{ + Network: image.Network{ + APIVIP: "ff02::1", + }, + }, + ExpectedFailedMessages: []string{ + "Invalid non-unicast cluster API address (ff02::1) for field 'apiVIP'.", + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + k := test.K8s + failures := validateNetwork(&k) + assert.Len(t, failures, len(test.ExpectedFailedMessages)) + + var foundMessages []string + for _, foundValidation := range failures { + foundMessages = append(foundMessages, foundValidation.UserMessage) + } + + for _, expectedMessage := range test.ExpectedFailedMessages { + assert.Contains(t, foundMessages, expectedMessage) + } + + }) + } +} diff --git a/pkg/kubernetes/cluster.go b/pkg/kubernetes/cluster.go index 7c9cd909..2b5eaf57 100644 --- a/pkg/kubernetes/cluster.go +++ b/pkg/kubernetes/cluster.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/fs" + "net/netip" "os" "path/filepath" "strings" @@ -54,7 +55,12 @@ func NewCluster(kubernetes *image.Kubernetes, configPath string) (*Cluster, erro return &Cluster{ServerConfig: serverConfig}, nil } - setMultiNodeConfigDefaults(kubernetes, serverConfig) + ip, err := netip.ParseAddr(kubernetes.Network.APIVIP) + if err != nil { + return nil, fmt.Errorf("parsing kubernetes apiVIP address: %w", err) + } + + setMultiNodeConfigDefaults(kubernetes, serverConfig, &ip) agentConfigPath := filepath.Join(configPath, agentConfigFile) agentConfig, err := ParseKubernetesConfig(agentConfigPath) @@ -150,22 +156,23 @@ func setSingleNodeConfigDefaults(kubernetes *image.Kubernetes, config map[string delete(config, serverKey) } -func setMultiNodeConfigDefaults(kubernetes *image.Kubernetes, config map[string]any) { +func setMultiNodeConfigDefaults(kubernetes *image.Kubernetes, config map[string]any, ip *netip.Addr) { const ( k3sServerPort = 6443 rke2ServerPort = 9345 ) if strings.Contains(kubernetes.Version, image.KubernetesDistroRKE2) { - setClusterAPIAddress(config, kubernetes.Network.APIVIP, rke2ServerPort) + setClusterAPIAddress(config, ip, rke2ServerPort) setClusterCNI(config) } else { - setClusterAPIAddress(config, kubernetes.Network.APIVIP, k3sServerPort) + setClusterAPIAddress(config, ip, k3sServerPort) appendDisabledServices(config, "servicelb") } setClusterToken(config) appendClusterTLSSAN(config, kubernetes.Network.APIVIP) + setSELinux(config) if kubernetes.Network.APIHost != "" { appendClusterTLSSAN(config, kubernetes.Network.APIHost) @@ -196,13 +203,13 @@ func setClusterCNI(config map[string]any) { config[cniKey] = cniDefaultValue } -func setClusterAPIAddress(config map[string]any, apiAddress string, port int) { - if apiAddress == "" { +func setClusterAPIAddress(config map[string]any, ip *netip.Addr, port uint16) { + if ip == nil { zap.S().Warn("Attempted to set an empty cluster API address") return } - config[serverKey] = fmt.Sprintf("https://%s:%d", apiAddress, port) + config[serverKey] = fmt.Sprintf("https://%s", netip.AddrPortFrom(*ip, port).String()) } func setSELinux(config map[string]any) { diff --git a/pkg/kubernetes/cluster_test.go b/pkg/kubernetes/cluster_test.go index e1e5a950..debb2ef7 100644 --- a/pkg/kubernetes/cluster_test.go +++ b/pkg/kubernetes/cluster_test.go @@ -1,6 +1,7 @@ package kubernetes import ( + "net/netip" "testing" "github.com/google/uuid" @@ -276,11 +277,20 @@ func TestIdentifyInitialiserNode(t *testing.T) { func TestSetClusterAPIAddress(t *testing.T) { config := map[string]any{} - setClusterAPIAddress(config, "", 9345) + setClusterAPIAddress(config, nil, 9345) assert.NotContains(t, config, "server") - setClusterAPIAddress(config, "192.168.122.50", 9345) + ip4, err := netip.ParseAddr("192.168.122.50") + assert.NoError(t, err) + + setClusterAPIAddress(config, &ip4, 9345) assert.Equal(t, "https://192.168.122.50:9345", config["server"]) + + ip6, err := netip.ParseAddr("fd12:3456:789a::21") + assert.NoError(t, err) + + setClusterAPIAddress(config, &ip6, 9345) + assert.Equal(t, "https://[fd12:3456:789a::21]:9345", config["server"]) } func TestAppendClusterTLSSAN(t *testing.T) {