From 987ec89cb152e05772e4549aaa4f061d540324d3 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Mon, 22 Jan 2024 16:57:44 +0800 Subject: [PATCH] chore: custom host port range (#6508) (#6509) --- cmd/manager/main.go | 2 + deploy/helm/templates/deployment.yaml | 4 + deploy/helm/values.yaml | 20 +++- pkg/constant/const.go | 4 + pkg/controllerutil/controller_common.go | 140 +++++++++++++++++++----- 5 files changed, 139 insertions(+), 31 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 7e0c1925c32..a2fbe1d9b36 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -118,6 +118,8 @@ func init() { viper.SetDefault("CONFIG_MANAGER_LOG_LEVEL", "info") viper.SetDefault(constant.CfgKeyCtrlrMgrNS, "default") viper.SetDefault(constant.CfgHostPortConfigMapName, "kubeblocks-host-ports") + viper.SetDefault(constant.CfgHostPortIncludeRanges, "1025-65536") + viper.SetDefault(constant.CfgHostPortExcludeRanges, "6443,10250,10257,10259,2379-2380,30000-32767") viper.SetDefault(constant.FeatureGateReplicatedStateMachine, true) viper.SetDefault(constant.KBDataScriptClientsImage, "apecloud/kubeblocks-datascript:latest") viper.SetDefault(constant.KubernetesClusterDomainEnv, constant.DefaultDNSDomain) diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index 812d0d98616..19e7f1f318a 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -122,6 +122,10 @@ spec: key: dataProtectionEncryptionKey - name: KUBE_PROVIDER value: {{ .Values.provider | quote }} + - name: HOST_PORT_INCLUDE_RANGES + value: '{{ join "," .Values.hostPorts.include }}' + - name: HOST_PORT_EXCLUDE_RANGES + value: '{{ join "," .Values.hostPorts.exclude }}' - name: HOST_PORT_CM_NAME value: {{ include "kubeblocks.fullname" . }}-host-ports {{- with .Values.securityContext }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index dd66558c121..bfc24ce8c77 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1820,4 +1820,22 @@ external-dns: value: "true" effect: NoSchedule -developMode: false \ No newline at end of file +developMode: false + +# the final host ports is the difference between include and exclude: include - exclude +hostPorts: + # https://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html + # The TCP/IP port numbers below 1024 are special in that normal users are not allowed to run servers on them. + # This is a security feaure, in that if you connect to a service on one of these ports you can be fairly sure + # that you have the real thing, and not a fake which some hacker has put up for you. + include: + - "1025-65536" + # https://kubernetes.io/docs/reference/networking/ports-and-protocols/ + # exclude ports used by kubernetes + exclude: + - "6443" + - "10250" + - "10257" + - "10259" + - "2379-2380" + - "30000-32767" diff --git a/pkg/constant/const.go b/pkg/constant/const.go index eaefb48e0b0..28fa10e2b27 100644 --- a/pkg/constant/const.go +++ b/pkg/constant/const.go @@ -30,6 +30,8 @@ const ( CfgRecoverVolumeExpansionFailure = "RECOVER_VOLUME_EXPANSION_FAILURE" // refer to feature gates RecoverVolumeExpansionFailure of k8s. CfgKeyProvider = "KUBE_PROVIDER" CfgHostPortConfigMapName = "HOST_PORT_CM_NAME" + CfgHostPortIncludeRanges = "HOST_PORT_INCLUDE_RANGES" + CfgHostPortExcludeRanges = "HOST_PORT_EXCLUDE_RANGES" // addon config keys CfgKeyAddonJobTTL = "ADDON_JOB_TTL" @@ -152,6 +154,8 @@ const ( ExtraEnvAnnotationKey = "kubeblocks.io/extra-env" LastRoleSnapshotVersionAnnotationKey = "apps.kubeblocks.io/last-role-snapshot-version" HostPortAnnotationKey = "kubeblocks.io/host-port" + HostPortIncludeAnnotationKey = "network.kubeblocks.io/host-ports-include" + HostPortExcludeAnnotationKey = "network.kubeblocks.io/host-ports-exclude" // NodePortSvcAnnotationKey defines the feature gate of NodePort Service defined in ComponentDefinition.Spec.Services. // Components defined in the annotation value, their all services of type NodePort defined in ComponentDefinition will be created; otherwise, they will be ignored. diff --git a/pkg/controllerutil/controller_common.go b/pkg/controllerutil/controller_common.go index dc706cad89b..ffc44daed83 100644 --- a/pkg/controllerutil/controller_common.go +++ b/pkg/controllerutil/controller_common.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -286,15 +285,6 @@ var ( portManager *PortManager ) -const ( - // https://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html - // The TCP/IP port numbers below 1024 are special in that normal users are not allowed to run servers on them. - // This is a security feaure, in that if you connect to a service on one of these ports you can be fairly sure - // that you have the real thing, and not a fake which some hacker has put up for you. - hostPortMin = int32(1025) - hostPortMax = int32(65536) -) - func InitHostPortManager(cli client.Client) error { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -303,13 +293,70 @@ func InitHostPortManager(cli client.Client) error { }, Data: make(map[string]string), } + parsePortRange := func(item string) (int64, int64, error) { + parts := strings.Split(item, "-") + var ( + from int64 + to int64 + err error + ) + switch len(parts) { + case 2: + from, err = strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return from, to, err + } + to, err = strconv.ParseInt(parts[1], 10, 32) + if err != nil { + return from, to, err + } + if from > to { + return from, to, fmt.Errorf("invalid port range %s", item) + } + case 1: + from, err = strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return from, to, err + } + to = from + default: + return from, to, fmt.Errorf("invalid port range %s", item) + } + return from, to, nil + } + parsePortRanges := func(portRanges string) ([]PortRange, error) { + var ranges []PortRange + for _, item := range strings.Split(portRanges, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + from, to, err := parsePortRange(item) + if err != nil { + return nil, err + } + ranges = append(ranges, PortRange{ + Min: int32(from), + Max: int32(to), + }) + } + return ranges, nil + } var err error if err = cli.Create(context.Background(), cm); err != nil { if !apierrors.IsAlreadyExists(err) { return err } } - portManager, err = NewPortManager(hostPortMin, hostPortMax, cli) + includes, err := parsePortRanges(viper.GetString(constant.CfgHostPortIncludeRanges)) + if err != nil { + return err + } + excludes, err := parsePortRanges(viper.GetString(constant.CfgHostPortExcludeRanges)) + if err != nil { + return err + } + portManager, err = NewPortManager(includes, excludes, cli) return err } @@ -323,22 +370,45 @@ func BuildHostPortName(clusterName, compName, containerName, portName string) st type PortManager struct { sync.Mutex - cli client.Client - min int32 - max int32 - used map[int32]string - cm *corev1.ConfigMap + cli client.Client + from int32 + to int32 + cursor int32 + includes []PortRange + excludes []PortRange + used map[int32]string + cm *corev1.ConfigMap +} + +type PortRange struct { + Min int32 + Max int32 } // NewPortManager creates a new PortManager // TODO[ziang] Putting all the port information in one configmap may have performance issues and is not secure enough. // There is a risk of accidental deletion leading to the loss of cluster port information. -func NewPortManager(min, max int32, cli client.Client) (*PortManager, error) { +func NewPortManager(includes []PortRange, excludes []PortRange, cli client.Client) (*PortManager, error) { + var ( + from int32 + to int32 + ) + for _, item := range includes { + if item.Min < from || from == 0 { + from = item.Min + } + if item.Max > to { + to = item.Max + } + } pm := &PortManager{ - min: min, - max: max, - cli: cli, - used: make(map[int32]string), + cli: cli, + from: from, + to: to, + cursor: from, + includes: includes, + excludes: excludes, + used: make(map[int32]string), } if err := pm.sync(); err != nil { return nil, err @@ -378,6 +448,11 @@ func (pm *PortManager) sync() error { } used[port] = key } + for _, item := range pm.excludes { + for port := item.Min; port <= item.Max; port++ { + used[port] = "" + } + } pm.cm = cm pm.used = used @@ -467,18 +542,23 @@ func (pm *PortManager) AllocatePort(key string) (int32, error) { return port, nil } - // allocate a new port randomly in range [hostPortMin, hostPortMax) - for i := 0; i < 10; i++ { - port := int32(rand.Int63nRange(int64(hostPortMin), int64(hostPortMax))) - if _, ok := pm.used[port]; ok { - continue + if len(pm.used) >= int(pm.to-pm.from)+1 { + return 0, fmt.Errorf("no available port") + } + + for { + if _, ok := pm.used[pm.cursor]; !ok { + break } - if err := pm.update(key, port); err != nil { - return 0, err + pm.cursor++ + if pm.cursor > pm.to { + pm.cursor = pm.from } - return port, nil } - return 0, fmt.Errorf("failed to allocate port") + if err := pm.update(key, pm.cursor); err != nil { + return 0, err + } + return pm.cursor, nil } func (pm *PortManager) ReleasePort(key string) error {