Merge pull request #1962 from pixiake/master

Added new network plugin Hybridnet
kubesphere · Aug 24, 2023 · c7aec0f · c7aec0f
2 parents 4878e8d + 12a9128
commit c7aec0f
Show file tree

Hide file tree

Showing 10 changed files with 382 additions and 16 deletions.
diff --git a/cmd/kk/apis/kubekey/v1alpha2/default.go b/cmd/kk/apis/kubekey/v1alpha2/default.go
@@ -51,6 +51,7 @@ const (
 	DefaultFlannelCniPluginVersion = "v1.1.2"
 	DefaultCniVersion              = "v1.2.0"
 	DefaultCiliumVersion           = "v1.11.7"
+	DefaulthybridnetVersion        = "v0.8.6"
 	DefaultKubeovnVersion          = "v1.10.6"
 	DefalutMultusVersion           = "v3.8"
 	DefaultHelmVersion             = "v3.9.0"

diff --git a/cmd/kk/apis/kubekey/v1alpha2/network_types.go b/cmd/kk/apis/kubekey/v1alpha2/network_types.go
@@ -17,13 +17,14 @@
 package v1alpha2
 
 type NetworkConfig struct {
-	Plugin          string     `yaml:"plugin" json:"plugin,omitempty"`
-	KubePodsCIDR    string     `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"`
-	KubeServiceCIDR string     `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"`
-	Calico          CalicoCfg  `yaml:"calico" json:"calico,omitempty"`
-	Flannel         FlannelCfg `yaml:"flannel" json:"flannel,omitempty"`
-	Kubeovn         KubeovnCfg `yaml:"kubeovn" json:"kubeovn,omitempty"`
-	MultusCNI       MultusCNI  `yaml:"multusCNI" json:"multusCNI,omitempty"`
+	Plugin          string       `yaml:"plugin" json:"plugin,omitempty"`
+	KubePodsCIDR    string       `yaml:"kubePodsCIDR" json:"kubePodsCIDR,omitempty"`
+	KubeServiceCIDR string       `yaml:"kubeServiceCIDR" json:"kubeServiceCIDR,omitempty"`
+	Calico          CalicoCfg    `yaml:"calico" json:"calico,omitempty"`
+	Flannel         FlannelCfg   `yaml:"flannel" json:"flannel,omitempty"`
+	Kubeovn         KubeovnCfg   `yaml:"kubeovn" json:"kubeovn,omitempty"`
+	MultusCNI       MultusCNI    `yaml:"multusCNI" json:"multusCNI,omitempty"`
+	Hybridnet       HybridnetCfg `yaml:"hybridnet" json:"hybridnet,omitempty"`
 }
 
 type CalicoCfg struct {
@@ -90,6 +91,36 @@ type KubeOvnPinger struct {
 	PingerExternalDomain  string `yaml:"pingerExternalDomain" json:"pingerExternalDomain,omitempty"`
 }
 
+type HybridnetCfg struct {
+	DefaultNetworkType    string             `yaml:"defaultNetworkType" json:"defaultNetworkType,omitempty"`
+	EnableNetworkPolicy   *bool              `yaml:"enableNetworkPolicy" json:"enableNetworkPolicy,omitempty"`
+	Init                  *bool              `yaml:"init" json:"init,omitempty"`
+	PreferVxlanInterfaces string             `yaml:"preferVxlanInterfaces" json:"preferVxlanInterfaces,omitempty"`
+	PreferVlanInterfaces  string             `yaml:"preferVlanInterfaces" json:"preferVlanInterfaces,omitempty"`
+	PreferBGPInterfaces   string             `yaml:"preferBGPInterfaces" json:"preferBGPInterfaces,omitempty"`
+	Networks              []HybridnetNetwork `yaml:"networks" json:"networks,omitempty"`
+}
+
+type HybridnetNetwork struct {
+	Name         string            `yaml:"name" json:"name,omitempty"`
+	NetID        *int              `yaml:"netID" json:"netID,omitempty"`
+	Type         string            `yaml:"type" json:"type,omitempty"`
+	Mode         string            `yaml:"mode" json:"mode,omitempty"`
+	NodeSelector map[string]string `yaml:"nodeSelector" json:"nodeSelector,omitempty"`
+	Subnets      []HybridnetSubnet `yaml:"subnets" json:"subnets,omitempty"`
+}
+
+type HybridnetSubnet struct {
+	Name        string   `yaml:"name" json:"name,omitempty"`
+	NetID       *int     `yaml:"netID" json:"netID,omitempty"`
+	CIDR        string   `yaml:"cidr" json:"cidr,omitempty"`
+	Gateway     string   `yaml:"gateway" json:"gateway,omitempty"`
+	Start       string   `yaml:"start" json:"start,omitempty"`
+	End         string   `yaml:"end" json:"end,omitempty"`
+	ReservedIPs []string `yaml:"reservedIPs" json:"reservedIPs,omitempty"`
+	ExcludeIPs  []string `yaml:"excludeIPs" json:"excludeIPs,omitempty"`
+}
+
 func (k *KubeovnCfg) KubeovnCheckGateway() bool {
 	if k.KubeOvnController.CheckGateway == nil {
 		return true
@@ -151,3 +182,19 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
 	}
 	return *c.DefaultIPPOOL
 }
+
+// EnableInit is used to determine whether to create default network
+func (h *HybridnetCfg) EnableInit() bool {
+	if h.Init == nil {
+		return true
+	}
+	return *h.Init
+}
+
+// NetworkPolicy is used to determine whether to enable network policy
+func (h *HybridnetCfg) NetworkPolicy() bool {
+	if h.EnableNetworkPolicy == nil {
+		return true
+	}
+	return *h.EnableNetworkPolicy
+}
diff --git a/cmd/kk/pkg/common/common.go b/cmd/kk/pkg/common/common.go
@@ -55,10 +55,11 @@ const (
 	IPv4Regexp = "[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+"
 	IPv6Regexp = "[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}"
 
-	Calico  = "calico"
-	Flannel = "flannel"
-	Cilium  = "cilium"
-	Kubeovn = "kubeovn"
+	Calico    = "calico"
+	Flannel   = "flannel"
+	Cilium    = "cilium"
+	Kubeovn   = "kubeovn"
+	Hybridnet = "hybridnet"
 
 	Docker     = "docker"
 	Crictl     = "crictl"

diff --git a/cmd/kk/pkg/images/images.go b/cmd/kk/pkg/images/images.go
@@ -54,6 +54,35 @@ func (image Image) ImageName() string {
 	return fmt.Sprintf("%s:%s", image.ImageRepo(), image.Tag)
 }
 
+// ImageNamespace is used to get image's namespace
+func (image Image) ImageNamespace() string {
+	if os.Getenv("KKZONE") == "cn" {
+		if image.RepoAddr == "" || image.RepoAddr == cnRegistry {
+			image.NamespaceOverride = cnNamespaceOverride
+		}
+	}
+
+	if image.NamespaceOverride != "" {
+		return image.NamespaceOverride
+	} else {
+		return image.Namespace
+	}
+}
+
+// ImageRegistryAddr is used to get image's registry address.
+func (image Image) ImageRegistryAddr() string {
+	if os.Getenv("KKZONE") == "cn" {
+		if image.RepoAddr == "" || image.RepoAddr == cnRegistry {
+			image.RepoAddr = cnRegistry
+		}
+	}
+	if image.RepoAddr != "" {
+		return image.RepoAddr
+	} else {
+		return "docker.io"
+	}
+}
+
 // ImageRepo is used to generate image's repo address.
 func (image Image) ImageRepo() string {
 	var prefix string

diff --git a/cmd/kk/pkg/images/tasks.go b/cmd/kk/pkg/images/tasks.go
@@ -126,6 +126,7 @@ func GetImage(runtime connector.ModuleRuntime, kubeConf *common.KubeConf, name s
 		"flannel-cni-plugin":      {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "flannel", Repo: "flannel-cni-plugin", Tag: kubekeyv1alpha2.DefaultFlannelCniPluginVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "flannel")},
 		"cilium":                  {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "cilium", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")},
 		"cilium-operator-generic": {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "cilium", Repo: "operator-generic", Tag: kubekeyv1alpha2.DefaultCiliumVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "cilium")},
+		"hybridnet":               {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "hybridnetdev", Repo: "hybridnet", Tag: kubekeyv1alpha2.DefaulthybridnetVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "hybridnet")},
 		"kubeovn":                 {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: "kubeovn", Repo: "kube-ovn", Tag: kubekeyv1alpha2.DefaultKubeovnVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.EqualFold(kubeConf.Cluster.Network.Plugin, "kubeovn")},
 		"multus":                  {RepoAddr: kubeConf.Cluster.Registry.PrivateRegistry, Namespace: kubekeyv1alpha2.DefaultKubeImageNamespace, Repo: "multus-cni", Tag: kubekeyv1alpha2.DefalutMultusVersion, Group: kubekeyv1alpha2.K8s, Enable: strings.Contains(kubeConf.Cluster.Network.Plugin, "multus")},
 		// storage

diff --git a/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz b/cmd/kk/pkg/plugins/network/hybridnet-0.6.6.tgz
diff --git a/cmd/kk/pkg/plugins/network/modules.go b/cmd/kk/pkg/plugins/network/modules.go
@@ -48,6 +48,8 @@ func (d *DeployNetworkPluginModule) Init() {
 		d.Tasks = deployCilium(d)
 	case common.Kubeovn:
 		d.Tasks = deployKubeOVN(d)
+	case common.Hybridnet:
+		d.Tasks = deployHybridnet(d)
 	default:
 		return
 	}
@@ -239,8 +241,8 @@ func deployCilium(d *DeployNetworkPluginModule) []task.Interface {
 	}
 
 	syncCiliumChart := &task.RemoteTask{
-		Name:     "SyncKubeBinary",
-		Desc:     "Synchronize kubernetes binaries",
+		Name:     "SyncCiliumChart",
+		Desc:     "Synchronize cilium chart",
 		Hosts:    d.Runtime.GetHostsByRole(common.Master),
 		Prepare:  new(common.OnlyFirstMaster),
 		Action:   new(SyncCiliumChart),
@@ -335,6 +337,41 @@ func deployKubeOVN(d *DeployNetworkPluginModule) []task.Interface {
 	}
 }
 
+func deployHybridnet(d *DeployNetworkPluginModule) []task.Interface {
+
+	releaseHybridnetChart := &task.LocalTask{
+		Name:   "GenerateHybridnetChart",
+		Desc:   "Generate hybridnet chart",
+		Action: new(ReleaseHybridnetChart),
+	}
+
+	syncHybridnetChart := &task.RemoteTask{
+		Name:     "SyncHybridnetChart",
+		Desc:     "Synchronize hybridnet chart",
+		Hosts:    d.Runtime.GetHostsByRole(common.Master),
+		Prepare:  new(common.OnlyFirstMaster),
+		Action:   new(SyncHybridnetChart),
+		Parallel: true,
+		Retry:    2,
+	}
+
+	deploy := &task.RemoteTask{
+		Name:     "DeployHybridnet",
+		Desc:     "Deploy hybridnet",
+		Hosts:    d.Runtime.GetHostsByRole(common.Master),
+		Prepare:  new(common.OnlyFirstMaster),
+		Action:   new(DeployHybridnet),
+		Parallel: true,
+		Retry:    5,
+	}
+
+	return []task.Interface{
+		releaseHybridnetChart,
+		syncHybridnetChart,
+		deploy,
+	}
+}
+
 func K8sVersionAtLeast(version string, compare string) bool {
 	cmp, err := versionutil.MustParseSemantic(version).Compare(compare)
 	if err != nil {

diff --git a/cmd/kk/pkg/plugins/network/tasks.go b/cmd/kk/pkg/plugins/network/tasks.go
@@ -19,11 +19,12 @@ package network
 import (
 	"embed"
 	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
-
-	"github.com/pkg/errors"
+	"strings"
+	"time"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
@@ -34,7 +35,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/plugins/network/templates"
 )
 
-//go:embed cilium-1.11.7.tgz
+//go:embed cilium-1.11.7.tgz hybridnet-0.6.6.tgz
 
 var f embed.FS
 
@@ -306,3 +307,117 @@ func (c *ChmodKubectlKo) Execute(runtime connector.Runtime) error {
 	}
 	return nil
 }
+
+// ReleaseHybridnetChart is used to release hybridnet chart to local path
+type ReleaseHybridnetChart struct {
+	common.KubeAction
+}
+
+func (r *ReleaseHybridnetChart) Execute(runtime connector.Runtime) error {
+	fs, err := os.Create(fmt.Sprintf("%s/hybridnet.tgz", runtime.GetWorkDir()))
+	if err != nil {
+		return err
+	}
+	chartFile, err := f.Open("hybridnet-0.6.6.tgz")
+	if err != nil {
+		return err
+	}
+	defer chartFile.Close()
+
+	_, err = io.Copy(fs, chartFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SyncHybridnetChart is used to sync hybridnet chart to contronplane
+type SyncHybridnetChart struct {
+	common.KubeAction
+}
+
+func (s *SyncHybridnetChart) Execute(runtime connector.Runtime) error {
+	src := filepath.Join(runtime.GetWorkDir(), "hybridnet.tgz")
+	dst := filepath.Join(common.TmpDir, "hybridnet.tgz")
+	if err := runtime.GetRunner().Scp(src, dst); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync hybridnet chart failed"))
+	}
+	if _, err := runtime.GetRunner().SudoCmd(fmt.Sprintf("mv %s/hybridnet.tgz /etc/kubernetes", common.TmpDir), true); err != nil {
+		return errors.Wrap(errors.WithStack(err), "sync hybrident chart failed")
+	}
+	return nil
+}
+
+type DeployHybridnet struct {
+	common.KubeAction
+}
+
+func (d *DeployHybridnet) Execute(runtime connector.Runtime) error {
+
+	cmd := fmt.Sprintf("/usr/local/bin/helm upgrade --install hybridnet /etc/kubernetes/hybridnet.tgz --namespace kube-system "+
+		"--set images.hybridnet.image=%s/%s "+
+		"--set images.hybridnet.tag=%s "+
+		"--set images.registryURL=%s ",
+		images.GetImage(runtime, d.KubeConf, "hybridnet").ImageNamespace(),
+		images.GetImage(runtime, d.KubeConf, "hybridnet").Repo,
+		images.GetImage(runtime, d.KubeConf, "hybridnet").Tag,
+		images.GetImage(runtime, d.KubeConf, "hybridnet").ImageRegistryAddr(),
+	)
+
+	if d.KubeConf.Cluster.Network.Hybridnet.EnableInit() {
+		cmd = fmt.Sprintf("%s --set init.cidr=%s", cmd, d.KubeConf.Cluster.Network.KubePodsCIDR)
+	} else {
+		cmd = fmt.Sprintf("%s --set init=null", cmd)
+	}
+
+	if !d.KubeConf.Cluster.Network.Hybridnet.NetworkPolicy() {
+		cmd = fmt.Sprintf("%s --set daemon.enableNetworkPolicy=false", cmd)
+	}
+
+	if d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces != "" {
+		cmd = fmt.Sprintf("%s --set daemon.preferBGPInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferBGPInterfaces)
+	}
+
+	if d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces != "" {
+		cmd = fmt.Sprintf("%s --set daemon.preferVlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVlanInterfaces)
+	}
+
+	if d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces != "" {
+		cmd = fmt.Sprintf("%s --set daemon.preferVxlanInterfaces=%s", cmd, d.KubeConf.Cluster.Network.Hybridnet.PreferVxlanInterfaces)
+	}
+
+	if _, err := runtime.GetRunner().SudoCmd(cmd, true); err != nil {
+		return errors.Wrap(errors.WithStack(err), "deploy hybridnet failed")
+	}
+
+	if len(d.KubeConf.Cluster.Network.Hybridnet.Networks) != 0 {
+		templateAction := action.Template{
+			Template: templates.HybridnetNetworks,
+			Dst:      filepath.Join(common.KubeConfigDir, templates.HybridnetNetworks.Name()),
+			Data: util.Data{
+				"Networks": d.KubeConf.Cluster.Network.Hybridnet.Networks,
+			},
+		}
+
+		templateAction.Init(nil, nil)
+		if err := templateAction.Execute(runtime); err != nil {
+			return err
+		}
+
+		for i := 0; i < 30; i++ {
+			fmt.Println("Waiting for hybridnet webhook running ... ", i+1)
+			time.Sleep(10 * time.Second)
+			output, _ := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl get pod -n kube-system -l  app=hybridnet,component=webhook | grep Running", false)
+			if strings.Contains(output, "1/1") {
+				time.Sleep(50 * time.Second)
+				break
+			}
+		}
+
+		if _, err := runtime.GetRunner().SudoCmd("/usr/local/bin/kubectl apply -f /etc/kubernetes/hybridnet-networks.yaml", true); err != nil {
+			return errors.Wrap(errors.WithStack(err), "apply hybridnet networks failed")
+		}
+	}
+	return nil
+}