Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: azure network plugin mode #3610

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion bcs-services/bcs-cluster-manager/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0
Expand Down Expand Up @@ -72,6 +71,11 @@ require (
sigs.k8s.io/aws-iam-authenticator v0.6.17
)

require (
github.com/Azure/azure-sdk-for-go/profile/p20200901 v0.1.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3 v3.0.0
)

require (
cloud.google.com/go v0.112.0 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ type GetProjectResourceQuotaUsageAction struct {

groups []cmproto.NodeGroup

regionInsTypes map[string][]string
regionInsTypes map[string][]string // nolint
}

// NewGetProjectResourceQuotaUsageAction create action
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"strings"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/pkg/errors"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"time"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/pkg/errors"

proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"context"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"

proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ package api
import (
"context"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/pkg/errors"

proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ import (
"strconv"
"strings"

"github.com/Azure/azure-sdk-for-go/profile/p20200901/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
armcontainerservicev3 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"

Expand All @@ -35,11 +36,11 @@ import (
// bcs nodeGroup 转换为 aks agentPool
type nodeGroupToPool struct {
group *proto.NodeGroup
pool *armcontainerservice.AgentPool
pool *armcontainerservicev3.AgentPool
}

// newNodeGroupToAgentPoolConverter create nodeGroupToPool
func newNodeGroupToAgentPoolConverter(group *proto.NodeGroup, pool *armcontainerservice.AgentPool) *nodeGroupToPool {
func newNodeGroupToAgentPoolConverter(group *proto.NodeGroup, pool *armcontainerservicev3.AgentPool) *nodeGroupToPool {
return &nodeGroupToPool{
group: group,
pool: pool,
Expand All @@ -49,7 +50,7 @@ func newNodeGroupToAgentPoolConverter(group *proto.NodeGroup, pool *armcontainer
// convert 转换
func (c *nodeGroupToPool) convert() {
if c.pool.Properties == nil {
c.pool.Properties = new(armcontainerservice.ManagedClusterAgentPoolProfileProperties)
c.pool.Properties = new(armcontainerservicev3.ManagedClusterAgentPoolProfileProperties)
}
properties := c.pool.Properties
// 以下为必填参数
Expand All @@ -65,7 +66,7 @@ func (c *nodeGroupToPool) convert() {
// 设置节点池大小
properties.Count = to.Ptr(int32(c.group.AutoScaling.DesiredSize))
// 设置节点池为用户模式
properties.Mode = to.Ptr(armcontainerservice.AgentPoolModeUser)
properties.Mode = to.Ptr(armcontainerservicev3.AgentPoolModeUser)
// 设置Azure节点池的弹性伸缩
properties.EnableAutoScaling = to.Ptr(false)
// 是否分配公网IP
Expand Down Expand Up @@ -108,7 +109,7 @@ func (c *nodeGroupToPool) setScalingMode() {
return
}
found := false
for _, mode := range armcontainerservice.PossibleScaleDownModeValues() {
for _, mode := range armcontainerservicev3.PossibleScaleDownModeValues() {
if string(mode) == asg.ScalingMode {
found = true
break
Expand All @@ -117,19 +118,19 @@ func (c *nodeGroupToPool) setScalingMode() {
if !found {
return
}
c.pool.Properties.ScaleDownMode = to.Ptr(armcontainerservice.ScaleDownMode(asg.ScalingMode))
c.pool.Properties.ScaleDownMode = to.Ptr(armcontainerservicev3.ScaleDownMode(asg.ScalingMode))
}

// setOSDiskType 设置系统盘类型
func (c *nodeGroupToPool) setOSDiskType() {
// 默认 使用托管类型(Managed)
c.pool.Properties.OSDiskType = to.Ptr(armcontainerservice.OSDiskTypeManaged)
c.pool.Properties.OSDiskType = to.Ptr(armcontainerservicev3.OSDiskTypeManaged)
lc := c.group.LaunchTemplate
if lc.SystemDisk == nil || len(lc.SystemDisk.DiskType) == 0 {
return
}
found := false
for _, osDiskType := range armcontainerservice.PossibleOSDiskTypeValues() {
for _, osDiskType := range armcontainerservicev3.PossibleOSDiskTypeValues() {
if string(osDiskType) == lc.SystemDisk.DiskType {
found = true
break
Expand All @@ -138,7 +139,7 @@ func (c *nodeGroupToPool) setOSDiskType() {
if !found {
return
}
c.pool.Properties.OSDiskType = to.Ptr(armcontainerservice.OSDiskType(lc.SystemDisk.DiskType))
c.pool.Properties.OSDiskType = to.Ptr(armcontainerservicev3.OSDiskType(lc.SystemDisk.DiskType))
}

// setOSDiskSizeGB 设置系统盘大小
Expand All @@ -165,14 +166,14 @@ func (c *nodeGroupToPool) setOSDiskSizeGB() {
func (c *nodeGroupToPool) setOSAndInstanceType() {
nodeOS := c.group.NodeOS
// 默认系统类型为linux
c.pool.Properties.OSType = to.Ptr(armcontainerservice.OSTypeLinux)
c.pool.Properties.OSType = to.Ptr(armcontainerservicev3.OSTypeLinux)
if strings.Contains(nodeOS, winTypeOS) {
c.pool.Properties.OSType = to.Ptr(armcontainerservice.OSTypeWindows)
c.pool.Properties.OSType = to.Ptr(armcontainerservicev3.OSTypeWindows)
}
// 默认OS为Ubuntu
c.pool.Properties.OSSKU = to.Ptr(armcontainerservice.OSSKUUbuntu)
c.pool.Properties.OSSKU = to.Ptr(armcontainerservicev3.OSSKUUbuntu)
if c.group.LaunchTemplate != nil && c.group.LaunchTemplate.ImageInfo != nil {
c.pool.Properties.OSSKU = to.Ptr(armcontainerservice.OSSKU(c.group.LaunchTemplate.ImageInfo.ImageName))
c.pool.Properties.OSSKU = to.Ptr(armcontainerservicev3.OSSKU(c.group.LaunchTemplate.ImageInfo.ImageName))
}
if c.group.LaunchTemplate != nil {
c.pool.Properties.VMSize = to.Ptr(c.group.LaunchTemplate.InstanceType)
Expand Down Expand Up @@ -306,14 +307,14 @@ func (c *nodeGroupToPool) regexpMaxPods(kp string) int32 {
// poolToNodeGroup AKS agentPool 转换为 BCS NodeGroup
type poolToNodeGroup struct {
group *proto.NodeGroup
pool *armcontainerservice.AgentPool
properties *armcontainerservice.ManagedClusterAgentPoolProfileProperties
pool *armcontainerservicev3.AgentPool
properties *armcontainerservicev3.ManagedClusterAgentPoolProfileProperties
}

// newPoolToNodeGroupConverter create poolToNodeGroup
func newPoolToNodeGroupConverter(pool *armcontainerservice.AgentPool, group *proto.NodeGroup) *poolToNodeGroup {
func newPoolToNodeGroupConverter(pool *armcontainerservicev3.AgentPool, group *proto.NodeGroup) *poolToNodeGroup {
if pool.Properties == nil {
pool.Properties = new(armcontainerservice.ManagedClusterAgentPoolProfileProperties)
pool.Properties = new(armcontainerservicev3.ManagedClusterAgentPoolProfileProperties)
}
return &poolToNodeGroup{
group: group,
Expand Down Expand Up @@ -408,9 +409,9 @@ func (c *poolToNodeGroup) setScaleDownMode() {
return
}
switch *mode {
case armcontainerservice.ScaleDownModeDelete:
case armcontainerservicev3.ScaleDownModeDelete:
asg.ScalingMode = "CLASSIC_SCALING"
case armcontainerservice.ScaleDownModeDeallocate:
case armcontainerservicev3.ScaleDownModeDeallocate:
asg.ScalingMode = "WAKE_UP_STOPPED_SCALING"
}
}
Expand Down Expand Up @@ -1496,12 +1497,12 @@ func SetImageReferenceNull(set *armcompute.VirtualMachineScaleSet) {
}

// SetAgentPoolFromNodeGroup 更新pool,只允许设置tag、vm、label、taint
func SetAgentPoolFromNodeGroup(group *proto.NodeGroup, pool *armcontainerservice.AgentPool) {
func SetAgentPoolFromNodeGroup(group *proto.NodeGroup, pool *armcontainerservicev3.AgentPool) {
if group == nil || pool == nil {
return
}
if pool.Properties == nil {
pool.Properties = new(armcontainerservice.ManagedClusterAgentPoolProfileProperties)
pool.Properties = new(armcontainerservicev3.ManagedClusterAgentPoolProfileProperties)
}
pool.Properties.Tags = make(map[string]*string)
for k := range group.Tags {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"time"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"

proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"fmt"
"sync"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"

proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"
"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/operator"
"github.com/pkg/errors"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"
"github.com/avast/retry-go"
Expand Down Expand Up @@ -168,7 +168,7 @@ func createAKSCluster(ctx context.Context, info *cloudprovider.CloudDependBasicI
}

// generateCreateClusterRequest generate create cluster request
func generateCreateClusterRequest(info *cloudprovider.CloudDependBasicInfo, groups []*proto.NodeGroup) (
func generateCreateClusterRequest(info *cloudprovider.CloudDependBasicInfo, groups []*proto.NodeGroup) ( // nolint
*armcontainerservice.ManagedCluster, error) {
cluster := info.Cluster
if cluster.NetworkSettings == nil {
Expand Down Expand Up @@ -240,11 +240,20 @@ func generateCreateClusterRequest(info *cloudprovider.CloudDependBasicInfo, grou
NetworkProfile: &armcontainerservice.NetworkProfile{
NetworkPlugin: to.Ptr(armcontainerservice.NetworkPluginAzure),
NetworkPolicy: to.Ptr(armcontainerservice.NetworkPolicyCalico),
PodCidr: to.Ptr(cluster.NetworkSettings.ClusterIPv4CIDR), // nolint
ServiceCidr: to.Ptr(cluster.NetworkSettings.ServiceIPv4CIDR), // nolint
DNSServiceIP: to.Ptr(genDNSServiceIP(cluster.NetworkSettings.ServiceIPv4CIDR)), // nolint
ServiceCidrs: []*string{to.Ptr(cluster.NetworkSettings.ServiceIPv4CIDR)}, // nolint
PodCidrs: []*string{to.Ptr(cluster.NetworkSettings.ClusterIPv4CIDR)}, // nolint
NetworkPluginMode: func() *armcontainerservice.NetworkPluginMode {
if cluster.ClusterAdvanceSettings.NetworkType == common.AzureCniOverlay {
// 使用azure cni overlay插件
return to.Ptr(armcontainerservice.NetworkPluginModeOverlay)
}

// 使用azure cni node subnet插件
return to.Ptr(armcontainerservice.NetworkPluginMode(""))
}(),
PodCidr: to.Ptr(cluster.NetworkSettings.ClusterIPv4CIDR),
ServiceCidr: to.Ptr(cluster.NetworkSettings.ServiceIPv4CIDR), // nolint
DNSServiceIP: to.Ptr(genDNSServiceIP(cluster.NetworkSettings.ServiceIPv4CIDR)), // nolint
ServiceCidrs: []*string{to.Ptr(cluster.NetworkSettings.ServiceIPv4CIDR)}, // nolint
//PodCidrs: []*string{to.Ptr(cluster.NetworkSettings.ClusterIPv4CIDR)}, // nolint
},
ServicePrincipalProfile: &armcontainerservice.ManagedClusterServicePrincipalProfile{
ClientID: to.Ptr(info.CmOption.Account.ClientID), // nolint
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"strconv"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"
"github.com/pkg/errors"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v3"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
"github.com/Tencent/bk-bcs/bcs-common/common/blog"
"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/operator"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ func (ng *NodeGroup) SwitchAutoScalingOptionStatus(scalingOption *proto.ClusterA
}

// GetProjectCaResourceQuota get project ca resource quota
func (ng *NodeGroup) GetProjectCaResourceQuota(groups []proto.NodeGroup,
func (ng *NodeGroup) GetProjectCaResourceQuota(groups []proto.NodeGroup, // nolint
opt *cloudprovider.CommonOption) ([]*proto.ProjectAutoscalerQuota, error) {

// 仅统计CA云梯资源 & 获取项目下所有节点池的资源使用情况 & 资源quota情况
Expand Down Expand Up @@ -457,8 +457,9 @@ func (ng *NodeGroup) GetProjectCaResourceQuota(groups []proto.NodeGroup,
}
} else {
insZoneQuota[group.GetLaunchTemplate().GetInstanceType()][zone].Used += uint32(num)
insZoneQuota[group.GetLaunchTemplate().GetInstanceType()][zone].GroupIds =
append(insZoneQuota[group.GetLaunchTemplate().GetInstanceType()][zone].GroupIds, group.GetNodeGroupID())
insZoneQuota[group.GetLaunchTemplate().GetInstanceType()][zone].GroupIds = append(
insZoneQuota[group.GetLaunchTemplate().GetInstanceType()][zone].GroupIds,
group.GetNodeGroupID())
}
}

Expand All @@ -477,8 +478,6 @@ func (ng *NodeGroup) GetProjectCaResourceQuota(groups []proto.NodeGroup,
}
}
lock.Unlock()

return
}(filterGroups[i])
}
barrier.Wait()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -920,4 +920,3 @@ func (ng *NodeGroup) GetProjectCaResourceQuota(groups []proto.NodeGroup,
opt *cloudprovider.CommonOption) ([]*proto.ProjectAutoscalerQuota, error) {
return nil, nil
}

16 changes: 8 additions & 8 deletions bcs-services/bcs-cluster-manager/internal/remote/cmdb/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,9 @@ func (c Condition) String() string {
}

var (
and Condition = "AND"
or Condition = "OR"
in Condition = "IN"// nolint
and Condition = "AND" // nolint
or Condition = "OR" // nolint
in Condition = "IN" // nolint
)

// Page page
Expand Down Expand Up @@ -297,11 +297,11 @@ type HostDetailData struct {
}

const (
keyBizID = "BsiId"
keySvrIP = "SvrIp"
methodBusiness = "Business"
methodServer = "Server"
methodBusinessRaw = "BusinessRaw"
keyBizID = "BsiId" // nolint
keySvrIP = "SvrIp" // nolint
methodBusiness = "Business" // nolint
methodServer = "Server" // nolint
methodBusinessRaw = "BusinessRaw" // nolint
)

// BizInfo business id info
Expand Down
Loading