Skip to content

Commit

Permalink
feat(controller): support vmnetcfg disabled mode
Browse files Browse the repository at this point in the history 
When an VirtualMachineNetworkConfig is disabled, it will free up the
slot it occupied in IPAM and remove the MAC cache entry. Also, the
referencing IPPool's status will be updated. However, the allocation
record still remain on its status field. When the
VirtualMachineNetworkConfig is enabled again, the same IP address will
be allocated.

Signed-off-by: Zespre Chang <[email protected]>
  • Loading branch information
@starbops
starbops committed Jan 22, 2024
1 parent e6a77dd commit 733fe3d
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 29 deletions.
23 changes: 17 additions & 6 deletions pkg/apis/network.harvesterhci.io/v1alpha1/virtualmachinenetworkconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,18 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
AllocatedState NetworkConfigState = "Allocated"
PendingState NetworkConfigState = "Pending"
)

var (
Allocated condition.Cond = "Allocated"
Disabled condition.Cond = "Disabled"
)

type NetworkConfigState string

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:resource:shortName=vmnetcfg;vmnetcfgs,scope=Namespaced
Expand All @@ -29,25 +36,29 @@ type VirtualMachineNetworkConfig struct {
type VirtualMachineNetworkConfigSpec struct {
VMName string `json:"vmName,omitempty"`
NetworkConfig []NetworkConfig `json:"networkConfig,omitempty"`

// +optional
Paused *bool `json:"paused,omitempty"`
}

type NetworkConfig struct {
NetworkName string `json:"networkName,omitempty"`
MACAddress string `json:"macAddress,omitempty"`

// +optional
IPAddress *string `json:"ipAddress,omitempty"`
}

type VirtualMachineNetworkConfigStatus struct {
NetworkConfig []NetworkConfigStatus `json:"networkConfig,omitempty"`
// Conditions is a list of Wrangler conditions that describe the state
// of the VirtualMachineNetworkConfigStatus.

// +optional
Conditions []genericcondition.GenericCondition `json:"conditions,omitempty"`
}

type NetworkConfigStatus struct {
AllocatedIPAddress string `json:"allocatedIPAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkName string `json:"networkName,omitempty"`
Status string `json:"status,omitempty"`
AllocatedIPAddress string `json:"allocatedIPAddress,omitempty"`
MACAddress string `json:"macAddress,omitempty"`
NetworkName string `json:"networkName,omitempty"`
State NetworkConfigState `json:"state,omitempty"`
}
102 changes: 83 additions & 19 deletions pkg/controller/vmnetcfg/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,50 @@ func Register(ctx context.Context, management *config.Management) error {
handler.Allocate,
)

vmnetcfgs.OnChange(ctx, controllerName, handler.OnChange)
vmnetcfgs.OnRemove(ctx, controllerName, handler.OnRemove)

return nil
}

func (h *Handler) OnChange(key string, vmNetCfg *networkv1.VirtualMachineNetworkConfig) (*networkv1.VirtualMachineNetworkConfig, error) {
if vmNetCfg == nil || vmNetCfg.DeletionTimestamp != nil {
return nil, nil
}

logrus.Debugf("(vmnetcfg.OnChange) vmnetcfg configuration %s has been changed: %+v", key, vmNetCfg.Spec.NetworkConfig)

vmNetCfgCpy := vmNetCfg.DeepCopy()

// Check if the VirtualMachineNetworkConfig is administratively disabled
if vmNetCfg.Spec.Paused != nil && *vmNetCfg.Spec.Paused {
logrus.Infof("(vmnetcfg.OnChange) try to cleanup ipam and cache, and update ippool status for vmnetcfg %s", key)
if err := h.cleanup(vmNetCfg); err != nil {
return vmNetCfg, err
}
networkv1.Disabled.True(vmNetCfgCpy)
updateAllNetworkConfigState(vmNetCfgCpy.Status.NetworkConfig)
if !reflect.DeepEqual(vmNetCfgCpy, vmNetCfg) {
return h.vmnetcfgClient.UpdateStatus(vmNetCfgCpy)
}
return vmNetCfg, nil
}
networkv1.Disabled.False(vmNetCfgCpy)

if !reflect.DeepEqual(vmNetCfgCpy, vmNetCfg) {
return h.vmnetcfgClient.UpdateStatus(vmNetCfgCpy)
}

return vmNetCfg, nil
}

func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, status networkv1.VirtualMachineNetworkConfigStatus) (networkv1.VirtualMachineNetworkConfigStatus, error) {
logrus.Debugf("(vmnetcfg.Allocate) allocate ip for vmnetcfg %s/%s", vmNetCfg.Namespace, vmNetCfg.Name)

if vmNetCfg.Spec.Paused != nil && *vmNetCfg.Spec.Paused {
return status, fmt.Errorf("vmnetcfg %s/%s was administratively disabled", vmNetCfg.Namespace, vmNetCfg.Name)
}

var ncStatuses []networkv1.NetworkConfigStatus
for _, nc := range vmNetCfg.Spec.NetworkConfig {
exists, err := h.cacheAllocator.HasMAC(nc.NetworkName, nc.MACAddress)
Expand All @@ -75,6 +111,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
}
if exists {
// Recover IP from cache

ip, err := h.cacheAllocator.GetIPByMAC(nc.NetworkName, nc.MACAddress)
if err != nil {
return status, err
Expand All @@ -85,7 +122,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
AllocatedIPAddress: ip,
MACAddress: nc.MACAddress,
NetworkName: nc.NetworkName,
Status: "Allocated",
State: networkv1.AllocatedState,
}
ncStatuses = append(ncStatuses, ncStatus)

Expand All @@ -95,7 +132,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
ncStatus.NetworkName,
ncStatus.MACAddress,
ncStatus.AllocatedIPAddress,
ncStatus.Status,
string(ncStatus.State),
)

// Update IPPool status
Expand Down Expand Up @@ -139,10 +176,15 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
}

// Allocate new IP

dIP := util.UnspecifiedIPAddress
if nc.IPAddress != nil {
dIP = *nc.IPAddress
}
// Recover IP from status (resume from paused state)
if oIP, err := findIPAddressFromNetworkConfigStatusByMACAddress(vmNetCfg.Status.NetworkConfig, nc.MACAddress); err == nil {
dIP = oIP
}

ip, err := h.ipAllocator.AllocateIP(nc.NetworkName, dIP)
if err != nil {
Expand All @@ -158,7 +200,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
AllocatedIPAddress: ip,
MACAddress: nc.MACAddress,
NetworkName: nc.NetworkName,
Status: "Allocated",
State: networkv1.AllocatedState,
}
ncStatuses = append(ncStatuses, ncStatus)

Expand All @@ -168,7 +210,7 @@ func (h *Handler) Allocate(vmNetCfg *networkv1.VirtualMachineNetworkConfig, stat
ncStatus.NetworkName,
ncStatus.MACAddress,
ncStatus.AllocatedIPAddress,
ncStatus.Status,
string(ncStatus.State),
)

// Update IPPool status
Expand Down Expand Up @@ -221,32 +263,40 @@ func (h *Handler) OnRemove(key string, vmNetCfg *networkv1.VirtualMachineNetwork

logrus.Debugf("(vmnetcfg.OnRemove) vmnetcfg configuration %s/%s has been removed", vmNetCfg.Namespace, vmNetCfg.Name)

h.metricsAllocator.DeleteVmNetCfgStatus(key)
if err := h.cleanup(vmNetCfg); err != nil {
return vmNetCfg, err
}

return vmNetCfg, nil
}

func (h *Handler) cleanup(vmNetCfg *networkv1.VirtualMachineNetworkConfig) error {
h.metricsAllocator.DeleteVmNetCfgStatus(vmNetCfg.Namespace + "/" + vmNetCfg.Name)

for _, nc := range vmNetCfg.Status.NetworkConfig {
for _, ncStatus := range vmNetCfg.Status.NetworkConfig {
// Deallocate IP address from IPAM
isAllocated, err := h.ipAllocator.IsAllocated(nc.NetworkName, nc.AllocatedIPAddress)
isAllocated, err := h.ipAllocator.IsAllocated(ncStatus.NetworkName, ncStatus.AllocatedIPAddress)
if err != nil {
return vmNetCfg, err
return err
}
if isAllocated {
if err := h.ipAllocator.DeallocateIP(nc.NetworkName, nc.AllocatedIPAddress); err != nil {
return vmNetCfg, err
if err := h.ipAllocator.DeallocateIP(ncStatus.NetworkName, ncStatus.AllocatedIPAddress); err != nil {
return err
}
}

// Remove entry from cache
exists, err := h.cacheAllocator.HasMAC(nc.NetworkName, nc.MACAddress)
exists, err := h.cacheAllocator.HasMAC(ncStatus.NetworkName, ncStatus.MACAddress)
if err != nil {
return vmNetCfg, err
return err
}
if exists {
if err := h.cacheAllocator.DeleteMAC(nc.NetworkName, nc.MACAddress); err != nil {
return vmNetCfg, err
if err := h.cacheAllocator.DeleteMAC(ncStatus.NetworkName, ncStatus.MACAddress); err != nil {
return err
}
}

ipPoolNamespace, ipPoolName := kv.RSplit(nc.NetworkName, "/")
ipPoolNamespace, ipPoolName := kv.RSplit(ncStatus.NetworkName, "/")
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
ipPool, err := h.ippoolCache.Get(ipPoolNamespace, ipPoolName)
if err != nil {
Expand All @@ -256,20 +306,34 @@ func (h *Handler) OnRemove(key string, vmNetCfg *networkv1.VirtualMachineNetwork
ipPoolCpy := ipPool.DeepCopy()

// Remove record in IPPool status
delete(ipPoolCpy.Status.IPv4.Allocated, nc.AllocatedIPAddress)
delete(ipPoolCpy.Status.IPv4.Allocated, ncStatus.AllocatedIPAddress)

if !reflect.DeepEqual(ipPoolCpy, ipPool) {
logrus.Infof("(vmnetcfg.OnRemove) update ippool %s/%s", ipPool.Namespace, ipPool.Name)
logrus.Infof("(vmnetcfg.cleanup) update ippool %s/%s", ipPool.Namespace, ipPool.Name)
ipPoolCpy.Status.LastUpdate = metav1.Now()
_, err := h.ippoolClient.UpdateStatus(ipPoolCpy)
return err
}

return nil
}); err != nil {
return vmNetCfg, err
return err
}
}
return nil
}

return vmNetCfg, nil
func findIPAddressFromNetworkConfigStatusByMACAddress(ncStatuses []networkv1.NetworkConfigStatus, macAddress string) (ipAddress string, err error) {
for _, ncStatus := range ncStatuses {
if ncStatus.MACAddress == macAddress && ncStatus.AllocatedIPAddress != "" {
return ncStatus.AllocatedIPAddress, nil
}
}
return util.UnspecifiedIPAddress, fmt.Errorf("could not find allocated ip for mac %s", macAddress)
}

func updateAllNetworkConfigState(ncStatuses []networkv1.NetworkConfigStatus) {
for i := range ncStatuses {
ncStatuses[i].State = networkv1.PendingState
}
}
8 changes: 4 additions & 4 deletions pkg/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ var (
LabelVmNetCfgName = "vmnetcfg"
LabelMACAddress = "mac"
LabelIPAddress = "ip"
LabelStatus = "status"
LabelState = "state"
)

type MetricsAllocator struct {
Expand Down Expand Up @@ -60,7 +60,7 @@ func NewMetricsAllocator() *MetricsAllocator {
LabelNetworkName,
LabelMACAddress,
LabelIPAddress,
LabelStatus,
LabelState,
},
),
}
Expand Down Expand Up @@ -103,13 +103,13 @@ func (a *MetricsAllocator) DeleteIPPool(name string, cidr string, networkName st
})
}

func (a *MetricsAllocator) UpdateVmNetCfgStatus(name, networkName, macAddress, ipAddress, status string) {
func (a *MetricsAllocator) UpdateVmNetCfgStatus(name, networkName, macAddress, ipAddress, state string) {
a.vmNetCfgStatus.With(prometheus.Labels{
LabelVmNetCfgName: name,
LabelNetworkName: networkName,
LabelMACAddress: macAddress,
LabelIPAddress: ipAddress,
LabelStatus: status,
LabelState: state,
}).Set(float64(1))
}

Expand Down

0 comments on commit 733fe3d

Please sign in to comment.