diff --git a/doc/crds/daemonset-install.yaml b/doc/crds/daemonset-install.yaml index 2c6ad832a..2951b557b 100644 --- a/doc/crds/daemonset-install.yaml +++ b/doc/crds/daemonset-install.yaml @@ -48,6 +48,7 @@ rules: verbs: - list - watch + - get - apiGroups: [""] resources: - nodes @@ -111,6 +112,8 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: WHEREABOUTS_RECONCILER_CRON + value: 1 * * * * - name: WHEREABOUTS_NAMESPACE valueFrom: fieldRef: diff --git a/pkg/config/config.go b/pkg/config/config.go index 685f2cbbb..15f805f64 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -52,7 +52,7 @@ func LoadIPAMConfig(bytes []byte, envArgs string, extraConfigPaths ...string) (* } n.IPAM.PodName = string(args.K8S_POD_NAME) n.IPAM.PodNamespace = string(args.K8S_POD_NAMESPACE) - + n.IPAM.Args = n.Args flatipam, foundflatfile, err := GetFlatIPAM(false, n.IPAM, extraConfigPaths...) if err != nil { return nil, "", err @@ -78,6 +78,11 @@ func LoadIPAMConfig(bytes []byte, envArgs string, extraConfigPaths ...string) (* logging.Debugf("Used defaults from parsed flat file config @ %s", foundflatfile) } + n.IPAM.Args = n.Args + logging.Debugf("carried over the IPAM args: %v", n.Args.CNI) + for k, v := range n.Args.CNI { + logging.Debugf("k: %q, v: %q", k, v) + } if n.IPAM.Range != "" { oldRange := types.RangeConfiguration{ diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 994d408e1..d29f14d18 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -398,6 +398,31 @@ var _ = Describe("Allocation operations", func() { HavePrefix( "LoadIPAMConfig - JSON Parsing Error: invalid character 'a' looking for beginning of object key string"))) }) + + It("can pass arbitrary cni-args", func() { + const ( + arbitraryCNIArgsKey = "asdfg" + arbitraryCNIArgsValue = "9872626" + ) + var globalConf string = `{ + "datastore": "kubernetes", + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" + }, + "log_file": "/tmp/whereabouts.log", + "log_level": "debug", + "gateway": "192.168.5.5", + "enable_overlapping_ranges": true + }` + Expect(os.WriteFile("/tmp/whereabouts.conf", []byte(globalConf), 0755)).To(Succeed()) + + ipamconfig, _, err := LoadIPAMConfig( + []byte(generateIPAMConfWithArbitraryCNIArg(arbitraryCNIArgsKey, arbitraryCNIArgsValue)), + "", + ) + Expect(err).NotTo(HaveOccurred()) + Expect(ipamconfig.Args.CNI).To(HaveKeyWithValue(arbitraryCNIArgsKey, arbitraryCNIArgsValue)) + }) }) func generateIPAMConfWithOverlappingRanges() string { @@ -429,3 +454,22 @@ func generateIPAMConfWithoutOverlappingRanges() string { } }` } + +func generateIPAMConfWithArbitraryCNIArg(k, v string) string { + return fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "mynet", + "type": "ipvlan", + "master": "foo0", + "ipam": { + "range": "192.168.2.230/24", + "configuration_path": "/tmp/whereabouts.conf", + "type": "whereabouts", + "args": { + "cni": { + %q: %q + } + } + } + }`, k, v) +} diff --git a/pkg/storage/kubernetes/ipam.go b/pkg/storage/kubernetes/ipam.go index 299c3ee58..2eea08895 100644 --- a/pkg/storage/kubernetes/ipam.go +++ b/pkg/storage/kubernetes/ipam.go @@ -29,6 +29,13 @@ import ( "gomodules.xyz/jsonpatch/v2" ) +const ( + StickyIPRequestOwnerTypeAnnotation = "whereabouts.cni.cncf.io/ownerType" + StickyIPRequestOwnerIDAnnotation = "whereabouts.cni.cncf.io/ownerID" + StickyIPRequestOwnerNameAnnotation = "whereabouts.cni.cncf.io/ownerName" + StickyIPRequestOwnerVersionAnnotation = "whereabouts.cni.cncf.io/ownerVersion" +) + // NewKubernetesIPAM returns a new KubernetesIPAM Client configured to a kubernetes CRD backend func NewKubernetesIPAM(containerID string, ipamConf whereaboutstypes.IPAMConfig) (*KubernetesIPAM, error) { var namespace string @@ -193,8 +200,8 @@ type KubernetesOverlappingRangeStore struct { } // GetOverlappingRangeStore returns a clusterstore interface -func (i *KubernetesIPAM) GetOverlappingRangeStore() (storage.OverlappingRangeStore, error) { - return &KubernetesOverlappingRangeStore{i.client, i.containerID, i.namespace}, nil +func (i *KubernetesIPAM) GetOverlappingRangeStore(namespace string) (storage.OverlappingRangeStore, error) { + return &KubernetesOverlappingRangeStore{i.client, i.containerID, namespace}, nil } // IsAllocatedInOverlappingRange checks for IP addresses to see if they're allocated cluster wide, for overlapping @@ -221,33 +228,99 @@ func (c *KubernetesOverlappingRangeStore) IsAllocatedInOverlappingRange(ctx cont return true, nil } +func (c *KubernetesOverlappingRangeStore) RetrievePreviousAllocation(ctx context.Context, ownerRef string, networkName string) (*whereaboutsv1alpha1.OverlappingRangeIPReservation, error) { + ownerLabelSelector := fmt.Sprintf("persistentips.cni.cncf.io/owner=%s", ownerRef) + logging.Debugf("owner label selector: %q", ownerLabelSelector) + prevAllocations, err := c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).List( + ctx, + metav1.ListOptions{LabelSelector: ownerLabelSelector}, + ) + if err != nil && errors.IsNotFound(err) { + // cluster ip reservation does not exist, this appears to be good news. + // logging.Debugf("IP %v is not reserved cluster wide, allowing.", ip) + return nil, nil + } else if err != nil { + logging.Errorf("error getting k8s OverlappingRangeIPReservation for network %q: %s", networkName, err) + return nil, fmt.Errorf("k8s get OverlappingRangeIPReservation error: %s", err) + } + if len(prevAllocations.Items) > 0 { + return &prevAllocations.Items[0], nil + } + return nil, nil +} + // UpdateOverlappingRangeAllocation updates clusterwide allocation for overlapping ranges. -func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, - containerID, podRef, networkName string) error { +func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation( + ctx context.Context, + mode int, + ip net.IP, + containerID, podRef, networkName string, + ownerReference *metav1.OwnerReference, + existingAllocation *whereaboutsv1alpha1.OverlappingRangeIPReservation, +) error { normalizedIP := normalizeIP(ip, networkName) - clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{ - ObjectMeta: metav1.ObjectMeta{Name: normalizedIP, Namespace: c.namespace}, + var ownerReferences []metav1.OwnerReference + if ownerReference != nil { + logging.Debugf("adding owner reference %v", ownerReference) + ownerReferences = append(ownerReferences, *ownerReference) } var err error var verb string + clusteripres := &whereaboutsv1alpha1.OverlappingRangeIPReservation{} + switch mode { case whereaboutstypes.Allocate: // Put together our cluster ip reservation verb = "allocate" - clusteripres.Spec = whereaboutsv1alpha1.OverlappingRangeIPReservationSpec{ - ContainerID: containerID, - PodRef: podRef, + if existingAllocation != nil { + clusteripres = existingAllocation.DeepCopy() + clusteripres.OwnerReferences = ownerReferences + clusteripres.Spec.PodRef = podRef + clusteripres.Spec.ContainerID = containerID + } else { + clusteripres = &whereaboutsv1alpha1.OverlappingRangeIPReservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: normalizedIP, + Namespace: c.namespace, + OwnerReferences: ownerReferences, + }, + Spec: whereaboutsv1alpha1.OverlappingRangeIPReservationSpec{ + ContainerID: containerID, + PodRef: podRef, + }, + } + } + + if ownerReference != nil { + logging.Debugf("properly labeling %s with %v", normalizedIP, *ownerReference) + clusteripres.Labels = map[string]string{ + "persistentips.cni.cncf.io/owner": ipReservationLabel(ownerReference), + } } - _, err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Create( - ctx, clusteripres, metav1.CreateOptions{}) + logging.Debugf("allocation to persist: %v", *clusteripres) + + if existingAllocation == nil { + logging.Debugf("will CREATE reservation for %q", normalizedIP) + _, err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Create( + ctx, clusteripres, metav1.CreateOptions{}) + } else { + logging.Debugf("will UPDATE reservation for %q", normalizedIP) + _, err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Update( + ctx, clusteripres, metav1.UpdateOptions{}, + ) + } case whereaboutstypes.Deallocate: verb = "deallocate" - err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Delete(ctx, clusteripres.GetName(), metav1.DeleteOptions{}) + err = c.client.WhereaboutsV1alpha1().OverlappingRangeIPReservations(c.namespace).Delete( + ctx, + normalizedIP, + metav1.DeleteOptions{}, + ) } if err != nil { @@ -258,6 +331,10 @@ func (c *KubernetesOverlappingRangeStore) UpdateOverlappingRangeAllocation(ctx c return nil } +func ipReservationLabel(ownerReference *metav1.OwnerReference) string { + return fmt.Sprintf("%s.%s.%s", ownerReference.Kind, ownerReference.Name, ownerReference.UID) +} + // normalizeIP normalizes the IP. This is important for IPv6 which doesn't make for valid CR names. It also allows us // to add the network-name when it's different from the unnamed network. func normalizeIP(ip net.IP, networkName string) string { @@ -485,6 +562,8 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete // handle the ip add/del until successful var overlappingrangeallocations []whereaboutstypes.IPReservation var ipforoverlappingrangeupdate net.IP + var previousAllocation *whereaboutsv1alpha1.OverlappingRangeIPReservation + for _, ipRange := range ipamConf.IPRanges { RETRYLOOP: for j := 0; j < storage.DatastoreRetries; j++ { @@ -495,7 +574,7 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete // retry the IPAM loop if the context has not been cancelled } - overlappingrangestore, err = ipam.GetOverlappingRangeStore() + overlappingrangestore, err = ipam.GetOverlappingRangeStore(ipamConf.PodNamespace) if err != nil { logging.Errorf("IPAM error getting OverlappingRangeStore: %v", err) return newips, err @@ -524,11 +603,45 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete // When it's allocated overlappingrange wide, we add it to a local reserved list // And we try again. if ipamConf.OverlappingRanges { - isAllocated, err := overlappingrangestore.IsAllocatedInOverlappingRange(requestCtx, newip.IP, - ipamConf.NetworkName) - if err != nil { - logging.Errorf("Error checking overlappingrange allocation: %v", err) - return newips, err + var isAllocated bool + ownerID, hasOwnerIDAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerIDAnnotation] + ownerName, hasOwnerNameAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerNameAnnotation] + ownerType, hasOwnerTypeAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerTypeAnnotation] + ownerVersion, hasOwnerVersionAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerVersionAnnotation] + + var ownerRef *metav1.OwnerReference + + hasOwnerInfo := hasOwnerTypeAnnotation && hasOwnerNameAnnotation && hasOwnerIDAnnotation && hasOwnerVersionAnnotation + if hasOwnerInfo { + logging.Debugf("adding owner reference to persistent IP Allocation %q", ipforoverlappingrangeupdate.String()) + ownerRef = &metav1.OwnerReference{ + APIVersion: ownerVersion, + Kind: ownerType, + Name: ownerName, + UID: types.UID(ownerID), + } + + previousAllocation, err = overlappingrangestore.RetrievePreviousAllocation( + requestCtx, + ipReservationLabel(ownerRef), + ipamConf.NetworkName, + ) + + if err == nil && previousAllocation != nil { + logging.Debugf("found previous allocation for %s: %s", podRef, previousAllocation) + newip.IP = net.ParseIP(previousAllocation.Name) + } else if err != nil { + _ = logging.Errorf("could not retrieve previous allocations for %v: %v", ownerRef, err) + } else { + logging.Debugf("no prev allocation found for %q", ipamConf.NetworkName) + } + } else { + isAllocated, err = overlappingrangestore.IsAllocatedInOverlappingRange(requestCtx, newip.IP, + ipamConf.NetworkName) + if err != nil { + logging.Errorf("Error checking overlappingrange allocation: %v", err) + return newips, err + } } if isAllocated { @@ -574,11 +687,39 @@ func IPManagementKubernetesUpdate(ctx context.Context, mode int, ipam *Kubernete } if ipamConf.OverlappingRanges { - err = overlappingrangestore.UpdateOverlappingRangeAllocation(requestCtx, mode, ipforoverlappingrangeupdate, - containerID, podRef, ipamConf.NetworkName) - if err != nil { - logging.Errorf("Error performing UpdateOverlappingRangeAllocation: %v", err) - return newips, err + ownerID, hasOwnerIDAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerIDAnnotation] + ownerName, hasOwnerNameAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerNameAnnotation] + ownerType, hasOwnerTypeAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerTypeAnnotation] + ownerVersion, hasOwnerVersionAnnotation := ipamConf.Args.CNI[StickyIPRequestOwnerVersionAnnotation] + + var ownerRef *metav1.OwnerReference + + hasOwnerInfo := hasOwnerTypeAnnotation && hasOwnerNameAnnotation && hasOwnerIDAnnotation && hasOwnerVersionAnnotation + if hasOwnerInfo { + logging.Debugf("adding owner reference to persistent IP Allocation %q", ipforoverlappingrangeupdate.String()) + ownerRef = &metav1.OwnerReference{ + APIVersion: ownerVersion, + Kind: ownerType, + Name: ownerName, + UID: types.UID(ownerID), + } + } + + // we only update the allocation for create / update *or* pods without owner references. + if mode == whereaboutstypes.Allocate || !hasOwnerInfo { + if previousAllocation != nil { + logging.Debugf( + "will UPDATE persistent IP allocation for %q. previousAllocation: %v", + ipforoverlappingrangeupdate.String(), + *previousAllocation, + ) + } + err = overlappingrangestore.UpdateOverlappingRangeAllocation(requestCtx, mode, ipforoverlappingrangeupdate, + containerID, podRef, ipamConf.NetworkName, ownerRef, previousAllocation) + if err != nil { + logging.Errorf("Error performing UpdateOverlappingRangeAllocation: %v", err) + return newips, err + } } } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 9ebac433d..53e0129e6 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -2,6 +2,8 @@ package storage import ( "context" + whereaboutsv1alpha1 "github.com/k8snetworkplumbingwg/whereabouts/pkg/api/whereabouts.cni.cncf.io/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "net" "time" @@ -30,11 +32,20 @@ type Store interface { Close() error } +type OverlappingRangeHandler func() error + // OverlappingRangeStore is an interface for wrapping overlappingrange storage options type OverlappingRangeStore interface { IsAllocatedInOverlappingRange(ctx context.Context, ip net.IP, networkName string) (bool, error) - UpdateOverlappingRangeAllocation(ctx context.Context, mode int, ip net.IP, containerID string, podRef, - networkName string) error + RetrievePreviousAllocation(ctx context.Context, ownerRef string, networkName string) (*whereaboutsv1alpha1.OverlappingRangeIPReservation, error) + UpdateOverlappingRangeAllocation( + ctx context.Context, + mode int, + ip net.IP, + containerID, podRef, networkName string, + ownerRef *metav1.OwnerReference, + existingAllocation *whereaboutsv1alpha1.OverlappingRangeIPReservation, + ) error } type Temporary interface { diff --git a/pkg/types/types.go b/pkg/types/types.go index a12ec43ed..963df7d1a 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -26,6 +26,9 @@ type Net struct { Name string `json:"name"` CNIVersion string `json:"cniVersion"` IPAM *IPAMConfig `json:"ipam"` + Args struct { + CNI map[string]string `json:"cni"` + } `json:"args"` } // NetConfList describes an ordered list of networks. @@ -70,7 +73,10 @@ type IPAMConfig struct { ConfigurationPath string `json:"configuration_path"` PodName string PodNamespace string - NetworkName string `json:"network_name,omitempty"` + NetworkName string `json:"network_name,omitempty"` + Args struct { + CNI map[string]string `json:"cni"` + } `json:"args"` } func (ic *IPAMConfig) UnmarshalJSON(data []byte) error { @@ -106,7 +112,10 @@ func (ic *IPAMConfig) UnmarshalJSON(data []byte) error { ConfigurationPath string `json:"configuration_path"` PodName string PodNamespace string - NetworkName string `json:"network_name,omitempty"` + NetworkName string `json:"network_name,omitempty"` + Args struct { + CNI map[string]string `json:"cni"` + } `json:"args"` } ipamConfigAlias := IPAMConfigAlias{ @@ -143,6 +152,7 @@ func (ic *IPAMConfig) UnmarshalJSON(data []byte) error { PodName: ipamConfigAlias.PodName, PodNamespace: ipamConfigAlias.PodNamespace, NetworkName: ipamConfigAlias.NetworkName, + Args: ipamConfigAlias.Args, } return nil }