From 0fb7d72dd8453da12b44b9e0ce85adafc5f94ec0 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:39:01 -0500 Subject: [PATCH 01/48] feat: Iniitial Create of EMLB Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 3 +++ metal/loadbalancers/emlb/emlb.go | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 metal/loadbalancers/emlb/emlb.go diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 4673adf6..b07bfa6c 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -90,6 +90,9 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr case "empty": klog.Info("loadbalancer implementation enabled: empty, bgp only") impl = empty.NewLB(k8sclient, lbconfig) + case "emlb": + klog.Info("loadbalancer implementation enabled: empty, bgp only") + impl = empty.NewLB(k8sclient, lbconfig) default: klog.Info("loadbalancer implementation disabled") impl = nil diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go new file mode 100644 index 00000000..28208fd0 --- /dev/null +++ b/metal/loadbalancers/emlb/emlb.go @@ -0,0 +1,27 @@ +// Implementation of Equinix Metal Load Balancer +package emlb + +import ( + "context" + + "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + "k8s.io/client-go/kubernetes" +) + +type LB struct{} + +func NewLB(k8sclient kubernetes.Interface, config string) *LB { + return &LB{} +} + +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { + return nil +} + +func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { + return nil +} + +func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { + return nil +} From f178e49d42caefaeb61b656c532a00e1030f2b53 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 26 Sep 2023 12:58:22 -0500 Subject: [PATCH 02/48] define an Equinix Metal Load Balancer client and initialize it --- metal/loadbalancers/emlb/emlb.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 28208fd0..533c2ded 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -4,14 +4,20 @@ package emlb import ( "context" + lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "k8s.io/client-go/kubernetes" ) -type LB struct{} +type LB struct { + client *lbaas.APIClient +} func NewLB(k8sclient kubernetes.Interface, config string) *LB { - return &LB{} + lb := &LB{} + emlbConfig := lbaas.NewConfiguration() + lb.client = lbaas.NewAPIClient(emlbConfig) + return lb } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { From 42faf2267dc191860907f5e621598301b74acc47 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:09:25 -0500 Subject: [PATCH 03/48] chore: standardize markdown Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- README.md | 347 +++++++++++++++++++++++++++++------------------------- 1 file changed, 189 insertions(+), 158 deletions(-) diff --git a/README.md b/README.md index 6d2d5501..0e6699f3 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ [![Twitter Follow](https://img.shields.io/twitter/follow/equinixmetal.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=equinixmetal&user_id=788180534543339520) ![Equinix Metal Maintained](https://img.shields.io/badge/stability-maintained-green.svg) - `cloud-provider-equinix-metal` is the Kubernetes CCM implementation for Equinix Metal. Read more about the CCM in [the official Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/). This repository is [Maintained](https://github.com/equinix-labs/equinix-labs/blob/main/maintained-statement.md)! @@ -21,10 +20,11 @@ At the current state of Kubernetes, running the CCM requires a few things. Please read through the requirements carefully as they are critical to running the CCM on a Kubernetes cluster. ### Version + Recommended versions of Equinix Metal CCM based on your Kubernetes version: -* Equinix Metal CCM version v0.0.4 supports Kubernetes version >=v1.10 -* Equinix Metal CCM version v1.0.0+ supports Kubernetes version >=1.15.0 +- Equinix Metal CCM version v0.0.4 supports Kubernetes version >=v1.10 +- Equinix Metal CCM version v1.0.0+ supports Kubernetes version >=1.15.0 ### BGP @@ -41,8 +41,8 @@ If they come from the _public_ address, they will be dropped. There are two ways to get the packets to have the correct source address: -* use BGP software that knows how to set the source address on a packet -* set static routes on your host +- use BGP software that knows how to set the source address on a packet +- set static routes on your host #### BGP Software @@ -60,8 +60,8 @@ If your BGP software does not support using a specific source IP, then you must You need to retrieve the following: -* your private IPv4 upstream gateway address -* your BGP peer addresses +- your private IPv4 upstream gateway address +- your BGP peer addresses Before you can retrieve the information, you must enable BGP at both the Equinix Metal project level, and for each device. You can do this in the Equinix Metal Web UI, API or CLI. CCM ensures these settings on the project and each device. However, @@ -92,8 +92,8 @@ done Control plane binaries in your cluster must start with the correct flags: -* `kubelet`: All kubelets in your cluster **MUST** set the flag `--cloud-provider=external`. This must be done for _every_ kubelet. Note that [k3s](https://k3s.io) sets its own CCM by default. If you want to use the CCM with k3s, you must disable the k3s CCM and enable this one, as `--disable-cloud-controller --kubelet-arg cloud-provider=external`. -* `kube-apiserver` and `kube-controller-manager` must **NOT** set the flag `--cloud-provider`. They then will use no cloud provider natively, leaving room for the Equinix Metal CCM. +- `kubelet`: All kubelets in your cluster **MUST** set the flag `--cloud-provider=external`. This must be done for _every_ kubelet. Note that [k3s](https://k3s.io) sets its own CCM by default. If you want to use the CCM with k3s, you must disable the k3s CCM and enable this one, as `--disable-cloud-controller --kubelet-arg cloud-provider=external`. +- `kube-apiserver` and `kube-controller-manager` must **NOT** set the flag `--cloud-provider`. They then will use no cloud provider natively, leaving room for the Equinix Metal CCM. **WARNING**: setting the kubelet flag `--cloud-provider=external` will taint all nodes in a cluster with `node.cloudprovider.kubernetes.io/uninitialized`. The CCM itself will untaint those nodes when it initializes them. @@ -155,10 +155,9 @@ stringData: { "apiKey": "abc123abc123abc123", "projectID": "abc123abc123abc123" - } + } ``` - Then apply the secret, e.g.: ```bash @@ -167,17 +166,18 @@ kubectl apply -f /tmp/secret.yaml` You can confirm that the secret was created with the following: -````bash +```bash $ kubectl -n kube-system get secrets metal-cloud-config NAME TYPE DATA AGE metal-cloud-config Opaque 1 2m -```` +``` ### Deploy CCM To apply the CCM itself, select your release and apply the manifest: Example: + ``` RELEASE=v3.6.2 kubectl apply -f https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${RELEASE}/deployment.yaml @@ -198,9 +198,9 @@ See further in this document under loadbalancing, for details. By default, ccm does minimal logging, relying on the supporting infrastructure from kubernetes. However, it does support optional additional logging levels via the `--v=` flag. In general: -* `--v=2`: log most function calls for devices and facilities, when relevant logging the returned values -* `--v=3`: log additional data when logging returned values, usually entire go structs -* `--v=5`: log every function call, including those called very frequently +- `--v=2`: log most function calls for devices and facilities, when relevant logging the returned values +- `--v=3`: log additional data when logging returned values, usually entire go structs +- `--v=5`: log every function call, including those called very frequently ## Configuration @@ -213,29 +213,29 @@ The Equinix Metal CCM has multiple configuration options. These include three di This section lists each configuration option, and whether it can be set by each method. -| Purpose | CLI Flag | Env Var | Secret Field | Default | -| --- | --- | --- | --- | --- | -| Path to config secret | `cloud-config` | | | error | -| API Key | | `METAL_API_KEY` | `apiKey` | error | -| Project ID | | `METAL_PROJECT_ID` | `projectID` | error | -| Metro in which to create LoadBalancer Elastic IPs | | `METAL_METRO_NAME` | `metro` | Service-specific annotation, else error | -| Facility in which to create LoadBalancer Elastic IPs, only if Metro is not set | | `METAL_FACILITY_NAME` | `facility` | Service-specific annotation, else metro | -| Base URL to Equinix API | | | `base-url` | Official Equinix Metal API | -| Load balancer setting | | `METAL_LOAD_BALANCER` | `loadbalancer` | none | -| BGP ASN for cluster nodes when enabling BGP on the project; if the project **already** has BGP enabled, will use the existing BGP local ASN from the project | | `METAL_LOCAL_ASN` | `localASN` | `65000` | -| BGP passphrase to use when enabling BGP on the project; if the project **already** has BGP enabled, will use the existing BGP pass from the project | | `METAL_BGP_PASS` | `bgpPass` | `""` | -| Kubernetes annotation to set node's BGP ASN, `{{n}}` replaced with ordinal index of peer | | `METAL_ANNOTATION_LOCAL_ASN` | `annotationLocalASN` | `"metal.equinix.com/bgp-peers-{{n}}-node-asn"` | -| Kubernetes annotation to set BGP peer's ASN, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_PEER_ASN` | `annotationPeerASN` | `"metal.equinix.com/bgp-peers-{{n}}-peer-asn"` | -| Kubernetes annotation to set BGP peer's IPs, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_PEER_IP` | `annotationPeerIP` | `"metal.equinix.com/bgp-peers-{{n}}-peer-ip"` | -| Kubernetes annotation to set source IP for BGP peering, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_SRC_IP` | `annotationSrcIP` | `"metal.equinix.com/bgp-peers-{{n}}-src-ip"` | -| Kubernetes annotation to set BGP MD5 password, base64-encoded (see security warning below) | | `METAL_ANNOTATION_BGP_PASS` | `annotationBGPPass` | `"metal.equinix.com/bgp-peers-{{n}}-bgp-pass"` | -| Kubernetes annotation to set the CIDR for the network range of the private address | | `METAL_ANNOTATION_NETWORK_IPV4_PRIVATE` | `annotationNetworkIPv4Private` | `metal.equinix.com/network-4-private` | -| Kubernetes Service annotation to set EIP metro | | `METAL_ANNOTATION_EIP_METRO` | `annotationEIPMetro` | `"metal.equinix.com/eip-metro"` | -| Kubernetes Service annotation to set EIP facility | | `METAL_ANNOTATION_EIP_FACILITY` | `annotationEIPFacility` | `"metal.equinix.com/eip-facility"` | -| Tag for control plane Elastic IP | | `METAL_EIP_TAG` | `eipTag` | No control plane Elastic IP | -| Kubernetes API server port for Elastic IP | | `METAL_API_SERVER_PORT` | `apiServerPort` | Same as `kube-apiserver` on control plane nodes, same as `0` | -| Filter for cluster nodes on which to enable BGP | | `METAL_BGP_NODE_SELECTOR` | `bgpNodeSelector` | All nodes | -| Use host IP for Control Plane endpoint health checks | | `METAL_EIP_HEALTH_CHECK_USE_HOST_IP` | `eipHealthCheckUseHostIP` | false | +| Purpose | CLI Flag | Env Var | Secret Field | Default | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------- | --------------------------------------- | ------------------------------ | ------------------------------------------------------------ | +| Path to config secret | `cloud-config` | | | error | +| API Key | | `METAL_API_KEY` | `apiKey` | error | +| Project ID | | `METAL_PROJECT_ID` | `projectID` | error | +| Metro in which to create LoadBalancer Elastic IPs | | `METAL_METRO_NAME` | `metro` | Service-specific annotation, else error | +| Facility in which to create LoadBalancer Elastic IPs, only if Metro is not set | | `METAL_FACILITY_NAME` | `facility` | Service-specific annotation, else metro | +| Base URL to Equinix API | | | `base-url` | Official Equinix Metal API | +| Load balancer setting | | `METAL_LOAD_BALANCER` | `loadbalancer` | none | +| BGP ASN for cluster nodes when enabling BGP on the project; if the project **already** has BGP enabled, will use the existing BGP local ASN from the project | | `METAL_LOCAL_ASN` | `localASN` | `65000` | +| BGP passphrase to use when enabling BGP on the project; if the project **already** has BGP enabled, will use the existing BGP pass from the project | | `METAL_BGP_PASS` | `bgpPass` | `""` | +| Kubernetes annotation to set node's BGP ASN, `{{n}}` replaced with ordinal index of peer | | `METAL_ANNOTATION_LOCAL_ASN` | `annotationLocalASN` | `"metal.equinix.com/bgp-peers-{{n}}-node-asn"` | +| Kubernetes annotation to set BGP peer's ASN, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_PEER_ASN` | `annotationPeerASN` | `"metal.equinix.com/bgp-peers-{{n}}-peer-asn"` | +| Kubernetes annotation to set BGP peer's IPs, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_PEER_IP` | `annotationPeerIP` | `"metal.equinix.com/bgp-peers-{{n}}-peer-ip"` | +| Kubernetes annotation to set source IP for BGP peering, {{n}} replaced with ordinal index of peer | | `METAL_ANNOTATION_SRC_IP` | `annotationSrcIP` | `"metal.equinix.com/bgp-peers-{{n}}-src-ip"` | +| Kubernetes annotation to set BGP MD5 password, base64-encoded (see security warning below) | | `METAL_ANNOTATION_BGP_PASS` | `annotationBGPPass` | `"metal.equinix.com/bgp-peers-{{n}}-bgp-pass"` | +| Kubernetes annotation to set the CIDR for the network range of the private address | | `METAL_ANNOTATION_NETWORK_IPV4_PRIVATE` | `annotationNetworkIPv4Private` | `metal.equinix.com/network-4-private` | +| Kubernetes Service annotation to set EIP metro | | `METAL_ANNOTATION_EIP_METRO` | `annotationEIPMetro` | `"metal.equinix.com/eip-metro"` | +| Kubernetes Service annotation to set EIP facility | | `METAL_ANNOTATION_EIP_FACILITY` | `annotationEIPFacility` | `"metal.equinix.com/eip-facility"` | +| Tag for control plane Elastic IP | | `METAL_EIP_TAG` | `eipTag` | No control plane Elastic IP | +| Kubernetes API server port for Elastic IP | | `METAL_API_SERVER_PORT` | `apiServerPort` | Same as `kube-apiserver` on control plane nodes, same as `0` | +| Filter for cluster nodes on which to enable BGP | | `METAL_BGP_NODE_SELECTOR` | `bgpNodeSelector` | All nodes | +| Use host IP for Control Plane endpoint health checks | | `METAL_EIP_HEALTH_CHECK_USE_HOST_IP` | `eipHealthCheckUseHostIP` | false | Security Warning Including your project's BGP password, even base64-encoded, may have security implications. Because Equinix Metal @@ -248,24 +248,55 @@ to system pods that have reasonable need to access them. The Kubernetes CCM for Equinix Metal deploys as a `Deployment` into your cluster with a replica of `1`. It provides the following services: -* lists and retrieves instances by ID, returning Equinix Metal servers -* manages load balancers +- lists and retrieves instances by ID, returning Equinix Metal servers +- manages load balancers -### Load Balancers +### Service Load Balancers -Equinix Metal does not offer managed load balancers like [AWS ELB](https://aws.amazon.com/elasticloadbalancing/) +~~Equinix Metal does not offer managed load balancers like [AWS ELB](https://aws.amazon.com/elasticloadbalancing/) or [GCP Load Balancers](https://cloud.google.com/load-balancing/). Instead, if configured to do so, -Equinix Metal CCM will interface with and configure external bare-metal loadbalancers. +Equinix Metal CCM will interface with and configure external bare-metal loadbalancers.~~ When a load balancer is enabled, the CCM does the following: 1. Enable BGP for the project 1. Enable BGP on each node as it comes up 1. Sets ASNs based on configuration or default -1. For each `Service` of `type=LoadBalancer`: - * If you have specified a load balancer IP on `Service.Spec.LoadBalancerIP` (bring your own IP, or BYOIP), do nothing - * If you have not specified a load balancer IP on `Service.Spec.LoadBalancerIP`, get an Equinix Metal Elastic IP and set it on `Service.Spec.LoadBalancerIP`, see below -1. Pass control to the specific load balancer implementation +1. Unless you are using EMLB, for each `Service` of `type=LoadBalancer`: + - If you have specified a load balancer IP on `Service.Spec.LoadBalancerIP` (bring your own IP, or BYOIP), do nothing + - If you have not specified a load balancer IP on `Service.Spec.LoadBalancerIP`, get an Equinix Metal Elastic IP and set it on `Service.Spec.LoadBalancerIP`, see below +1. Then, depending on the load balancer implementation: + + 1. If MetalLB, set up the CRDs necessary for MetalLB to create the load balancer and configure it on the relevant nodes + 1. If EMLB + + 1. Ensure preferred load balancer metro is configured (make it part of the schema?) + 1. Create an origin pool per service + 1. Pool Name, based on service + 1. Node IP + 1. Port (the nodeport) - chris's guess + 1. Reuse or Create load balancer + 1. Check annotation for load balancer name/metro + 1. Already exists? + 1. Yes + 1. Get the load balancer + 1. Make sure public port is free + 1. If not, do we enable creating new load balancers? $$$$$ + 1. Specify Origin Pool created above + 1. No + 1. Create it + 1. Listener Port (public port requested in Service.Spec.Port) + 1. Specify Origin Pool created above + 1. Delete Load Balancer + 1. Remove origin pool + 1. Remove listener Port + 1. If no more listeners, delete load balancer ? + 1. Update Load Balancer + 1. Get Origin Pool + 1. Diff + 1. Update if needed + + 1. If Kube-vip or empty, do nothing #### Service Load Balancer IP @@ -303,7 +334,7 @@ CCM will detect that `loadBalancerIP` already was set and not try to create a ne ##### Equinix EIP -If the `Service.Spec.LoadBalancerIP` was *not* set, then CCM will use the Equinix Metal API to request a new, +If the `Service.Spec.LoadBalancerIP` was _not_ set, then CCM will use the Equinix Metal API to request a new, metro- or facility-specific Elastic IP and set it to `Service.Spec.LoadBalancerIP`. The CCM needs to determine where to request the EIP. It does not attempt to figure out where the nodes are, as that can change over time, @@ -339,24 +370,24 @@ Loadbalancing is enabled as follows. The value of the loadbalancing configuration is `:///` where: -* `` is the named supported type, of one of those listed below -* `` is any additional detail needed to configure the implementation, details in the description below +- `` is the named supported type, of one of those listed below +- `` is any additional detail needed to configure the implementation, details in the description below For loadbalancing for Kubernetes `Service` of `type=LoadBalancer`, the following implementations are supported: -* [kube-vip](#kube-vip) -* [MetalLB](#metallb) -* [empty](#empty) +- [kube-vip](#kube-vip) +- [MetalLB](#metallb) +- [empty](#empty) CCM does **not** deploy _any_ load balancers for you. It limits itself to managing the Equinix Metal-specific API calls to support a load balancer, and providing configuration for supported load balancers. ##### kube-vip -**Supported Versions**: +**Supported Versions**: -* Equinix Metal CCM version < v3.3.0 supports kube-vip version < v0.5.11 -* Equinix Metal CCM version >= v3.3.0 supports kube-vip version >= v0.5.11 +- Equinix Metal CCM version < v3.3.0 supports kube-vip version < v0.5.11 +- Equinix Metal CCM version >= v3.3.0 supports kube-vip version >= v0.5.11 When the [kube-vip](https://kube-vip.io) option is enabled, for user-deployed Kubernetes `Service` of `type=LoadBalancer`, the Equinix Metal CCM enables BGP on the project and nodes, assigns an EIP for each such @@ -369,22 +400,22 @@ To enable it, set the configuration `METAL_LOAD_BALANCER` or config `loadbalance kube-vip:// ``` -Directions on using configuring kube-vip in this method are available at the kube-vip [site](https://kube-vip.io/hybrid/daemonset/#equinix-metal-overview-(using-the-%5Bequinix-cloud-provider-equinix-metal%5D(https://github.com/equinix/cloud-provider-equinix-metal))) +Directions on using configuring kube-vip in this method are available at the kube-vip [site]() If `kube-vip` management is enabled, then CCM does the following. 1. Enable BGP on the Equinix Metal project 1. For each node currently in the cluster or added: - * retrieve the node's Equinix Metal ID via the node provider ID - * retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP - * add the information to appropriate annotations on the node + - retrieve the node's Equinix Metal ID via the node provider ID + - retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP + - add the information to appropriate annotations on the node 1. For each service of `type=LoadBalancer` currently in the cluster or added: - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core) - * if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core) + - if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. 1. For each service of `type=LoadBalancer` deleted from the cluster: - * find the Elastic IP address from the service spec and remove it - * delete the Elastic IP reservation from Equinix Metal + - find the Elastic IP address from the service spec and remove it + - delete the Elastic IP reservation from Equinix Metal ##### MetalLB @@ -396,7 +427,7 @@ requiring an additional managed service (or hop). BGP route advertisements enabl to route traffic for your services at the Elastic IP to the correct host. **NOTE:** MetalLB 0.13.2+ [uses CRs for configuration](https://metallb.universe.tf/release-notes/#version-0-13-2), and no longer uses a ConfigMap. -Currently, the CCM defaults to using a ConfigMap for backwards compatibility. In a future release, the CCM will default to using CRDs with MetalLB. +Currently, the CCM defaults to using a ConfigMap for backwards compatibility. In a future release, the CCM will default to using CRDs with MetalLB. To configure the CCM to integrate with MetalLB <= v0.12.1, follow the instructions in [MetalLB from v0.11.0 to v0.12.1](#metallb-from-v0110-to-v0121). @@ -411,7 +442,7 @@ that are specifically structured to be ignored by metallb. For example: ```yaml - node-selectors: +node-selectors: - match-labels: kubernetes.io/hostname: dc-worker-1 - match-labels: @@ -441,13 +472,13 @@ metallb://// For example: -* `metallb:///metallb-system/config` - enable `MetalLB` management and update the configmap `config` in the namespace `metallb-system` -* `metallb:///foonamespace/myconfig` - - enable `MetalLB` management and update the configmap `myconfig` in the namespace `foonamespae` -* `metallb:///` - enable `MetalLB` management and update the default configmap, i.e. `config` in the namespace `metallb-system` +- `metallb:///metallb-system/config` - enable `MetalLB` management and update the configmap `config` in the namespace `metallb-system` +- `metallb:///foonamespace/myconfig` - - enable `MetalLB` management and update the configmap `myconfig` in the namespace `foonamespae` +- `metallb:///` - enable `MetalLB` management and update the default configmap, i.e. `config` in the namespace `metallb-system` -Notice the **three* slashes. In the URL, the namespace and the configmap are in the path. +Notice the \*_three_ slashes. In the URL, the namespace and the configmap are in the path. -By default, the CCM configures MetalLB using a ConfigMap. ConfigMap configuration only works with MetalLB <= v0.12.1. For forward compatibility, you may optionally append `?crdConfiguration=false` to the configuration string in order to explicitly tell the CCM to use a ConfigMap to configure MetalLB. In a future release, the CCM will default to using CRDs with MetalLB. +By default, the CCM configures MetalLB using a ConfigMap. ConfigMap configuration only works with MetalLB <= v0.12.1. For forward compatibility, you may optionally append `?crdConfiguration=false` to the configuration string in order to explicitly tell the CCM to use a ConfigMap to configure MetalLB. In a future release, the CCM will default to using CRDs with MetalLB. When enabled, CCM controls the loadbalancer by updating the provided `ConfigMap`. @@ -457,26 +488,26 @@ If `MetalLB` management is enabled, then CCM does the following. 1. If the `ConfigMap` does not exist, do the rest of the behaviours, but do not update the `ConfigMap` 1. Enable BGP on the Equinix Metal project 1. For each node currently in the cluster or added: - * retrieve the node's Equinix Metal ID via the node provider ID - * retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP - * add them to the metallb `ConfigMap` with a kubernetes selector ensuring that the peer is only for this node + - retrieve the node's Equinix Metal ID via the node provider ID + - retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP + - add them to the metallb `ConfigMap` with a kubernetes selector ensuring that the peer is only for this node 1. For each node deleted from the cluster: - * remove the node from the MetalLB `ConfigMap` + - remove the node from the MetalLB `ConfigMap` 1. For each service of `type=LoadBalancer` currently in the cluster or added: - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#servicespec-v1-core) and ensure it is in the pools of the MetalLB `ConfigMap` with `auto-assign: false` - * if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec, and ensure is in the pools of the metallb `ConfigMap` with `auto-assign: false`; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#servicespec-v1-core) and ensure it is in the pools of the MetalLB `ConfigMap` with `auto-assign: false` + - if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec, and ensure is in the pools of the metallb `ConfigMap` with `auto-assign: false`; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. 1. For each service of `type=LoadBalancer` deleted from the cluster: - * find the Elastic IP address from the service spec and remove it - * remove the IP from the `ConfigMap` - * delete the Elastic IP reservation from Equinix Metal + - find the Elastic IP address from the service spec and remove it + - remove the IP from the `ConfigMap` + - delete the Elastic IP reservation from Equinix Metal CCM itself does **not** deploy the load-balancer or any part of it, including the `ConfigMap`. It only modifies an existing `ConfigMap`. This can be deployed by the administrator separately, using the manifest provided in the releases page, or in any other manner. In order to instruct metallb which IPs to announce and from where, CCM takes direct responsibility for managing the -metallb `ConfigMap`. As described above, this is normally at `metallb-system/config`. +metallb `ConfigMap`. As described above, this is normally at `metallb-system/config`. You **should not** attempt to modify this `ConfigMap` separately, as CCM will modify it with each loop. Modifying it separately is likely to break metallb's functioning. @@ -489,42 +520,42 @@ To enable the CCM to use MetalLB v0.13.2+, you must set the configuration `METAL metallb:///?crdConfiguration=true ``` -Note that the `?crdConfiguration=true` is _required_ in order for the CCM to correctly configure MetalLB v0.13.2+ via CRDs instead of using a ConfigMap. Currently, the CCM defaults to using a ConfigMap for backwards compatibility. In a future release, the CCM will default to using CRDs with MetalLB. +Note that the `?crdConfiguration=true` is _required_ in order for the CCM to correctly configure MetalLB v0.13.2+ via CRDs instead of using a ConfigMap. Currently, the CCM defaults to using a ConfigMap for backwards compatibility. In a future release, the CCM will default to using CRDs with MetalLB. For example: -* `metallb:///metallb-system?crdConfiguration=true` - enable `MetalLB` management and update configuration in the namespace `metallb-system` (default) -* `metallb:///foonamespace?crdConfiguration=true` - enable `MetalLB` management and update configuration in the namespace `metallb-system` -* `metallb:///?crdConfiguration=true` - enable `MetalLB` management and update configuration in the default namespace `metallb-system` +- `metallb:///metallb-system?crdConfiguration=true` - enable `MetalLB` management and update configuration in the namespace `metallb-system` (default) +- `metallb:///foonamespace?crdConfiguration=true` - enable `MetalLB` management and update configuration in the namespace `metallb-system` +- `metallb:///?crdConfiguration=true` - enable `MetalLB` management and update configuration in the default namespace `metallb-system` -Notice the **three* slashes. In the URL, the namespace are in the path. +Notice the \*_three_ slashes. In the URL, the namespace are in the path. If `MetalLB` management is enabled, then CCM does the following. 1. Get the appropriate namespace, based on the rules above. 1. Enable BGP on the Equinix Metal project 1. For each node currently in the cluster or added: - * retrieve the node's Equinix Metal ID via the node provider ID - * retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP - * create a `bgpeers.metallb.io` for each peer IP with a kubernetes selector ensuring that those BGPPeers are only for this node + - retrieve the node's Equinix Metal ID via the node provider ID + - retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP + - create a `bgpeers.metallb.io` for each peer IP with a kubernetes selector ensuring that those BGPPeers are only for this node 1. For each node deleted from the cluster: - * delete the affiliated BGPeers. + - delete the affiliated BGPeers. 1. For each service of `type=LoadBalancer` currently in the cluster or added: - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#servicespec-v1-core) and ensure there is an `ipaddresspools.metallb.io` with `auto-assign: false`, and there is an elegible `bgpadvertisement.metallb.io`. If no bgpadvertisement exists with the appropriate tag ("cloud-provider":"equinix-metal"), a default bgpadvertisement "equinix-metal-bgp-adv" with the ipaddresspool name in the ipAddressPools spec will be created. - * if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec, and ensure there is an `ipaddresspools.metallb.io` with `auto-assign: false`, and there is an elegible `bgpadvertisement.metallb.io`. If no bgpadvertisement exists with the appropriate tag ("cloud-provider":"equinix-metal"), a default bgpadvertisement "equinix-metal-bgp-adv" with the ipaddresspool name in the ipAddressPools spec will be created; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#servicespec-v1-core) and ensure there is an `ipaddresspools.metallb.io` with `auto-assign: false`, and there is an elegible `bgpadvertisement.metallb.io`. If no bgpadvertisement exists with the appropriate tag ("cloud-provider":"equinix-metal"), a default bgpadvertisement "equinix-metal-bgp-adv" with the ipaddresspool name in the ipAddressPools spec will be created. + - if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec, and ensure there is an `ipaddresspools.metallb.io` with `auto-assign: false`, and there is an elegible `bgpadvertisement.metallb.io`. If no bgpadvertisement exists with the appropriate tag ("cloud-provider":"equinix-metal"), a default bgpadvertisement "equinix-metal-bgp-adv" with the ipaddresspool name in the ipAddressPools spec will be created; see [Equinix EIP][Equinix EIP] to control in which metro or facility the EIP will be created. 1. For each service of `type=LoadBalancer` deleted from the cluster: - * find the Elastic IP address from the service spec and remove it - * remove the affiliated `ipaddresspools.metallb.io` - * If there is no other service, delete all CCM managed `bgpeers` and the default `bgpadvertisement` - * delete the Elastic IP reservation from Equinix Metal + - find the Elastic IP address from the service spec and remove it + - remove the affiliated `ipaddresspools.metallb.io` + - If there is no other service, delete all CCM managed `bgpeers` and the default `bgpadvertisement` + - delete the Elastic IP reservation from Equinix Metal **NOTE:** (IP Address sharing)[https://metallb.universe.tf/usage/#ip-address-sharing] is not yet supported in Cloud Provider Equinix Metal. CCM itself does **not** install/deploy the load-balancer and it may exists before enable it. This can be deployed by the administrator separately, using the manifest provided in the releases page, or in any other manner. Not having metallb installed but enabled in the CCM configuration will end up allowing you to continue deploying kubernetes services, but the external ip assignment will remain pending, making it useless. In order to instruct metallb which IPs to announce and from where, CCM takes direct responsibility for managing the -metallb configuration. As described above, this is normally at `metallb-system`. Users can create and manage their own `bgpadvertisements.metallb.io` resources for advanced configuration, but they must have the appropriate tag ("cloud-provider":"equinix-metal") to prevent the CCM from creating a default bgpadvertisement. +metallb configuration. As described above, this is normally at `metallb-system`. Users can create and manage their own `bgpadvertisements.metallb.io` resources for advanced configuration, but they must have the appropriate tag ("cloud-provider":"equinix-metal") to prevent the CCM from creating a default bgpadvertisement. You **should not** attempt to modify metallb resources created by the CCM separately, as CCM will modify it with each loop. Modifying it separately is likely to break metallb's functioning. @@ -546,25 +577,25 @@ If `empty` management is enabled, then CCM does the following. 1. Enable BGP on the Equinix Metal project 1. For each node currently in the cluster or added: - * retrieve the node's Equinix Metal ID via the node provider ID - * retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP - * add the information to appropriate annotations on the node + - retrieve the node's Equinix Metal ID via the node provider ID + - retrieve the device's BGP configuration: node ASN, peer ASN, peer IPs, source IP + - add the information to appropriate annotations on the node 1. For each service of `type=LoadBalancer` currently in the cluster or added: - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore - * if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core) - * if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` already has that IP address affiliated with it, it is ready; ignore + - if an Elastic IP address reservation with the appropriate tags exists, and the `Service` does not have that IP affiliated with it, add it to the [service spec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#servicespec-v1-core) + - if an Elastic IP address reservation with the appropriate tags does not exist, create it and add it to the services spec 1. For each service of `type=LoadBalancer` deleted from the cluster: - * find the Elastic IP address from the service spec and remove it - * delete the Elastic IP reservation from Equinix Metal + - find the Elastic IP address from the service spec and remove it + - delete the Elastic IP reservation from Equinix Metal ### Language In order to ease understanding, we use several different terms for an IP address: -* Requested: A dedicated `/32` IP address has been requested for the service from Equinix Metal. It may be returned immediately, or it may need to wait for Equinix Metal intervention. -* Reserved: A dedicated `/32` IP address has been reserved for the service from Equinix Metal. -* Assigned: The dedicated IP address has been marked on the service as `Service.Spec.LoadBalancerIP` as assigned. -* Mapped: The dedicated IP address has been added to the metallb `ConfigMap` as available. +- Requested: A dedicated `/32` IP address has been requested for the service from Equinix Metal. It may be returned immediately, or it may need to wait for Equinix Metal intervention. +- Reserved: A dedicated `/32` IP address has been reserved for the service from Equinix Metal. +- Assigned: The dedicated IP address has been marked on the service as `Service.Spec.LoadBalancerIP` as assigned. +- Mapped: The dedicated IP address has been added to the metallb `ConfigMap` as available. From Equinix Metal's perspective, the IP reservation is either Requested or Reserved, but not both. For the load balancer to work, the IP address needs to be all of: Reserved, Assigned, Mapped. @@ -576,9 +607,9 @@ ability to assign that EIP to different devices. You have several options for control plane load-balancing: -* CCM managed -* kube-vip managed -* No control plane load-balancing (or at least, none known to CCM) +- CCM managed +- kube-vip managed +- No control plane load-balancing (or at least, none known to CCM) ### CCM Managed @@ -591,10 +622,10 @@ To enable CCM to manage the control plane EIP: 1. Create an Elastic IP, using the Equinix Metal API, Web UI or CLI 1. Put an arbitrary but unique tag on the EIP 1. When starting the CCM - * set the [configuration][Configuration] for the control plane EIP tag, e.g. env var `METAL_EIP_TAG=`, where `` is whatever tag you set on the EIP - * (optional) set the port that the EIP should listen on; by default, or when set to `0`, it will use the same port as the `kube-apiserver` on the control plane nodes. This port can also be specified with `METAL_API_SERVER_PORT=.` - * (optional) set the [configuration][Configuration] for using the host IP for control plane endpoint health checks. This is - needed when the EIP is configured as an loopback IP address, such as the case with [CAPP](https://github.com/kubernetes-sigs/cluster-api-provider-packet) + - set the [configuration][Configuration] for the control plane EIP tag, e.g. env var `METAL_EIP_TAG=`, where `` is whatever tag you set on the EIP + - (optional) set the port that the EIP should listen on; by default, or when set to `0`, it will use the same port as the `kube-apiserver` on the control plane nodes. This port can also be specified with `METAL_API_SERVER_PORT=.` + - (optional) set the [configuration][Configuration] for using the host IP for control plane endpoint health checks. This is + needed when the EIP is configured as an loopback IP address, such as the case with [CAPP](https://github.com/kubernetes-sigs/cluster-api-provider-packet) In [CAPP](https://github.com/kubernetes-sigs/cluster-api-provider-packet) we create one for every cluster for example. Equinix Metal does not provide an as a @@ -636,23 +667,23 @@ creates an `Endpoints` structure that includes all of the functioning control pl nodes. The CCM does the following on each loop: 1. Reads the Kubernetes-created `default/kubernetes` service to discover: - * what port `kube-apiserver` is listening on from `targetPort` - * all of the endpoints, i.e. control plane nodes where `kube-apiserver` is running + - what port `kube-apiserver` is listening on from `targetPort` + - all of the endpoints, i.e. control plane nodes where `kube-apiserver` is running 1. Creates a service named `kube-system/cloud-provider-equinix-metal-kubernetes-external` with the following settings: - * `type=LoadBalancer` - * `spec.loadBalancerIP=` - * `status.loadBalancer.ingress[0].ip=` - * `metadata.annotations["metallb.universe.tf/address-pool"]=disabled-metallb-do-not-use-any-address-pool` - * `spec.ports[0].targetPort=` - * `spec.ports[0].port=` + - `type=LoadBalancer` + - `spec.loadBalancerIP=` + - `status.loadBalancer.ingress[0].ip=` + - `metadata.annotations["metallb.universe.tf/address-pool"]=disabled-metallb-do-not-use-any-address-pool` + - `spec.ports[0].targetPort=` + - `spec.ports[0].port=` 1. Updates the service `kube-system/cloud-provider-equinix-metal-kubernetes-external` to have endpoints identical to those in `default/kubernetes` This has the following effect: -* the annotation prevents metallb from trying to manage the IP -* the name prevents CCM from passing it to the loadbalancer provider address mapping, thus preventing any of them from managing it -* the `spec.loadBalancerIP` and `status.loadBalancer.ingress[0].ip` cause kube-proxy to set up routes on all of the nodes -* the endpoints cause the traffic to be routed to the control plane nodes +- the annotation prevents metallb from trying to manage the IP +- the name prevents CCM from passing it to the loadbalancer provider address mapping, thus preventing any of them from managing it +- the `spec.loadBalancerIP` and `status.loadBalancer.ingress[0].ip` cause kube-proxy to set up routes on all of the nodes +- the endpoints cause the traffic to be routed to the control plane nodes Note that we _wanted_ to just set `externalIPs` on the original `default/kubernetes`, but that would prevent traffic from being routed to it from the control nodes, due to iptables rules. LoadBalancer types allow local traffic. @@ -672,21 +703,21 @@ The CCM does not maintain its own control loop, instead relying on the services On startup, the CCM: 1. Implements the [cloud-provider interface](https://pkg.go.dev/k8s.io/cloud-provider#Interface), providing primarily the following API calls: - * `Initialize()` - * `InstancesV2()` - * `LoadBalancer()` + - `Initialize()` + - `InstancesV2()` + - `LoadBalancer()` 1. In `Initialize`: 1. If BGP is configured, enable BGP on the project 1. If EIP control plane management is enabled, create an informer for `Service`, `Node` and `Endpoints`, updating the control plane EIP as needed. The CCM then relies on the cloud-provider control loop to call it: -* whenever a `Node` is added, to get node metadata -* whenever a `Service` of `type=LoadBalancer` is added, removed or updated -* if EIP control plane management is enabled, via shared informers: - * whenever a control plane `Node` is added, removed or updated - * whenever the `default/kubernetes` service is added or updated - * whenever the endpoints behind the `default/kubernetes` service are added, updated or removed +- whenever a `Node` is added, to get node metadata +- whenever a `Service` of `type=LoadBalancer` is added, removed or updated +- if EIP control plane management is enabled, via shared informers: + - whenever a control plane `Node` is added, removed or updated + - whenever the `default/kubernetes` service is added or updated + - whenever the endpoints behind the `default/kubernetes` service are added, updated or removed Further, it relies on the `resync` property of the above to ensure it always is up to date, and did not miss any events. @@ -695,8 +726,8 @@ Further, it relies on the `resync` property of the above to ensure it always is If a loadbalancer is enabled, the CCM enables BGP for the project and enables it by default on all nodes as they come up. It sets the ASNs as follows: -* Node, a.k.a. local, ASN: `65000` -* Peer Router ASN: `65530` +- Node, a.k.a. local, ASN: `65000` +- Peer Router ASN: `65530` These are the settings per Equinix Metal's BGP config, see [here](https://github.com/packet-labs/kubernetes-bgp). It is _not_ recommended to override them. However, you can do so, using the options in [Configuration][Configuration]. @@ -708,35 +739,35 @@ Value for node selector should be a valid Kubernetes label selector (e.g. key1=v The Equinix Metal CCM sets Kubernetes annotations on each cluster node. -* Node, or local, ASN, default annotation `metal.equinix.com/bgp-peers-{{n}}-node-asn` -* Peer ASN, default annotation `metal.equinix.com/bgp-peers-{{n}}-peer-asn` -* Peer IP, default annotation `metal.equinix.com/bgp-peers-{{n}}-peer-ip` -* Source IP to use when communicating with peer, default annotation `metal.equinix.com/bgp-peers-{{n}}-src-ip` -* BGP password for peer, default annotation `metal.equinix.com/bgp-peers-{{n}}-bgp-pass` -* CIDR of the private network range in the project which this node is part of, default annotation `metal.equinix.com/network-4-private` +- Node, or local, ASN, default annotation `metal.equinix.com/bgp-peers-{{n}}-node-asn` +- Peer ASN, default annotation `metal.equinix.com/bgp-peers-{{n}}-peer-asn` +- Peer IP, default annotation `metal.equinix.com/bgp-peers-{{n}}-peer-ip` +- Source IP to use when communicating with peer, default annotation `metal.equinix.com/bgp-peers-{{n}}-src-ip` +- BGP password for peer, default annotation `metal.equinix.com/bgp-peers-{{n}}-bgp-pass` +- CIDR of the private network range in the project which this node is part of, default annotation `metal.equinix.com/network-4-private` These annotation names can be overridden, if you so choose, using the options in [Configuration][Configuration]. Note that the annotations for BGP peering are a _pattern_. There is one annotation per data point per peer, following the pattern `metal.equinix.com/bgp-peers-{{n}}-`, where: -* `{{n}}` is the number of the peer, **always** starting with `0` -* `` is the relevant information, such as `node-asn` or `peer-ip` +- `{{n}}` is the number of the peer, **always** starting with `0` +- `` is the relevant information, such as `node-asn` or `peer-ip` For example: -* `metal.equinix.com/bgp-peers-0-peer-asn` - ASN of peer 0 -* `metal.equinix.com/bgp-peers-1-peer-asn` - ASN of peer 1 -* `metal.equinix.com/bgp-peers-0-peer-ip` - IP of peer 0 -* `metal.equinix.com/bgp-peers-1-peer-ip` - IP of peer 1 +- `metal.equinix.com/bgp-peers-0-peer-asn` - ASN of peer 0 +- `metal.equinix.com/bgp-peers-1-peer-asn` - ASN of peer 1 +- `metal.equinix.com/bgp-peers-0-peer-ip` - IP of peer 0 +- `metal.equinix.com/bgp-peers-1-peer-ip` - IP of peer 1 ## Elastic IP Configuration If a loadbalancer is enabled, CCM creates an Equinix Metal Elastic IP (EIP) reservation for each `Service` of `type=LoadBalancer`. It tags the Reservation with the following tags: -* `usage="cloud-provider-equinix-metal-auto"` -* `service=""` where `` is the sha256 hash of `/`. We do this so that the name of the service does not leak out to Equinix Metal itself. -* `cluster=` where `` is the UID of the immutable `kube-system` namespace. We do this so that if someone runs two clusters in the same project, and there is one `Service` in each cluster with the same namespace and name, then the two EIPs will not conflict. +- `usage="cloud-provider-equinix-metal-auto"` +- `service=""` where `` is the sha256 hash of `/`. We do this so that the name of the service does not leak out to Equinix Metal itself. +- `cluster=` where `` is the UID of the immutable `kube-system` namespace. We do this so that if someone runs two clusters in the same project, and there is one `Service` in each cluster with the same namespace and name, then the two EIPs will not conflict. IP addresses always are created `/32`. From 2fa2f7fc99ed2c856dd32c69b5eda562b2af6b2b Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:09:46 -0500 Subject: [PATCH 04/48] chore: tabs to spaces Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- Tiltfile | 38 ++++++++++++++++++++++++++ cloud-sa.json | 4 +++ dev/web-updated.yaml | 36 +++++++++++++++++++++++++ dev/web-updated2.yaml | 36 +++++++++++++++++++++++++ dev/web.yaml | 36 +++++++++++++++++++++++++ dev/words.sql | 55 ++++++++++++++++++++++++++++++++++++++ dev/wordsmith-ingress.yaml | 34 +++++++++++++++++++++++ 7 files changed, 239 insertions(+) create mode 100644 Tiltfile create mode 100644 cloud-sa.json create mode 100644 dev/web-updated.yaml create mode 100644 dev/web-updated2.yaml create mode 100644 dev/web.yaml create mode 100644 dev/words.sql create mode 100644 dev/wordsmith-ingress.yaml diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 00000000..27d0d76c --- /dev/null +++ b/Tiltfile @@ -0,0 +1,38 @@ +# Specify the registry you wish to store the image in +registry = 'sabin1001' + +# List the k8s context you wish to run this in +allow_k8s_contexts('capi-quicktest-admin@capi-quicktest') + +# Specify docker registry you wish to store image in +docker_build(registry + '/cloud-provider-equinix-metal', + context='.', + dockerfile='./Dockerfile', + ignore=['cloud-sa.json','dev/'], +) + +# read in the yaml file and replace the image name with the one we built +deployment = read_yaml_stream('deploy/template/deployment.yaml') +deployment[0]['spec']['template']['spec']['containers'][0]['image'] = registry + '/cloud-provider-equinix-metal' +deployment[0]['spec']['template']['spec']['containers'][0]['env']=[] +deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_METRO_NAME","value":"da"}) +deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_LOAD_BALANCER","value":"emlb:///da"}) +k8s_yaml(encode_yaml_stream(deployment)) +k8s_resource(workload='cloud-provider-equinix-metal',objects=['cloud-controller-manager:ServiceAccount:kube-system','system\\:cloud-controller-manager:ClusterRole:default','system\\:cloud-controller-manager:ClusterRoleBinding:default']) +k8s_resource(new_name='metal-cloud-config',objects=['metal-cloud-config:Secret:kube-system']) + +# Load the secret extension +load('ext://secret', 'secret_create_generic') + +# Create the cloud-provider-equinix-metal secret based on the contents of the +# file named cloud-sa.json put the apiKey and projectID in it +# The file should look like this: +# { +# "apiKey":"YOUR_API_KEY", +# "projectID":"YOUR_PROJECT_ID" +# } +secret_create_generic( + 'metal-cloud-config', + 'kube-system', + from_file='cloud-sa.json=./cloud-sa.json' +) \ No newline at end of file diff --git a/cloud-sa.json b/cloud-sa.json new file mode 100644 index 00000000..6b923941 --- /dev/null +++ b/cloud-sa.json @@ -0,0 +1,4 @@ +{ + "apiKey":"HDNdRFiLyhv54G6YajToD34X2d5XvGmq", + "projectID": "587aeda9-ee51-447d-87ce-068c57846508" +} \ No newline at end of file diff --git a/dev/web-updated.yaml b/dev/web-updated.yaml new file mode 100644 index 00000000..9cf2e94c --- /dev/null +++ b/dev/web-updated.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + app: web +spec: + ports: + - port: 8100 + targetPort: 80 + name: web + selector: + app: web + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: dockersamples/wordsmith-web + ports: + - containerPort: 80 + name: web diff --git a/dev/web-updated2.yaml b/dev/web-updated2.yaml new file mode 100644 index 00000000..c28a0224 --- /dev/null +++ b/dev/web-updated2.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: webby + labels: + app: web +spec: + ports: + - port: 8200 + targetPort: 80 + name: web + selector: + app: web + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: dockersamples/wordsmith-web + ports: + - containerPort: 80 + name: web diff --git a/dev/web.yaml b/dev/web.yaml new file mode 100644 index 00000000..ecad38be --- /dev/null +++ b/dev/web.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + app: web +spec: + ports: + - port: 8080 + targetPort: 80 + name: web + selector: + app: web + type: LoadBalancer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + containers: + - name: web + image: dockersamples/wordsmith-web + ports: + - containerPort: 80 + name: web diff --git a/dev/words.sql b/dev/words.sql new file mode 100644 index 00000000..ec13172c --- /dev/null +++ b/dev/words.sql @@ -0,0 +1,55 @@ +CREATE TABLE nouns (word TEXT NOT NULL); +CREATE TABLE verbs (word TEXT NOT NULL); +CREATE TABLE adjectives (word TEXT NOT NULL); + +INSERT INTO nouns(word) VALUES + ('cloud'), + ('elephant'), + ('gø language'), + ('laptøp'), + ('cøntainer'), + ('micrø-service'), + ('turtle'), + ('whale'), + ('gøpher'), + ('møby døck'), + ('server'), + ('bicycle'), + ('viking'), + ('mermaid'), + ('fjørd'), + ('legø'), + ('flødebolle'), + ('smørrebrød'); + +INSERT INTO verbs(word) VALUES + ('will drink'), + ('smashes'), + ('smøkes'), + ('eats'), + ('walks tøwards'), + ('løves'), + ('helps'), + ('pushes'), + ('debugs'), + ('invites'), + ('hides'), + ('will ship'); + +INSERT INTO adjectives(word) VALUES + ('the exquisite'), + ('a pink'), + ('the røtten'), + ('a red'), + ('the serverless'), + ('a brøken'), + ('a shiny'), + ('the pretty'), + ('the impressive'), + ('an awesøme'), + ('the famøus'), + ('a gigantic'), + ('the gløriøus'), + ('the nørdic'), + ('the welcøming'), + ('the deliciøus'); diff --git a/dev/wordsmith-ingress.yaml b/dev/wordsmith-ingress.yaml new file mode 100644 index 00000000..3ba7a95d --- /dev/null +++ b/dev/wordsmith-ingress.yaml @@ -0,0 +1,34 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minimal-ingress + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: websvc + port: + number: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: websvc + labels: + app: web +spec: + ports: + - port: 8080 + targetPort: 80 + name: web + selector: + app: web + type: ClusterIP From e3af08dd932e376a40649c1cbcf4e0472b909ff0 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:10:09 -0500 Subject: [PATCH 05/48] chore: formatting remove line Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index b07bfa6c..b72cb878 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -91,7 +91,7 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr klog.Info("loadbalancer implementation enabled: empty, bgp only") impl = empty.NewLB(k8sclient, lbconfig) case "emlb": - klog.Info("loadbalancer implementation enabled: empty, bgp only") + klog.Info("loadbalancer implementation enabled: emlb") impl = empty.NewLB(k8sclient, lbconfig) default: klog.Info("loadbalancer implementation disabled") From 248a36c1ae6d54ce2afa8e1ad80d243655799758 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 26 Sep 2023 17:16:54 -0500 Subject: [PATCH 06/48] feat: parse config for new emlb lbs Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 3 ++- metal/loadbalancers/emlb/emlb.go | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index b72cb878..e6f1ae91 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/emlb" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/empty" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/kubevip" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/metallb" @@ -92,7 +93,7 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr impl = empty.NewLB(k8sclient, lbconfig) case "emlb": klog.Info("loadbalancer implementation enabled: emlb") - impl = empty.NewLB(k8sclient, lbconfig) + impl = emlb.NewLB(k8sclient, lbconfig) default: klog.Info("loadbalancer implementation disabled") impl = nil diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 533c2ded..cdc834b9 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -10,13 +10,20 @@ import ( ) type LB struct { - client *lbaas.APIClient + client *lbaas.APIClient + loadBalancerLocation *lbaas.LoadBalancerLocation } func NewLB(k8sclient kubernetes.Interface, config string) *LB { + // Parse config for Equinix Metal Load Balancer + // An example config using Dallas as the location would look like + // The format is emlb:// + // it may have an extra slash at the beginning or end, so get rid of it + lb := &LB{} emlbConfig := lbaas.NewConfiguration() lb.client = lbaas.NewAPIClient(emlbConfig) + lb.loadBalancerLocation.Id = &config return lb } From 1aef0da114477dcb0750c6eb8cc402c535daba49 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Wed, 27 Sep 2023 13:56:08 -0500 Subject: [PATCH 07/48] exchange Metal API key for LB OAuth tokenbefore making LB API requests --- metal/loadbalancers/emlb/emlb.go | 36 ++++++++++++ .../emlb/metal_token_exchanger.go | 57 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 metal/loadbalancers/emlb/metal_token_exchanger.go diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index cdc834b9..7c45e6ef 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -28,10 +28,46 @@ func NewLB(k8sclient kubernetes.Interface, config string) *LB { } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { + tokenExchanger := &MetalTokenExchanger{ + metalAPIKey: "", // TODO pass this in somehow (maybe add it to the context?) + client: l.client.GetConfig().HTTPClient, + } + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + + lbCreateRequest := lbaas.LoadBalancerCreate{ + Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? + LocationId: "", // TODO In the first pass, this comes from the config string? Or an annotation + ProviderId: "", // TODO I have a working value for this (same as what the portal uses) but waiting on feedback from LBaaS team + } + + // TODO lb, resp, err := + _, _, err := l.client.ProjectsApi.CreateLoadBalancer(ctx, "TODO: project ID").LoadBalancerCreate(lbCreateRequest).Execute() + if err != nil { + return err + } + + // TODO create other resources + return nil } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { + tokenExchanger := &MetalTokenExchanger{ + metalAPIKey: "TODO", + client: l.client.GetConfig().HTTPClient, + } + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + + loadBalancerId := "TODO" + + // TODO delete other resources + + // TODO lb, resp, err := + _, err := l.client.LoadBalancersApi.DeleteLoadBalancer(ctx, loadBalancerId).Execute() + if err != nil { + return err + } + return nil } diff --git a/metal/loadbalancers/emlb/metal_token_exchanger.go b/metal/loadbalancers/emlb/metal_token_exchanger.go new file mode 100644 index 00000000..0157f470 --- /dev/null +++ b/metal/loadbalancers/emlb/metal_token_exchanger.go @@ -0,0 +1,57 @@ +package emlb + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "golang.org/x/oauth2" +) + +type MetalTokenExchanger struct { + metalAPIKey string + client *http.Client +} + +func (m *MetalTokenExchanger) Token() (*oauth2.Token, error) { + tokenExchangeURL := "https://iam.metalctrl.io/api-keys/exchange" + tokenExchangeRequest, err := http.NewRequest("POST", tokenExchangeURL, nil) + if err != nil { + return nil, err + } + tokenExchangeRequest.Header.Add("Authorization", fmt.Sprintf("Bearer %v", m.metalAPIKey)) + + resp, err := m.client.Do(tokenExchangeRequest) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("token exchange request failed with status %v, body %v", resp.StatusCode, body) + } + + token := oauth2.Token{} + err = json.Unmarshal(body, &token) + if err != nil { + fmt.Println(len(body)) + fmt.Println(token) + fmt.Println(err) + return nil, err + } + + expiresIn := token.Extra("expires_in") + if expiresIn != nil { + expiresInSeconds := expiresIn.(int) + token.Expiry = time.Now().Add(time.Second * time.Duration(expiresInSeconds)) + } + + return &token, nil +} From f2bbcf0559467969bed5785acd63e4ed5befc48a Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 28 Sep 2023 11:00:51 -0500 Subject: [PATCH 08/48] add map of known locations --- metal/loadbalancers/emlb/emlb.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 7c45e6ef..ac0e49e3 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -3,12 +3,22 @@ package emlb import ( "context" + "fmt" + "reflect" lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "k8s.io/client-go/kubernetes" ) +const ProviderID = "loadpvd-gOB_-byp5ebFo7A3LHv2B" + +var LBMetros = map[string]string{ + "da": "lctnloc--uxs0GLeAELHKV8GxO_AI", + "ny": "lctnloc-Vy-1Qpw31mPi6RJQwVf9A", + "sv": "lctnloc-H5rl2M2VL5dcFmdxhbEKx", +} + type LB struct { client *lbaas.APIClient loadBalancerLocation *lbaas.LoadBalancerLocation @@ -34,10 +44,16 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n } ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + metro := "da" // TODO get this from somewhere else + + locationId, ok := LBMetros[metro] + if !ok { + return fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).Keys()) + } lbCreateRequest := lbaas.LoadBalancerCreate{ Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? - LocationId: "", // TODO In the first pass, this comes from the config string? Or an annotation - ProviderId: "", // TODO I have a working value for this (same as what the portal uses) but waiting on feedback from LBaaS team + LocationId: locationId, + ProviderId: ProviderID, } // TODO lb, resp, err := From 1182dde3a9eca27abd6c4b8c858e59a7e1b8d46b Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 28 Sep 2023 12:18:24 -0500 Subject: [PATCH 09/48] scaffold out the LBaaS integration --- metal/loadbalancers/emlb/controller.go | 98 ++++++++++++++++++++++++++ metal/loadbalancers/emlb/emlb.go | 71 ++++++++----------- 2 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 metal/loadbalancers/emlb/controller.go diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go new file mode 100644 index 00000000..2bc3ff64 --- /dev/null +++ b/metal/loadbalancers/emlb/controller.go @@ -0,0 +1,98 @@ +package emlb + +import ( + "context" + "fmt" + "reflect" + + lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" +) + +const ProviderID = "loadpvd-gOB_-byp5ebFo7A3LHv2B" + +var LBMetros = map[string]string{ + "da": "lctnloc--uxs0GLeAELHKV8GxO_AI", + "ny": "lctnloc-Vy-1Qpw31mPi6RJQwVf9A", + "sv": "lctnloc-H5rl2M2VL5dcFmdxhbEKx", +} + +type controller struct { + client *lbaas.APIClient + tokenExchanger *MetalTokenExchanger +} + +func NewController(metalAPIKey string) *controller { + controller := &controller{} + emlbConfig := lbaas.NewConfiguration() + + controller.client = lbaas.NewAPIClient(emlbConfig) + controller.tokenExchanger = &MetalTokenExchanger{ + metalAPIKey: metalAPIKey, + client: controller.client.GetConfig().HTTPClient, + } + + return controller +} + +func (c *controller) createLoadBalancer(ctx context.Context, config map[string]string) (map[string]string, error) { + outputProperties := map[string]string{} + + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) + + metro := config["metro"] + + locationId, ok := LBMetros[metro] + if !ok { + return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).Keys()) + } + lbCreateRequest := lbaas.LoadBalancerCreate{ + Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? + LocationId: locationId, + ProviderId: ProviderID, + } + + // TODO lb, resp, err := + _, _, err := c.client.ProjectsApi.CreateLoadBalancer(ctx, "TODO: project ID").LoadBalancerCreate(lbCreateRequest).Execute() + if err != nil { + return nil, err + } + + // TODO create other resources + return outputProperties, nil +} + +func (c *controller) updateLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { + outputProperties := map[string]string{} + + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) + + // TODO delete other resources + + // TODO lb, resp, err := + _, err := c.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + if err != nil { + return nil, err + } + + return outputProperties, nil +} + +func (c *controller) deleteLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { + outputProperties := map[string]string{} + + tokenExchanger := &MetalTokenExchanger{ + metalAPIKey: "TODO", + client: c.client.GetConfig().HTTPClient, + } + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + + // TODO delete other resources + + // TODO lb, resp, err := + _, err := c.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + if err != nil { + return nil, err + } + + return outputProperties, nil +} diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index ac0e49e3..59878354 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -3,24 +3,14 @@ package emlb import ( "context" - "fmt" - "reflect" lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "k8s.io/client-go/kubernetes" ) -const ProviderID = "loadpvd-gOB_-byp5ebFo7A3LHv2B" - -var LBMetros = map[string]string{ - "da": "lctnloc--uxs0GLeAELHKV8GxO_AI", - "ny": "lctnloc-Vy-1Qpw31mPi6RJQwVf9A", - "sv": "lctnloc-H5rl2M2VL5dcFmdxhbEKx", -} - type LB struct { - client *lbaas.APIClient + controller *controller loadBalancerLocation *lbaas.LoadBalancerLocation } @@ -30,63 +20,60 @@ func NewLB(k8sclient kubernetes.Interface, config string) *LB { // The format is emlb:// // it may have an extra slash at the beginning or end, so get rid of it + metalAPIKey := "TODO" + lb := &LB{} - emlbConfig := lbaas.NewConfiguration() - lb.client = lbaas.NewAPIClient(emlbConfig) lb.loadBalancerLocation.Id = &config + + lb.controller = NewController(metalAPIKey) return lb } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { - tokenExchanger := &MetalTokenExchanger{ - metalAPIKey: "", // TODO pass this in somehow (maybe add it to the context?) - client: l.client.GetConfig().HTTPClient, - } - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + additionalProperties := map[string]string{} - metro := "da" // TODO get this from somewhere else + // 2. Create the infrastructure (what do we need to return here? lb name and/or ID? anything else?) + _, err := l.controller.createLoadBalancer(ctx, additionalProperties) - locationId, ok := LBMetros[metro] - if !ok { - return fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).Keys()) - } - lbCreateRequest := lbaas.LoadBalancerCreate{ - Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? - LocationId: locationId, - ProviderId: ProviderID, - } - - // TODO lb, resp, err := - _, _, err := l.client.ProjectsApi.CreateLoadBalancer(ctx, "TODO: project ID").LoadBalancerCreate(lbCreateRequest).Execute() if err != nil { return err } - // TODO create other resources - + // 3. Add the annotations return nil } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { - tokenExchanger := &MetalTokenExchanger{ - metalAPIKey: "TODO", - client: l.client.GetConfig().HTTPClient, - } - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) - + // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) loadBalancerId := "TODO" + additionalProperties := map[string]string{} - // TODO delete other resources + // 2. Delete the infrastructure (do we need to return anything here?) + _, err := l.controller.deleteLoadBalancer(ctx, loadBalancerId, additionalProperties) - // TODO lb, resp, err := - _, err := l.client.LoadBalancersApi.DeleteLoadBalancer(ctx, loadBalancerId).Execute() if err != nil { return err } + // 3. Remove the annotations + return nil } func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { + // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + loadBalancerId := "TODO" + additionalProperties := map[string]string{} + + // 2. Update infrastructure change (do we need to return anything here? or are all changes reflected by properties from [1]?) + _, err := l.controller.updateLoadBalancer(ctx, loadBalancerId, additionalProperties) + + if err != nil { + return err + } + + // 3. Update the annotations + return nil } From 15c1de1bd69bbcb588bd3c5d2c4079f10e4f0956 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 28 Sep 2023 14:40:53 -0500 Subject: [PATCH 10/48] add more detailed implementation outline in comments --- metal/loadbalancers/emlb/emlb.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 59878354..43ce5d73 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -30,7 +30,12 @@ func NewLB(k8sclient kubernetes.Interface, config string) *LB { } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { - // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + /* + 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + What we need here is: + - The NodePort (for first pass we will use the same port on the LB, so if NodePort is 8000 we use 8000 on LB) + - The public IPs of the nodes on which the service is running + */ additionalProperties := map[string]string{} // 2. Create the infrastructure (what do we need to return here? lb name and/or ID? anything else?) @@ -40,12 +45,19 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return err } - // 3. Add the annotations + /* + 3. Add the annotations + - ID of the load balancer + - Name of the load balancer + - Metro of the load balancer + - IP address of the load balancer + - Listener port that this service is using + */ return nil } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { - // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + // 1. Gather the properties we need: ID of load balancer loadBalancerId := "TODO" additionalProperties := map[string]string{} @@ -56,13 +68,18 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string return err } - // 3. Remove the annotations + // 3. No need to remove the annotations because the annotated object was deleted return nil } func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { - // 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) + /* + 1. Gather the properties we need: + - load balancer ID + - NodePort + - Public IP addresses of the nodes on which the target pods are running + */ loadBalancerId := "TODO" additionalProperties := map[string]string{} @@ -73,7 +90,10 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no return err } - // 3. Update the annotations + /* + 3. Update the annotations + - Listener port that this service is using + */ return nil } From 69fd1a98876417e3feadc4ebb7e3f89adfe6c8f9 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 28 Sep 2023 15:03:33 -0500 Subject: [PATCH 11/48] pass in API key and project ID for EMLB --- metal/loadbalancers.go | 2 +- metal/loadbalancers/emlb/controller.go | 6 +++++- metal/loadbalancers/emlb/emlb.go | 14 +++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index e6f1ae91..256cd737 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -93,7 +93,7 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr impl = empty.NewLB(k8sclient, lbconfig) case "emlb": klog.Info("loadbalancer implementation enabled: emlb") - impl = emlb.NewLB(k8sclient, lbconfig) + impl = emlb.NewLB(k8sclient, lbconfig, client.APIKey, projectID) default: klog.Info("loadbalancer implementation disabled") impl = nil diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go index 2bc3ff64..20f2f0f2 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/controller.go @@ -19,9 +19,11 @@ var LBMetros = map[string]string{ type controller struct { client *lbaas.APIClient tokenExchanger *MetalTokenExchanger + projectID string + metro string } -func NewController(metalAPIKey string) *controller { +func NewController(metalAPIKey, projectID, metro string) *controller { controller := &controller{} emlbConfig := lbaas.NewConfiguration() @@ -30,6 +32,8 @@ func NewController(metalAPIKey string) *controller { metalAPIKey: metalAPIKey, client: controller.client.GetConfig().HTTPClient, } + controller.projectID = projectID + controller.metro = metro return controller } diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 43ce5d73..bc630241 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -4,28 +4,24 @@ package emlb import ( "context" - lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "k8s.io/client-go/kubernetes" ) type LB struct { - controller *controller - loadBalancerLocation *lbaas.LoadBalancerLocation + controller *controller } -func NewLB(k8sclient kubernetes.Interface, config string) *LB { +func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string) *LB { // Parse config for Equinix Metal Load Balancer - // An example config using Dallas as the location would look like // The format is emlb:// + // An example config using Dallas as the location would look like emlb://da // it may have an extra slash at the beginning or end, so get rid of it - - metalAPIKey := "TODO" + metro := config lb := &LB{} - lb.loadBalancerLocation.Id = &config + lb.controller = NewController(metalAPIKey, projectID, metro) - lb.controller = NewController(metalAPIKey) return lb } From 9d7d97f273085e0bea7c120abfc9dd455376d289 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Thu, 28 Sep 2023 15:40:02 -0500 Subject: [PATCH 12/48] fix: MapKeys() not Keys() Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers/emlb/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go index 20f2f0f2..cd8732b7 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/controller.go @@ -47,7 +47,7 @@ func (c *controller) createLoadBalancer(ctx context.Context, config map[string]s locationId, ok := LBMetros[metro] if !ok { - return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).Keys()) + return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).MapKeys()) } lbCreateRequest := lbaas.LoadBalancerCreate{ Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? From e53c43fcb7b35ab9abd598263d48727dd61ff548 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:48:50 -0500 Subject: [PATCH 13/48] feat: support tilt based development Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- .gitignore | 1 + Tiltfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c1844549..0a82df5c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ vendor/ hacks/ tmp/ .idea +cloud-sa.json \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index 27d0d76c..185103ae 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,5 +1,5 @@ # Specify the registry you wish to store the image in -registry = 'sabin1001' +registry = 'your-registry-here' # List the k8s context you wish to run this in allow_k8s_contexts('capi-quicktest-admin@capi-quicktest') From c08e63c15b17bf68fdd0ab1f9dca1d5b86971384 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:57:33 -0500 Subject: [PATCH 14/48] fix: get off experimental builder Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7e1cde81..0bb8f599 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.1-experimental +# syntax=docker/dockerfile:1 # Copyright 2020 The Kubernetes Authors. # From 1c80d871c4ced838248caea8e3d4b424b91ef488 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Mon, 2 Oct 2023 14:26:18 -0500 Subject: [PATCH 15/48] refactor: introduce usesBGP flag for loadbalancers --- metal/loadbalancers.go | 364 ++++++++++++++++++++++------------------- 1 file changed, 196 insertions(+), 168 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 256cd737..b8a670ce 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -48,6 +48,7 @@ type loadBalancers struct { eipFacilityAnnotation string nodeSelector labels.Selector eipTag string + usesBGP bool } func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, projectID, metro, facility, config string, localASN int, bgpPass, annotationNetwork, annotationLocalASN, annotationPeerASN, annotationPeerIP, annotationSrcIP, annotationBgpPass, eipMetroAnnotation, eipFacilityAnnotation, nodeSelector, eipTag string) (*loadBalancers, error) { @@ -56,7 +57,11 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr selector, _ = labels.Parse(nodeSelector) } - l := &loadBalancers{client, k8sclient, projectID, metro, facility, "", nil, config, localASN, bgpPass, annotationNetwork, annotationLocalASN, annotationPeerASN, annotationPeerIP, annotationSrcIP, annotationBgpPass, eipMetroAnnotation, eipFacilityAnnotation, selector, eipTag} + // TODO: refactor this and related functions so we can move common code + // for BGP-based load balancers somewhere else + defaultUsesBgp := true + + l := &loadBalancers{client, k8sclient, projectID, metro, facility, "", nil, config, localASN, bgpPass, annotationNetwork, annotationLocalASN, annotationPeerASN, annotationPeerIP, annotationSrcIP, annotationBgpPass, eipMetroAnnotation, eipFacilityAnnotation, selector, eipTag, defaultUsesBgp} // parse the implementor config and see what kind it is - allow for no config if l.implementorConfig == "" { @@ -94,6 +99,8 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr case "emlb": klog.Info("loadbalancer implementation enabled: emlb") impl = emlb.NewLB(k8sclient, lbconfig, client.APIKey, projectID) + // TODO remove when common BGP code has been refactored to somewhere else + l.usesBGP = false default: klog.Info("loadbalancer implementation disabled") impl = nil @@ -155,20 +162,16 @@ func (l *loadBalancers) GetLoadBalancerName(ctx context.Context, clusterName str // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { klog.V(2).Infof("EnsureLoadBalancer(): add: service %s/%s", service.Namespace, service.Name) - // get IP address reservations and check if they any exists for this svc - ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) - if err != nil { - return nil, fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) - } var ipCidr string + var err error // handling is completely different if it is the control plane vs a regular service of type=LoadBalancer if service.Name == externalServiceName && service.Namespace == externalServiceNamespace { - ipCidr, err = l.retrieveIPByTag(ctx, service, ips, l.eipTag) + ipCidr, err = l.retrieveIPByTag(ctx, service, l.eipTag) if err != nil { return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } } else { - ipCidr, err = l.addService(ctx, service, ips, filterNodes(nodes, l.nodeSelector)) + ipCidr, err = l.addService(ctx, service, filterNodes(nodes, l.nodeSelector)) if err != nil { return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } @@ -191,39 +194,44 @@ func (l *loadBalancers) UpdateLoadBalancer(ctx context.Context, clusterName stri klog.V(2).Infof("UpdateLoadBalancer(): service %s", service.Name) var n []loadbalancers.Node - for _, node := range filterNodes(nodes, l.nodeSelector) { - klog.V(2).Infof("UpdateLoadBalancer(): %s", node.Name) - // get the node provider ID - id := node.Spec.ProviderID - if id == "" { - return fmt.Errorf("no provider ID given for node %s, skipping", node.Name) - } - // ensure BGP is enabled for the node - if err := ensureNodeBGPEnabled(id, l.client); err != nil { - klog.Errorf("could not ensure BGP enabled for node %s: %w", node.Name, err) - continue - } - klog.V(2).Infof("bgp enabled on node %s", node.Name) - // ensure the node has the correct annotations - if err := l.annotateNode(ctx, node); err != nil { - return fmt.Errorf("failed to annotate node %s: %w", node.Name, err) - } - var ( - peer *packngo.BGPNeighbor - err error - ) - if peer, err = getNodeBGPConfig(id, l.client); err != nil || peer == nil { - return fmt.Errorf("could not add metallb node peer address for node %s: %w", node.Name, err) + + // TODO remove this conditional when common BGP code has been refactored to somewhere else + if l.usesBGP { + for _, node := range filterNodes(nodes, l.nodeSelector) { + klog.V(2).Infof("UpdateLoadBalancer(): %s", node.Name) + // get the node provider ID + id := node.Spec.ProviderID + if id == "" { + return fmt.Errorf("no provider ID given for node %s, skipping", node.Name) + } + // ensure BGP is enabled for the node + if err := ensureNodeBGPEnabled(id, l.client); err != nil { + klog.Errorf("could not ensure BGP enabled for node %s: %w", node.Name, err) + continue + } + klog.V(2).Infof("bgp enabled on node %s", node.Name) + // ensure the node has the correct annotations + if err := l.annotateNode(ctx, node); err != nil { + return fmt.Errorf("failed to annotate node %s: %w", node.Name, err) + } + var ( + peer *packngo.BGPNeighbor + err error + ) + if peer, err = getNodeBGPConfig(id, l.client); err != nil || peer == nil { + return fmt.Errorf("could not add metallb node peer address for node %s: %w", node.Name, err) + } + n = append(n, loadbalancers.Node{ + Name: node.Name, + LocalASN: peer.CustomerAs, + PeerASN: peer.PeerAs, + SourceIP: peer.CustomerIP, + Peers: peer.PeerIps, + Password: peer.Md5Password, + }) } - n = append(n, loadbalancers.Node{ - Name: node.Name, - LocalASN: peer.CustomerAs, - PeerASN: peer.PeerAs, - SourceIP: peer.CustomerIP, - Peers: peer.PeerIps, - Password: peer.Md5Password, - }) } + return l.implementor.UpdateService(ctx, service.Namespace, service.Name, n) } @@ -244,33 +252,38 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa var svcIPCidr string - // get IP address reservations and check if they any exists for this svc - ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) - if err != nil { - return fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) - } + if l.usesBGP { - ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) + // get IP address reservations and check if they any exists for this svc + ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) + if err != nil { + return fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) + } - klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: %s with existing IP assignment %s", svcName, svcIP) + ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) - // get the IPs and see if there is anything to clean up - if ipReservation == nil { - klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: no IP reservation found for %s, nothing to delete", svcName) - return nil - } - // delete the reservation - klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: for %s EIP ID %s", svcName, ipReservation.ID) - if _, err := l.client.ProjectIPs.Remove(ipReservation.ID); err != nil { - return fmt.Errorf("failed to remove IP address reservation %s from project: %w", ipReservation.String(), err) + klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: %s with existing IP assignment %s", svcName, svcIP) + + // get the IPs and see if there is anything to clean up + if ipReservation == nil { + klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: no IP reservation found for %s, nothing to delete", svcName) + return nil + } + // delete the reservation + klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: for %s EIP ID %s", svcName, ipReservation.ID) + if _, err := l.client.ProjectIPs.Remove(ipReservation.ID); err != nil { + return fmt.Errorf("failed to remove IP address reservation %s from project: %w", ipReservation.String(), err) + } + // remove it from any implementation-specific parts + svcIPCidr = fmt.Sprintf("%s/%d", ipReservation.Address, ipReservation.CIDR) + klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: for %s entry %s", svcName, svcIPCidr) } - // remove it from any implementation-specific parts - svcIPCidr = fmt.Sprintf("%s/%d", ipReservation.Address, ipReservation.CIDR) - klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: for %s entry %s", svcName, svcIPCidr) + if err := l.implementor.RemoveService(ctx, service.Namespace, service.Name, svcIPCidr); err != nil { return fmt.Errorf("error removing IP from configmap for %s: %w", svcName, err) } klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: removed service %s from implementation", svcName) + return nil } @@ -365,7 +378,7 @@ func (l *loadBalancers) annotateNode(ctx context.Context, node *v1.Node) error { } // addService add a single service; wraps the implementation -func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, ips []packngo.IPAddressReservation, nodes []*v1.Node) (string, error) { +func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, nodes []*v1.Node) (string, error) { svcName := serviceRep(svc) svcTag := serviceTag(svc) svcRegion := serviceAnnotation(svc, l.eipMetroAnnotation) @@ -376,135 +389,150 @@ func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, ips []p var ( svcIPCidr string err error + n []loadbalancers.Node + ips []packngo.IPAddressReservation ) - ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) - klog.V(2).Infof("processing %s with existing IP assignment %s", svcName, svcIP) - // if it already has an IP, no need to get it one - if svcIP == "" { - klog.V(2).Infof("no IP assigned for service %s; searching reservations", svcName) - - // if no IP found, request a new one - if ipReservation == nil { + if l.usesBGP { + // get IP address reservations and check if they any exists for this svc + ips, _, err = l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) + if err != nil { + return "", fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) + } + ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) + + klog.V(2).Infof("processing %s with existing IP assignment %s", svcName, svcIP) + // if it already has an IP, no need to get it one + if svcIP == "" { + klog.V(2).Infof("no IP assigned for service %s; searching reservations", svcName) + + // if no IP found, request a new one + if ipReservation == nil { + + // if we did not find an IP reserved, create a request + klog.V(2).Infof("no IP assignment found for %s, requesting", svcName) + // create a request + // our logic as to where to create the IP: + // 1. if metro is set globally, use it; else + // 2. if facility is set globally, use it; else + // 3. if Service.Metadata.Labels["topology.kubernetes.io/region"] is set, use it; else + // 4. if Service.Metadata.Labels["topology.kubernetes.io/zone"] is set, use it; else + // 5. Return error, cannot set an EIP + facility := l.facility + metro := l.metro + req := packngo.IPReservationRequest{ + Type: "public_ipv4", + Quantity: 1, + Description: ccmIPDescription, + Tags: []string{ + emTag, + svcTag, + clsTag, + }, + FailOnApprovalRequired: true, + } + switch { + case svcRegion != "": + req.Metro = &svcRegion + case svcZone != "": + req.Facility = &svcZone + case metro != "": + req.Metro = &metro + case facility != "": + req.Facility = &facility + default: + return "", errors.New("unable to create load balancer when no IP, region or zone specified, either globally or on service") + } + + ipReservation, _, err = l.client.ProjectIPs.Request(l.project, &req) + if err != nil { + return "", fmt.Errorf("failed to request an IP for the load balancer: %w", err) + } + } - // if we did not find an IP reserved, create a request - klog.V(2).Infof("no IP assignment found for %s, requesting", svcName) - // create a request - // our logic as to where to create the IP: - // 1. if metro is set globally, use it; else - // 2. if facility is set globally, use it; else - // 3. if Service.Metadata.Labels["topology.kubernetes.io/region"] is set, use it; else - // 4. if Service.Metadata.Labels["topology.kubernetes.io/zone"] is set, use it; else - // 5. Return error, cannot set an EIP - facility := l.facility - metro := l.metro - req := packngo.IPReservationRequest{ - Type: "public_ipv4", - Quantity: 1, - Description: ccmIPDescription, - Tags: []string{ - emTag, - svcTag, - clsTag, - }, - FailOnApprovalRequired: true, + // if we have no IP from existing or a new reservation, log it and return + if ipReservation == nil { + klog.V(2).Infof("no IP to assign to service %s, will need to wait until it is allocated", svcName) + return "", nil } - switch { - case svcRegion != "": - req.Metro = &svcRegion - case svcZone != "": - req.Facility = &svcZone - case metro != "": - req.Metro = &metro - case facility != "": - req.Facility = &facility - default: - return "", errors.New("unable to create load balancer when no IP, region or zone specified, either globally or on service") + + // we have an IP, either found from existing reservations or a new reservation. + // map and assign it + svcIP = ipReservation.Address + + // assign the IP and save it + klog.V(2).Infof("assigning IP %s to %s", svcIP, svcName) + intf := l.k8sclient.CoreV1().Services(svc.Namespace) + existing, err := intf.Get(ctx, svc.Name, metav1.GetOptions{}) + if err != nil || existing == nil { + klog.V(2).Infof("failed to get latest for service %s: %v", svcName, err) + return "", fmt.Errorf("failed to get latest for service %s: %w", svcName, err) } + existing.Spec.LoadBalancerIP = svcIP - ipReservation, _, err = l.client.ProjectIPs.Request(l.project, &req) + _, err = intf.Update(ctx, existing, metav1.UpdateOptions{}) if err != nil { - return "", fmt.Errorf("failed to request an IP for the load balancer: %w", err) + klog.V(2).Infof("failed to update service %s: %v", svcName, err) + return "", fmt.Errorf("failed to update service %s: %w", svcName, err) } + klog.V(2).Infof("successfully assigned %s update service %s", svcIP, svcName) } - - // if we have no IP from existing or a new reservation, log it and return - if ipReservation == nil { - klog.V(2).Infof("no IP to assign to service %s, will need to wait until it is allocated", svcName) - return "", nil - } - - // we have an IP, either found from existing reservations or a new reservation. - // map and assign it - svcIP = ipReservation.Address - - // assign the IP and save it - klog.V(2).Infof("assigning IP %s to %s", svcIP, svcName) - intf := l.k8sclient.CoreV1().Services(svc.Namespace) - existing, err := intf.Get(ctx, svc.Name, metav1.GetOptions{}) - if err != nil || existing == nil { - klog.V(2).Infof("failed to get latest for service %s: %v", svcName, err) - return "", fmt.Errorf("failed to get latest for service %s: %w", svcName, err) - } - existing.Spec.LoadBalancerIP = svcIP - - _, err = intf.Update(ctx, existing, metav1.UpdateOptions{}) - if err != nil { - klog.V(2).Infof("failed to update service %s: %v", svcName, err) - return "", fmt.Errorf("failed to update service %s: %w", svcName, err) + // our default CIDR for each address is 32 + cidr := 32 + if ipReservation != nil { + cidr = ipReservation.CIDR } - klog.V(2).Infof("successfully assigned %s update service %s", svcIP, svcName) - } - // our default CIDR for each address is 32 - cidr := 32 - if ipReservation != nil { - cidr = ipReservation.CIDR - } - svcIPCidr = fmt.Sprintf("%s/%d", svcIP, cidr) - // now need to pass it the nodes - - var n []loadbalancers.Node - for _, node := range nodes { - // get the node provider ID - id := node.Spec.ProviderID - if id == "" { - klog.Errorf("no provider ID given for node %s, skipping", node.Name) - continue - } - // ensure BGP is enabled for the node - if err := ensureNodeBGPEnabled(id, l.client); err != nil { - klog.Errorf("could not ensure BGP enabled for node %s: %w", node.Name, err) - continue - } - klog.V(2).Infof("bgp enabled on node %s", node.Name) - // ensure the node has the correct annotations - if err := l.annotateNode(ctx, node); err != nil { - klog.Errorf("failed to annotate node %s: %w", node.Name, err) - continue - } - peer, err := getNodeBGPConfig(id, l.client) - if err != nil || peer == nil { - klog.Errorf("loadbalancers.addService(): could not get node peer address for node %s: %w", node.Name, err) - continue + svcIPCidr = fmt.Sprintf("%s/%d", svcIP, cidr) + // now need to pass it the nodes + + for _, node := range nodes { + // get the node provider ID + id := node.Spec.ProviderID + if id == "" { + klog.Errorf("no provider ID given for node %s, skipping", node.Name) + continue + } + // ensure BGP is enabled for the node + if err := ensureNodeBGPEnabled(id, l.client); err != nil { + klog.Errorf("could not ensure BGP enabled for node %s: %w", node.Name, err) + continue + } + klog.V(2).Infof("bgp enabled on node %s", node.Name) + // ensure the node has the correct annotations + if err := l.annotateNode(ctx, node); err != nil { + klog.Errorf("failed to annotate node %s: %w", node.Name, err) + continue + } + peer, err := getNodeBGPConfig(id, l.client) + if err != nil || peer == nil { + klog.Errorf("loadbalancers.addService(): could not get node peer address for node %s: %w", node.Name, err) + continue + } + n = append(n, loadbalancers.Node{ + Name: node.Name, + LocalASN: peer.CustomerAs, + PeerASN: peer.PeerAs, + SourceIP: peer.CustomerIP, + Peers: peer.PeerIps, + Password: peer.Md5Password, + }) } - n = append(n, loadbalancers.Node{ - Name: node.Name, - LocalASN: peer.CustomerAs, - PeerASN: peer.PeerAs, - SourceIP: peer.CustomerIP, - Peers: peer.PeerIps, - Password: peer.Md5Password, - }) } return svcIPCidr, l.implementor.AddService(ctx, svc.Namespace, svc.Name, svcIPCidr, n) } -func (l *loadBalancers) retrieveIPByTag(ctx context.Context, svc *v1.Service, ips []packngo.IPAddressReservation, tag string) (string, error) { +func (l *loadBalancers) retrieveIPByTag(ctx context.Context, svc *v1.Service, tag string) (string, error) { svcName := serviceRep(svc) svcIP := svc.Spec.LoadBalancerIP cidr := 32 + // get IP address reservations and check if they any exists for this svc + ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) + if err != nil { + return "", err + } + var svcIPCidr string ipReservation := ipReservationByAllTags([]string{tag}, ips) From 24b62b087131752b73bbae1051743bab8c2f3eb4 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Mon, 2 Oct 2023 17:40:20 -0500 Subject: [PATCH 16/48] feat: move example dev files to dev/ --- .gitignore | 3 ++- Tiltfile => dev/Tiltfile | 10 +++++----- dev/cloud-sa.json | 4 ++++ 3 files changed, 11 insertions(+), 6 deletions(-) rename Tiltfile => dev/Tiltfile (85%) create mode 100644 dev/cloud-sa.json diff --git a/.gitignore b/.gitignore index 0a82df5c..9c40695d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ vendor/ hacks/ tmp/ .idea -cloud-sa.json \ No newline at end of file +/cloud-sa.json +/Tiltfile \ No newline at end of file diff --git a/Tiltfile b/dev/Tiltfile similarity index 85% rename from Tiltfile rename to dev/Tiltfile index 185103ae..796f38d4 100644 --- a/Tiltfile +++ b/dev/Tiltfile @@ -1,22 +1,22 @@ # Specify the registry you wish to store the image in -registry = 'your-registry-here' +registry = 'YOUR_REGISTRY_HERE' # List the k8s context you wish to run this in -allow_k8s_contexts('capi-quicktest-admin@capi-quicktest') +allow_k8s_contexts('YOUR_K8S_CONTEXT_HERE') # Specify docker registry you wish to store image in docker_build(registry + '/cloud-provider-equinix-metal', context='.', dockerfile='./Dockerfile', - ignore=['cloud-sa.json','dev/'], + ignore=['cloud-sa.json'], ) # read in the yaml file and replace the image name with the one we built deployment = read_yaml_stream('deploy/template/deployment.yaml') deployment[0]['spec']['template']['spec']['containers'][0]['image'] = registry + '/cloud-provider-equinix-metal' deployment[0]['spec']['template']['spec']['containers'][0]['env']=[] -deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_METRO_NAME","value":"da"}) -deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_LOAD_BALANCER","value":"emlb:///da"}) +deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_METRO_NAME","value":"YOUR_METRO_HERE"}) +deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_LOAD_BALANCER","value":"YOUR_LOAD_BALANCER_HERE"}) k8s_yaml(encode_yaml_stream(deployment)) k8s_resource(workload='cloud-provider-equinix-metal',objects=['cloud-controller-manager:ServiceAccount:kube-system','system\\:cloud-controller-manager:ClusterRole:default','system\\:cloud-controller-manager:ClusterRoleBinding:default']) k8s_resource(new_name='metal-cloud-config',objects=['metal-cloud-config:Secret:kube-system']) diff --git a/dev/cloud-sa.json b/dev/cloud-sa.json new file mode 100644 index 00000000..0523008e --- /dev/null +++ b/dev/cloud-sa.json @@ -0,0 +1,4 @@ +{ + "apiKey":"YOUR_API_KEY", + "projectID": "YOUR_PROJECT_ID" +} \ No newline at end of file From 978e07b40c1ce0ae78ba6efafd6072aec4d8182d Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:05:32 -0500 Subject: [PATCH 17/48] fix: ignore dev directory for rebuilds Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- dev/Tiltfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Tiltfile b/dev/Tiltfile index 796f38d4..9753a34f 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -8,7 +8,7 @@ allow_k8s_contexts('YOUR_K8S_CONTEXT_HERE') docker_build(registry + '/cloud-provider-equinix-metal', context='.', dockerfile='./Dockerfile', - ignore=['cloud-sa.json'], + ignore=['cloud-sa.json','dev/'], ) # read in the yaml file and replace the image name with the one we built From 322373ce486d1478dac71a052b215f5c78040fce Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 3 Oct 2023 13:57:26 -0500 Subject: [PATCH 18/48] feat: create an Equinix Metal Load Balancer for a k8s service --- metal/loadbalancers.go | 2 +- metal/loadbalancers/emlb/controller.go | 75 ++++++++++++++----- metal/loadbalancers/emlb/emlb.go | 59 ++++++++++----- .../emlb/metal_token_exchanger.go | 2 +- metal/loadbalancers/empty/empty.go | 5 +- metal/loadbalancers/interface.go | 4 +- metal/loadbalancers/kubevip/kubevip.go | 5 +- metal/loadbalancers/metallb/metallb.go | 3 +- 8 files changed, 113 insertions(+), 42 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index b8a670ce..943c17ac 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -519,7 +519,7 @@ func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, nodes [ } } - return svcIPCidr, l.implementor.AddService(ctx, svc.Namespace, svc.Name, svcIPCidr, n) + return svcIPCidr, l.implementor.AddService(ctx, svc.Namespace, svc.Name, svcIPCidr, n, svc, nodes) } func (l *loadBalancers) retrieveIPByTag(ctx context.Context, svc *v1.Service, tag string) (string, error) { diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go index cd8732b7..24338301 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/controller.go @@ -38,31 +38,77 @@ func NewController(metalAPIKey, projectID, metro string) *controller { return controller } -func (c *controller) createLoadBalancer(ctx context.Context, config map[string]string) (map[string]string, error) { - outputProperties := map[string]string{} - +func (c *controller) createLoadBalancer(ctx context.Context, name string, port int32, ips []string) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) - metro := config["metro"] - - locationId, ok := LBMetros[metro] + locationId, ok := LBMetros[c.metro] if !ok { - return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", metro, reflect.ValueOf(LBMetros).MapKeys()) + return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", c.metro, reflect.ValueOf(LBMetros).MapKeys()) } + lbCreateRequest := lbaas.LoadBalancerCreate{ - Name: "", // TODO generate from service definition. Maybe "svcNamespace:svcName"? Do we need to know the cluster name here? + Name: name, LocationId: locationId, ProviderId: ProviderID, } // TODO lb, resp, err := - _, _, err := c.client.ProjectsApi.CreateLoadBalancer(ctx, "TODO: project ID").LoadBalancerCreate(lbCreateRequest).Execute() + lbCreated, _, err := c.client.ProjectsApi.CreateLoadBalancer(ctx, c.projectID).LoadBalancerCreate(lbCreateRequest).Execute() if err != nil { return nil, err } - // TODO create other resources - return outputProperties, nil + loadBalancerID := lbCreated.GetId() + + createPoolRequest := lbaas.LoadBalancerPoolCreate{ + Name: fmt.Sprintf("%v-pool", name), + Protocol: lbaas.LoadBalancerPoolCreateProtocol{ + LoadBalancerPoolProtocol: lbaas.LOADBALANCERPOOLPROTOCOL_TCP.Ptr(), + }, + } + + poolCreated, _, err := c.client.ProjectsApi.CreatePool(ctx, c.projectID).LoadBalancerPoolCreate(createPoolRequest).Execute() + if err != nil { + return nil, err + } + + poolID := poolCreated.GetId() + + for i, ip := range ips { + createOriginRequest := lbaas.LoadBalancerPoolOriginCreate{ + Name: fmt.Sprintf("%v-origin-%v", name, i), + Target: ip, + PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ + Int32: &port, + }, + Active: true, + PoolId: poolID, + } + // TODO do we need the origin IDs for something? + _, _, err := c.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + if err != nil { + return nil, err + } + } + + createPortRequest := lbaas.LoadBalancerPortCreate{ + Name: fmt.Sprintf("%v-port-%v", name, port), + Number: port, + PoolIds: []string{poolID}, + } + + // TODO do we need the port ID for something? + _, _, err = c.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() + if err != nil { + return nil, err + } + + lb, _, err := c.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() + if err != nil { + return nil, err + } + + return lb, nil } func (c *controller) updateLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { @@ -83,12 +129,7 @@ func (c *controller) updateLoadBalancer(ctx context.Context, id string, config m func (c *controller) deleteLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { outputProperties := map[string]string{} - - tokenExchanger := &MetalTokenExchanger{ - metalAPIKey: "TODO", - client: c.client.GetConfig().HTTPClient, - } - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, tokenExchanger) + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) // TODO delete other resources diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index bc630241..e30a1122 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -3,8 +3,12 @@ package emlb import ( "context" + "errors" + "strings" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" ) @@ -12,12 +16,14 @@ type LB struct { controller *controller } +var _ loadbalancers.LB = (*LB)(nil) + func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string) *LB { // Parse config for Equinix Metal Load Balancer // The format is emlb:// // An example config using Dallas as the location would look like emlb://da // it may have an extra slash at the beginning or end, so get rid of it - metro := config + metro := strings.TrimPrefix(config, "/") lb := &LB{} lb.controller = NewController(metalAPIKey, projectID, metro) @@ -25,30 +31,45 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string return lb } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { - /* - 1. Gather the properties we need: Metal API key, port number(s), cluster name(?), target IP(s?) - What we need here is: - - The NodePort (for first pass we will use the same port on the LB, so if NodePort is 8000 we use 8000 on LB) - - The public IPs of the nodes on which the service is running - */ - additionalProperties := map[string]string{} +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { + name := rand.String(32) - // 2. Create the infrastructure (what do we need to return here? lb name and/or ID? anything else?) - _, err := l.controller.createLoadBalancer(ctx, additionalProperties) + if len(svc.Spec.Ports) < 1 { + return errors.New("cannot add loadbalancer service; no nodeport assigned") + } + + port := svc.Spec.Ports[0].NodePort + var ips []string + + for _, node := range n { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeExternalIP { + ips = append(ips, address.Address) + } + } + } + + if len(ips) < 1 { + return errors.New("cannot add loadbalancer service; failed to find an external IP address for any node") + } + + loadBalancer, err := l.controller.createLoadBalancer(ctx, name, port, ips) if err != nil { return err } - /* - 3. Add the annotations - - ID of the load balancer - - Name of the load balancer - - Metro of the load balancer - - IP address of the load balancer - - Listener port that this service is using - */ + var ingress []v1.LoadBalancerIngress + for _, ip := range loadBalancer.GetIps() { + ingress = append(ingress, v1.LoadBalancerIngress{ + IP: ip, + }) + } + svc.Status.LoadBalancer.Ingress = ingress + + svc.Annotations["equinix.com/loadbalancerID"] = loadBalancer.GetId() + svc.Annotations["equinix.com/loadbalancerMetro"] = l.controller.metro + return nil } diff --git a/metal/loadbalancers/emlb/metal_token_exchanger.go b/metal/loadbalancers/emlb/metal_token_exchanger.go index 0157f470..b2e5d879 100644 --- a/metal/loadbalancers/emlb/metal_token_exchanger.go +++ b/metal/loadbalancers/emlb/metal_token_exchanger.go @@ -35,7 +35,7 @@ func (m *MetalTokenExchanger) Token() (*oauth2.Token, error) { } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("token exchange request failed with status %v, body %v", resp.StatusCode, body) + return nil, fmt.Errorf("token exchange request failed with status %v, body %v", resp.StatusCode, string(body[:])) } token := oauth2.Token{} diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index c63e76af..38304f30 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -5,16 +5,19 @@ import ( "context" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) type LB struct{} +var _ loadbalancers.LB = (*LB)(nil) + func NewLB(k8sclient kubernetes.Interface, config string) *LB { return &LB{} } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { return nil } diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index 887f13db..7f112379 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -2,11 +2,13 @@ package loadbalancers import ( "context" + + v1 "k8s.io/api/core/v1" ) type LB interface { // AddService add a service with the provided name and IP - AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []Node) error + AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []Node, svc *v1.Service, n []*v1.Node) error // RemoveService remove service with the given IP RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error // UpdateService ensure that the nodes handled by the service are correct diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index 11fdf55d..63a44067 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -5,16 +5,19 @@ import ( "context" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) type LB struct{} +var _ loadbalancers.LB = (*LB)(nil) + func NewLB(k8sclient kubernetes.Interface, config string) *LB { return &LB{} } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { return nil } diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index c6a4e704..f33a4255 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -9,6 +9,7 @@ import ( "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" metallbv1beta1 "go.universe.tf/metallb/api/v1beta1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -115,7 +116,7 @@ func NewLB(k8sclient kubernetes.Interface, config string, featureFlags url.Value return lb } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { config := l.configurer if err := config.Get(ctx); err != nil { return fmt.Errorf("unable to add service: %w", err) From 716c3ef5c89630dd34e68209e914210c26b082d1 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 3 Oct 2023 14:54:11 -0500 Subject: [PATCH 19/48] return actual service status for non-BGP load balancers --- metal/loadbalancers.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 943c17ac..af872937 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -176,14 +176,19 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } } - // get the IP only - ip := strings.SplitN(ipCidr, "/", 2) - return &v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - {IP: ip[0]}, - }, - }, nil + if l.usesBGP { + // get the IP only + ip := strings.SplitN(ipCidr, "/", 2) + + return &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + {IP: ip[0]}, + }, + }, nil + } else { + return &service.Status.LoadBalancer, nil + } } // UpdateLoadBalancer updates hosts under the specified load balancer. From abd52f3fcff4a677b46d600606bcde0d7993301d Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 3 Oct 2023 14:54:43 -0500 Subject: [PATCH 20/48] set LoadBalancerIP for backwards compatibility --- metal/loadbalancers/emlb/emlb.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index e30a1122..f36c1531 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -64,6 +64,8 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n ingress = append(ingress, v1.LoadBalancerIngress{ IP: ip, }) + // TODO: this is here for backwards compatibility and should be removed ASAP + svc.Spec.LoadBalancerIP = ip } svc.Status.LoadBalancer.Ingress = ingress From 4456f42be4859e0c874714a79e699c81b93f2f9b Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:14:25 -0500 Subject: [PATCH 21/48] fix: various updates and comments - add the front end port - use the front end port for the load balancer - point out the service annotations don't work Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers/emlb/emlb.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index f36c1531..be1fd07e 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -38,7 +38,9 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return errors.New("cannot add loadbalancer service; no nodeport assigned") } - port := svc.Spec.Ports[0].NodePort + // TODO: Loop through and add ALL the ports + port := svc.Spec.Ports[0].Port + nodePort := svc.Spec.Ports[0].NodePort var ips []string for _, node := range n { @@ -52,8 +54,8 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n if len(ips) < 1 { return errors.New("cannot add loadbalancer service; failed to find an external IP address for any node") } - - loadBalancer, err := l.controller.createLoadBalancer(ctx, name, port, ips) + // TODO move port looping inside of here so we don't get a new loadbalancer per port + loadBalancer, err := l.controller.createLoadBalancer(ctx, name, port, nodePort, ips) if err != nil { return err @@ -67,6 +69,8 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n // TODO: this is here for backwards compatibility and should be removed ASAP svc.Spec.LoadBalancerIP = ip } + + // TODO: THIS DOES NOT ACTUALLY UPDATE THE SERVICE! svc.Status.LoadBalancer.Ingress = ingress svc.Annotations["equinix.com/loadbalancerID"] = loadBalancer.GetId() From b44bae8c907204980d4abd9b6476aff1d1776291 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:14:40 -0500 Subject: [PATCH 22/48] fix: ensure LoadBalancer is implemented Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index af872937..64493a53 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/labels" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" ) @@ -112,7 +113,8 @@ func newLoadBalancers(client *packngo.Client, k8sclient kubernetes.Interface, pr return l, nil } -// implementation of cloudprovider.LoadBalancer +// validate our implementation of cloudprovider.LoadBalancer +var _ cloudprovider.LoadBalancer = (*loadBalancers)(nil) // GetLoadBalancer returns whether the specified load balancer exists, and // if so, what its status is. From 0fe57f0a4544850d48efc4653dc4ae3db6e03b4f Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:14:55 -0500 Subject: [PATCH 23/48] fix: use front end port Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers/emlb/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go index 24338301..31034641 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/controller.go @@ -38,7 +38,7 @@ func NewController(metalAPIKey, projectID, metro string) *controller { return controller } -func (c *controller) createLoadBalancer(ctx context.Context, name string, port int32, ips []string) (*lbaas.LoadBalancer, error) { +func (c *controller) createLoadBalancer(ctx context.Context, name string, port int32, nodePort int32, ips []string) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) locationId, ok := LBMetros[c.metro] @@ -79,7 +79,7 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i Name: fmt.Sprintf("%v-origin-%v", name, i), Target: ip, PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ - Int32: &port, + Int32: &nodePort, }, Active: true, PoolId: poolID, From 20e73f5cbddb66216995610a9c895cd6d9eb6ba7 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:35:49 -0500 Subject: [PATCH 24/48] fix: don't call this controller Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers/emlb/controller.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/controller.go index 31034641..56935579 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/controller.go @@ -6,6 +6,7 @@ import ( "reflect" lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" + "k8s.io/client-go/kubernetes" ) const ProviderID = "loadpvd-gOB_-byp5ebFo7A3LHv2B" @@ -18,12 +19,13 @@ var LBMetros = map[string]string{ type controller struct { client *lbaas.APIClient - tokenExchanger *MetalTokenExchanger - projectID string + k8sclient kubernetes.Interface metro string + projectID string + tokenExchanger *MetalTokenExchanger } -func NewController(metalAPIKey, projectID, metro string) *controller { +func NewController(k8sclient kubernetes.Interface, metalAPIKey, projectID, metro string) *controller { controller := &controller{} emlbConfig := lbaas.NewConfiguration() From e62ed275e7a8687639462cfebb6de3d9411ca288 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:35:57 -0500 Subject: [PATCH 25/48] fix: don't call this controller Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers/emlb/emlb.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index be1fd07e..d66c58f4 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -12,6 +12,8 @@ import ( "k8s.io/client-go/kubernetes" ) +// TODO: don't call this controller + type LB struct { controller *controller } @@ -20,13 +22,13 @@ var _ loadbalancers.LB = (*LB)(nil) func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string) *LB { // Parse config for Equinix Metal Load Balancer - // The format is emlb:// - // An example config using Dallas as the location would look like emlb://da + // The format is emlb:/// + // An example config using Dallas as the location would look like emlb:///da // it may have an extra slash at the beginning or end, so get rid of it metro := strings.TrimPrefix(config, "/") lb := &LB{} - lb.controller = NewController(metalAPIKey, projectID, metro) + lb.controller = NewController(k8sclient, metalAPIKey, projectID, metro) return lb } From 9a9185f903a0d7ac5a20fc0430656b1c6e5ceaf7 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Wed, 4 Oct 2023 11:28:43 -0500 Subject: [PATCH 26/48] chore: refactor infrastructure management code This moves & renames the code related to managing LBaaS infrastructure to avoid name collisions between the k8s concept of a controller and the architecture concept of a controller, and to reflect the separation between k8s interactions in emlb.go and Equinix platform interactions in the infrastructure management code. --- metal/loadbalancers/emlb/emlb.go | 17 +++--- .../manager.go} | 58 ++++++++++--------- .../token_exchanger.go} | 6 +- 3 files changed, 42 insertions(+), 39 deletions(-) rename metal/loadbalancers/emlb/{controller.go => infrastructure/manager.go} (57%) rename metal/loadbalancers/emlb/{metal_token_exchanger.go => infrastructure/token_exchanger.go} (91%) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index d66c58f4..455a41f7 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -7,15 +7,15 @@ import ( "strings" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" + "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/emlb/infrastructure" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" ) -// TODO: don't call this controller - type LB struct { - controller *controller + manager *infrastructure.Manager + k8sclient kubernetes.Interface } var _ loadbalancers.LB = (*LB)(nil) @@ -28,7 +28,8 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string metro := strings.TrimPrefix(config, "/") lb := &LB{} - lb.controller = NewController(k8sclient, metalAPIKey, projectID, metro) + lb.manager = infrastructure.NewManager(metalAPIKey, projectID, metro) + lb.k8sclient = k8sclient return lb } @@ -57,7 +58,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return errors.New("cannot add loadbalancer service; failed to find an external IP address for any node") } // TODO move port looping inside of here so we don't get a new loadbalancer per port - loadBalancer, err := l.controller.createLoadBalancer(ctx, name, port, nodePort, ips) + loadBalancer, err := l.manager.CreateLoadBalancer(ctx, name, port, nodePort, ips) if err != nil { return err @@ -76,7 +77,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n svc.Status.LoadBalancer.Ingress = ingress svc.Annotations["equinix.com/loadbalancerID"] = loadBalancer.GetId() - svc.Annotations["equinix.com/loadbalancerMetro"] = l.controller.metro + svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() return nil } @@ -87,7 +88,7 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string additionalProperties := map[string]string{} // 2. Delete the infrastructure (do we need to return anything here?) - _, err := l.controller.deleteLoadBalancer(ctx, loadBalancerId, additionalProperties) + _, err := l.manager.DeleteLoadBalancer(ctx, loadBalancerId, additionalProperties) if err != nil { return err @@ -109,7 +110,7 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no additionalProperties := map[string]string{} // 2. Update infrastructure change (do we need to return anything here? or are all changes reflected by properties from [1]?) - _, err := l.controller.updateLoadBalancer(ctx, loadBalancerId, additionalProperties) + _, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, additionalProperties) if err != nil { return err diff --git a/metal/loadbalancers/emlb/controller.go b/metal/loadbalancers/emlb/infrastructure/manager.go similarity index 57% rename from metal/loadbalancers/emlb/controller.go rename to metal/loadbalancers/emlb/infrastructure/manager.go index 56935579..f9207128 100644 --- a/metal/loadbalancers/emlb/controller.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -1,4 +1,4 @@ -package emlb +package infrastructure import ( "context" @@ -6,7 +6,6 @@ import ( "reflect" lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" - "k8s.io/client-go/kubernetes" ) const ProviderID = "loadpvd-gOB_-byp5ebFo7A3LHv2B" @@ -17,35 +16,38 @@ var LBMetros = map[string]string{ "sv": "lctnloc-H5rl2M2VL5dcFmdxhbEKx", } -type controller struct { +type Manager struct { client *lbaas.APIClient - k8sclient kubernetes.Interface metro string projectID string - tokenExchanger *MetalTokenExchanger + tokenExchanger *TokenExchanger } -func NewController(k8sclient kubernetes.Interface, metalAPIKey, projectID, metro string) *controller { - controller := &controller{} +func NewManager(metalAPIKey, projectID, metro string) *Manager { + manager := &Manager{} emlbConfig := lbaas.NewConfiguration() - controller.client = lbaas.NewAPIClient(emlbConfig) - controller.tokenExchanger = &MetalTokenExchanger{ + manager.client = lbaas.NewAPIClient(emlbConfig) + manager.tokenExchanger = &TokenExchanger{ metalAPIKey: metalAPIKey, - client: controller.client.GetConfig().HTTPClient, + client: manager.client.GetConfig().HTTPClient, } - controller.projectID = projectID - controller.metro = metro + manager.projectID = projectID + manager.metro = metro - return controller + return manager } -func (c *controller) createLoadBalancer(ctx context.Context, name string, port int32, nodePort int32, ips []string) (*lbaas.LoadBalancer, error) { - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) +func (m *Manager) GetMetro() string { + return m.metro +} + +func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, port int32, nodePort int32, ips []string) (*lbaas.LoadBalancer, error) { + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - locationId, ok := LBMetros[c.metro] + locationId, ok := LBMetros[m.metro] if !ok { - return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", c.metro, reflect.ValueOf(LBMetros).MapKeys()) + return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", m.metro, reflect.ValueOf(LBMetros).MapKeys()) } lbCreateRequest := lbaas.LoadBalancerCreate{ @@ -55,7 +57,7 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i } // TODO lb, resp, err := - lbCreated, _, err := c.client.ProjectsApi.CreateLoadBalancer(ctx, c.projectID).LoadBalancerCreate(lbCreateRequest).Execute() + lbCreated, _, err := m.client.ProjectsApi.CreateLoadBalancer(ctx, m.projectID).LoadBalancerCreate(lbCreateRequest).Execute() if err != nil { return nil, err } @@ -69,7 +71,7 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i }, } - poolCreated, _, err := c.client.ProjectsApi.CreatePool(ctx, c.projectID).LoadBalancerPoolCreate(createPoolRequest).Execute() + poolCreated, _, err := m.client.ProjectsApi.CreatePool(ctx, m.projectID).LoadBalancerPoolCreate(createPoolRequest).Execute() if err != nil { return nil, err } @@ -87,7 +89,7 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i PoolId: poolID, } // TODO do we need the origin IDs for something? - _, _, err := c.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + _, _, err := m.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() if err != nil { return nil, err } @@ -100,12 +102,12 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i } // TODO do we need the port ID for something? - _, _, err = c.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() + _, _, err = m.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() if err != nil { return nil, err } - lb, _, err := c.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() + lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() if err != nil { return nil, err } @@ -113,15 +115,15 @@ func (c *controller) createLoadBalancer(ctx context.Context, name string, port i return lb, nil } -func (c *controller) updateLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { +func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { outputProperties := map[string]string{} - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) // TODO delete other resources // TODO lb, resp, err := - _, err := c.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + _, err := m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() if err != nil { return nil, err } @@ -129,14 +131,14 @@ func (c *controller) updateLoadBalancer(ctx context.Context, id string, config m return outputProperties, nil } -func (c *controller) deleteLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { +func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { outputProperties := map[string]string{} - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, c.tokenExchanger) + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) // TODO delete other resources // TODO lb, resp, err := - _, err := c.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + _, err := m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() if err != nil { return nil, err } diff --git a/metal/loadbalancers/emlb/metal_token_exchanger.go b/metal/loadbalancers/emlb/infrastructure/token_exchanger.go similarity index 91% rename from metal/loadbalancers/emlb/metal_token_exchanger.go rename to metal/loadbalancers/emlb/infrastructure/token_exchanger.go index b2e5d879..31fab909 100644 --- a/metal/loadbalancers/emlb/metal_token_exchanger.go +++ b/metal/loadbalancers/emlb/infrastructure/token_exchanger.go @@ -1,4 +1,4 @@ -package emlb +package infrastructure import ( "encoding/json" @@ -10,12 +10,12 @@ import ( "golang.org/x/oauth2" ) -type MetalTokenExchanger struct { +type TokenExchanger struct { metalAPIKey string client *http.Client } -func (m *MetalTokenExchanger) Token() (*oauth2.Token, error) { +func (m *TokenExchanger) Token() (*oauth2.Token, error) { tokenExchangeURL := "https://iam.metalctrl.io/api-keys/exchange" tokenExchangeRequest, err := http.NewRequest("POST", tokenExchangeURL, nil) if err != nil { From 5a5df10baec921de908204a7c8ed5e15a1bca86f Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 5 Oct 2023 10:58:24 -0500 Subject: [PATCH 27/48] fix: handle multiple ports for a service --- metal/loadbalancers/emlb/emlb.go | 41 ++++---- .../emlb/infrastructure/manager.go | 94 +++++++++++-------- 2 files changed, 80 insertions(+), 55 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 455a41f7..2762f7e6 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -38,27 +38,12 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n name := rand.String(32) if len(svc.Spec.Ports) < 1 { - return errors.New("cannot add loadbalancer service; no nodeport assigned") + return errors.New("cannot add loadbalancer service; no ports assigned") } - // TODO: Loop through and add ALL the ports - port := svc.Spec.Ports[0].Port - nodePort := svc.Spec.Ports[0].NodePort - var ips []string + pools := l.convertToPools(svc, n) - for _, node := range n { - for _, address := range node.Status.Addresses { - if address.Type == v1.NodeExternalIP { - ips = append(ips, address.Address) - } - } - } - - if len(ips) < 1 { - return errors.New("cannot add loadbalancer service; failed to find an external IP address for any node") - } - // TODO move port looping inside of here so we don't get a new loadbalancer per port - loadBalancer, err := l.manager.CreateLoadBalancer(ctx, name, port, nodePort, ips) + loadBalancer, err := l.manager.CreateLoadBalancer(ctx, name, pools) if err != nil { return err @@ -123,3 +108,23 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no return nil } + +func (l *LB) convertToPools(svc *v1.Service, nodes []*v1.Node) infrastructure.Pools { + pools := infrastructure.Pools{} + for _, svcPort := range svc.Spec.Ports { + targets := []infrastructure.Target{} + for _, node := range nodes { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeExternalIP { + targets = append(targets, infrastructure.Target{ + IP: address.Address, + Port: svcPort.NodePort, + }) + } + } + } + pools[svcPort.Port] = targets + } + + return pools +} diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index f9207128..a2bc77ed 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -16,6 +16,13 @@ var LBMetros = map[string]string{ "sv": "lctnloc-H5rl2M2VL5dcFmdxhbEKx", } +type Pools map[int32][]Target + +type Target struct { + IP string + Port int32 +} + type Manager struct { client *lbaas.APIClient metro string @@ -42,7 +49,7 @@ func (m *Manager) GetMetro() string { return m.metro } -func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, port int32, nodePort int32, ips []string) (*lbaas.LoadBalancer, error) { +func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, pools Pools) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) locationId, ok := LBMetros[m.metro] @@ -64,47 +71,24 @@ func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, port int3 loadBalancerID := lbCreated.GetId() - createPoolRequest := lbaas.LoadBalancerPoolCreate{ - Name: fmt.Sprintf("%v-pool", name), - Protocol: lbaas.LoadBalancerPoolCreateProtocol{ - LoadBalancerPoolProtocol: lbaas.LOADBALANCERPOOLPROTOCOL_TCP.Ptr(), - }, - } - - poolCreated, _, err := m.client.ProjectsApi.CreatePool(ctx, m.projectID).LoadBalancerPoolCreate(createPoolRequest).Execute() - if err != nil { - return nil, err - } - - poolID := poolCreated.GetId() - - for i, ip := range ips { - createOriginRequest := lbaas.LoadBalancerPoolOriginCreate{ - Name: fmt.Sprintf("%v-origin-%v", name, i), - Target: ip, - PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ - Int32: &nodePort, - }, - Active: true, - PoolId: poolID, - } - // TODO do we need the origin IDs for something? - _, _, err := m.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + for externalPort, pool := range pools { + poolName := fmt.Sprintf("%v-pool-%v", name, externalPort) + poolID, err := m.createPool(ctx, poolName, pool) if err != nil { return nil, err } - } - createPortRequest := lbaas.LoadBalancerPortCreate{ - Name: fmt.Sprintf("%v-port-%v", name, port), - Number: port, - PoolIds: []string{poolID}, - } + createPortRequest := lbaas.LoadBalancerPortCreate{ + Name: fmt.Sprintf("%v-port-%v", name, externalPort), + Number: externalPort, + PoolIds: []string{poolID}, + } - // TODO do we need the port ID for something? - _, _, err = m.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() - if err != nil { - return nil, err + // TODO do we need the port ID for something? + _, _, err = m.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() + if err != nil { + return nil, err + } } lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() @@ -145,3 +129,39 @@ func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string, config map[ return outputProperties, nil } + +func (m *Manager) createPool(ctx context.Context, name string, targets []Target) (string, error) { + createPoolRequest := lbaas.LoadBalancerPoolCreate{ + Name: name, + Protocol: lbaas.LoadBalancerPoolCreateProtocol{ + LoadBalancerPoolProtocol: lbaas.LOADBALANCERPOOLPROTOCOL_TCP.Ptr(), + }, + } + + poolCreated, _, err := m.client.ProjectsApi.CreatePool(ctx, m.projectID).LoadBalancerPoolCreate(createPoolRequest).Execute() + + if err != nil { + return "", err + } + + poolID := poolCreated.GetId() + + for i, target := range targets { + createOriginRequest := lbaas.LoadBalancerPoolOriginCreate{ + Name: fmt.Sprintf("%v-origin-%v", name, i), + Target: target.IP, + PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ + Int32: &target.Port, + }, + Active: true, + PoolId: poolID, + } + // TODO do we need the origin IDs for something? + _, _, err := m.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + if err != nil { + return "", err + } + } + + return poolID, nil +} From 7e80f1227b86863d86837e35cf0c3807587e39bf Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Fri, 6 Oct 2023 11:06:20 -0500 Subject: [PATCH 28/48] fix: use a consistent load balancer name --- metal/loadbalancers.go | 7 ++++--- metal/loadbalancers/emlb/emlb.go | 6 ++---- metal/loadbalancers/empty/empty.go | 2 +- metal/loadbalancers/interface.go | 2 +- metal/loadbalancers/kubevip/kubevip.go | 2 +- metal/loadbalancers/metallb/metallb.go | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 64493a53..1e1cec4f 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -173,7 +173,8 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } } else { - ipCidr, err = l.addService(ctx, service, filterNodes(nodes, l.nodeSelector)) + loadBalancerName := l.GetLoadBalancerName(ctx, clusterName, service) + ipCidr, err = l.addService(ctx, service, filterNodes(nodes, l.nodeSelector), loadBalancerName) if err != nil { return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } @@ -385,7 +386,7 @@ func (l *loadBalancers) annotateNode(ctx context.Context, node *v1.Node) error { } // addService add a single service; wraps the implementation -func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, nodes []*v1.Node) (string, error) { +func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, nodes []*v1.Node, loadBalancerName string) (string, error) { svcName := serviceRep(svc) svcTag := serviceTag(svc) svcRegion := serviceAnnotation(svc, l.eipMetroAnnotation) @@ -526,7 +527,7 @@ func (l *loadBalancers) addService(ctx context.Context, svc *v1.Service, nodes [ } } - return svcIPCidr, l.implementor.AddService(ctx, svc.Namespace, svc.Name, svcIPCidr, n, svc, nodes) + return svcIPCidr, l.implementor.AddService(ctx, svc.Namespace, svc.Name, svcIPCidr, n, svc, nodes, loadBalancerName) } func (l *loadBalancers) retrieveIPByTag(ctx context.Context, svc *v1.Service, tag string) (string, error) { diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 2762f7e6..4d30b83a 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -9,7 +9,6 @@ import ( "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/emlb/infrastructure" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" ) @@ -34,8 +33,7 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string return lb } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { - name := rand.String(32) +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { if len(svc.Spec.Ports) < 1 { return errors.New("cannot add loadbalancer service; no ports assigned") @@ -43,7 +41,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n pools := l.convertToPools(svc, n) - loadBalancer, err := l.manager.CreateLoadBalancer(ctx, name, pools) + loadBalancer, err := l.manager.CreateLoadBalancer(ctx, loadBalancerName, pools) if err != nil { return err diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index 38304f30..74865dd1 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -17,7 +17,7 @@ func NewLB(k8sclient kubernetes.Interface, config string) *LB { return &LB{} } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { return nil } diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index 7f112379..cbef2b5d 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -8,7 +8,7 @@ import ( type LB interface { // AddService add a service with the provided name and IP - AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []Node, svc *v1.Service, n []*v1.Node) error + AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error // RemoveService remove service with the given IP RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error // UpdateService ensure that the nodes handled by the service are correct diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index 63a44067..6c58e93a 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -17,7 +17,7 @@ func NewLB(k8sclient kubernetes.Interface, config string) *LB { return &LB{} } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { return nil } diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index f33a4255..73ef4f04 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -116,7 +116,7 @@ func NewLB(k8sclient kubernetes.Interface, config string, featureFlags url.Value return lb } -func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { +func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { config := l.configurer if err := config.Get(ctx); err != nil { return fmt.Errorf("unable to add service: %w", err) From f9ede3eab6a7fdbcf1bea1f45d32288d797e454d Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:38:10 -0500 Subject: [PATCH 29/48] fix: commit svc updates Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 38 ++++++++++++++++++-------------- metal/loadbalancers/emlb/emlb.go | 23 ++++++++++++++++++- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 1e1cec4f..09d3f84d 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -127,27 +127,31 @@ func (l *loadBalancers) GetLoadBalancer(ctx context.Context, clusterName string, svcIP := service.Spec.LoadBalancerIP var svcIPCidr string + if l.usesBGP { + // get IP address reservations and check if they any exists for this svc + ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) + if err != nil { + return nil, false, fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) + } - // get IP address reservations and check if they any exists for this svc - ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) - if err != nil { - return nil, false, fmt.Errorf("unable to retrieve IP reservations for project %s: %w", l.project, err) - } - - ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) + ipReservation := ipReservationByAllTags([]string{svcTag, emTag, clsTag}, ips) - klog.V(2).Infof("GetLoadBalancer(): remove: %s with existing IP assignment %s", svcName, svcIP) + klog.V(2).Infof("GetLoadBalancer(): remove: %s with existing IP assignment %s", svcName, svcIP) - // get the IPs and see if there is anything to clean up - if ipReservation == nil { - return nil, false, nil + // get the IPs and see if there is anything to clean up + if ipReservation == nil { + return nil, false, nil + } + svcIPCidr = fmt.Sprintf("%s/%d", ipReservation.Address, ipReservation.CIDR) + return &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + {IP: svcIPCidr}, + }, + }, true, nil + } else { + // TODO: Actually check if the load balancer exists + return &service.Status.LoadBalancer, &service.Status.LoadBalancer == nil, nil } - svcIPCidr = fmt.Sprintf("%s/%d", ipReservation.Address, ipReservation.CIDR) - return &v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - {IP: svcIPCidr}, - }, - }, true, nil } // GetLoadBalancerName returns the name of the load balancer. Implementations must treat the diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 4d30b83a..c6911b4f 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -9,12 +9,16 @@ import ( "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers/emlb/infrastructure" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config" ) type LB struct { manager *infrastructure.Manager k8sclient kubernetes.Interface + client client.Client } var _ loadbalancers.LB = (*LB)(nil) @@ -26,10 +30,27 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string // it may have an extra slash at the beginning or end, so get rid of it metro := strings.TrimPrefix(config, "/") + // Create a new LB object. lb := &LB{} + + // Set the manager subobject to have the API key and project id and metro. lb.manager = infrastructure.NewManager(metalAPIKey, projectID, metro) + + // Pass the k8sclient into the LB object. lb.k8sclient = k8sclient + // Set up a new controller-runtime k8s client for LB object. + scheme := runtime.NewScheme() + err := v1.AddToScheme(scheme) + if err != nil { + panic(err) + } + newClient, err := client.New(clientconfig.GetConfigOrDie(), client.Options{Scheme: scheme}) + if err != nil { + panic(err) + } + lb.client = newClient + return lb } @@ -62,7 +83,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n svc.Annotations["equinix.com/loadbalancerID"] = loadBalancer.GetId() svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - return nil + return l.client.Update(ctx, svc) } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { From b5b86cf5495b365b7922c566434c926665897a7b Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Fri, 6 Oct 2023 13:31:41 -0500 Subject: [PATCH 30/48] fix: if service already has a load balancer ID, don't create a new one --- metal/loadbalancers/emlb/emlb.go | 20 +++++++++---------- .../emlb/infrastructure/manager.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index c6911b4f..40f1f684 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -21,6 +21,10 @@ type LB struct { client client.Client } +const ( + LoadBalancerIDAnnotation = "equinix.com/loadbalancerID" +) + var _ loadbalancers.LB = (*LB)(nil) func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string) *LB { @@ -55,7 +59,10 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { - + if svc.Annotations[LoadBalancerIDAnnotation] != "" { + // TODO UpdateService will be updated later to accept *v1.Service and []*v1.Nodes + return l.UpdateService(ctx, svcNamespace, svcName, nodes) + } if len(svc.Spec.Ports) < 1 { return errors.New("cannot add loadbalancer service; no ports assigned") } @@ -80,7 +87,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n // TODO: THIS DOES NOT ACTUALLY UPDATE THE SERVICE! svc.Status.LoadBalancer.Ingress = ingress - svc.Annotations["equinix.com/loadbalancerID"] = loadBalancer.GetId() + svc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() return l.client.Update(ctx, svc) @@ -92,7 +99,7 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string additionalProperties := map[string]string{} // 2. Delete the infrastructure (do we need to return anything here?) - _, err := l.manager.DeleteLoadBalancer(ctx, loadBalancerId, additionalProperties) + _, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, additionalProperties) if err != nil { return err @@ -110,15 +117,8 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no - NodePort - Public IP addresses of the nodes on which the target pods are running */ - loadBalancerId := "TODO" - additionalProperties := map[string]string{} // 2. Update infrastructure change (do we need to return anything here? or are all changes reflected by properties from [1]?) - _, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, additionalProperties) - - if err != nil { - return err - } /* 3. Update the annotations diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index a2bc77ed..311b0664 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -107,7 +107,7 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, config map[ // TODO delete other resources // TODO lb, resp, err := - _, err := m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + _, _, err := m.client.LoadBalancersApi.UpdateLoadBalancer(ctx, id).Execute() if err != nil { return nil, err } From 21328f01daaac388977e1531a8267ff2d46627aa Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:44:59 -0500 Subject: [PATCH 31/48] feat: add code for deletes Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- go.mod | 2 +- metal/loadbalancers.go | 3 +- metal/loadbalancers/emlb/emlb.go | 15 +++------- .../emlb/infrastructure/manager.go | 30 ++++++++++++++----- metal/loadbalancers/empty/empty.go | 2 +- metal/loadbalancers/interface.go | 2 +- metal/loadbalancers/kubevip/kubevip.go | 2 +- metal/loadbalancers/metallb/metallb.go | 2 +- 8 files changed, 33 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 25a0b0a4..6a8aed26 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/pallinder/go-randomdata v1.2.0 go.universe.tf/metallb v0.13.7 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b k8s.io/api v0.26.4 k8s.io/apimachinery v0.26.4 k8s.io/client-go v0.26.4 @@ -91,7 +92,6 @@ require ( go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.1.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 09d3f84d..a3808285 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -265,7 +265,6 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa var svcIPCidr string if l.usesBGP { - // get IP address reservations and check if they any exists for this svc ips, _, err := l.client.ProjectIPs.List(l.project, &packngo.ListOptions{}) if err != nil { @@ -291,7 +290,7 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: for %s entry %s", svcName, svcIPCidr) } - if err := l.implementor.RemoveService(ctx, service.Namespace, service.Name, svcIPCidr); err != nil { + if err := l.implementor.RemoveService(ctx, service.Namespace, service.Name, svcIPCidr, service); err != nil { return fmt.Errorf("error removing IP from configmap for %s: %w", svcName, err) } klog.V(2).Infof("EnsureLoadBalancerDeleted(): remove: removed service %s from implementation", svcName) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 40f1f684..5b1d5933 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -93,21 +93,14 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return l.client.Update(ctx, svc) } -func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { +func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { // 1. Gather the properties we need: ID of load balancer - loadBalancerId := "TODO" - additionalProperties := map[string]string{} + loadBalancerId := svc.Annotations[LoadBalancerIDAnnotation] // 2. Delete the infrastructure (do we need to return anything here?) - _, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, additionalProperties) + err := l.manager.DeleteLoadBalancer(ctx, loadBalancerId) - if err != nil { - return err - } - - // 3. No need to remove the annotations because the annotated object was deleted - - return nil + return err } func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index 311b0664..326972b8 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -49,6 +49,27 @@ func (m *Manager) GetMetro() string { return m.metro } +func (m *Manager) GetLoadBalancer(ctx context.Context, id string) (*lbaas.LoadBalancerCollection, error) { + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) + + LoadBalancers, _, err := m.client.ProjectsApi.ListLoadBalancers(ctx, m.projectID).Execute() + return LoadBalancers, err +} + +func (m *Manager) GetLoadBalancers(ctx context.Context) (*lbaas.LoadBalancerCollection, error) { + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) + + LoadBalancers, _, err := m.client.ProjectsApi.ListLoadBalancers(ctx, m.projectID).Execute() + return LoadBalancers, err +} + +func (m *Manager) GetPools(ctx context.Context) (*lbaas.LoadBalancerPoolCollection, error) { + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) + + LoadBalancerPools, _, err := m.client.ProjectsApi.ListPools(ctx, m.projectID).Execute() + return LoadBalancerPools, err +} + func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, pools Pools) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) @@ -115,19 +136,14 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, config map[ return outputProperties, nil } -func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { - outputProperties := map[string]string{} +func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) // TODO delete other resources // TODO lb, resp, err := _, err := m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() - if err != nil { - return nil, err - } - - return outputProperties, nil + return err } func (m *Manager) createPool(ctx context.Context, name string, targets []Target) (string, error) { diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index 74865dd1..192bb8ad 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -21,7 +21,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return nil } -func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { +func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { return nil } diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index cbef2b5d..fb5fe2e2 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -10,7 +10,7 @@ type LB interface { // AddService add a service with the provided name and IP AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error // RemoveService remove service with the given IP - RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error + RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error // UpdateService ensure that the nodes handled by the service are correct UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []Node) error } diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index 6c58e93a..cbb230ff 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -21,7 +21,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return nil } -func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { +func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { return nil } diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index 73ef4f04..d8feef63 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -132,7 +132,7 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return nil } -func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string) error { +func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { config := l.configurer if err := config.Get(ctx); err != nil { return fmt.Errorf("unable to remove service: %w", err) From 937350f62a3d7dd374b8cc4ced04358b00b63dd8 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Mon, 9 Oct 2023 16:55:12 -0500 Subject: [PATCH 32/48] feat: add interface function so deletes work Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 12 ++++++++++-- metal/loadbalancers/emlb/emlb.go | 11 +++++++++++ metal/loadbalancers/emlb/infrastructure/manager.go | 8 +++++--- metal/loadbalancers/empty/empty.go | 3 +++ metal/loadbalancers/interface.go | 2 ++ metal/loadbalancers/kubevip/kubevip.go | 3 +++ metal/loadbalancers/metallb/metallb.go | 4 ++++ 7 files changed, 38 insertions(+), 5 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index a3808285..ef6a1e18 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -150,8 +150,17 @@ func (l *loadBalancers) GetLoadBalancer(ctx context.Context, clusterName string, }, true, nil } else { // TODO: Actually check if the load balancer exists - return &service.Status.LoadBalancer, &service.Status.LoadBalancer == nil, nil + lbList, err := l.implementor.GetLoadBalancerList(ctx) + if err != nil { + return nil, false, fmt.Errorf("unable to retrieve load balancers for project %s: %w", l.project, err) + } + for i := range lbList { + if lbList[i] == l.GetLoadBalancerName(ctx, clusterName, service) { + return &service.Status.LoadBalancer, true, nil + } + } } + return nil, false, nil } // GetLoadBalancerName returns the name of the load balancer. Implementations must treat the @@ -299,7 +308,6 @@ func (l *loadBalancers) EnsureLoadBalancerDeleted(ctx context.Context, clusterNa } // utility funcs - // annotateNode ensure a node has the correct annotations. func (l *loadBalancers) annotateNode(ctx context.Context, node *v1.Node) error { klog.V(2).Infof("annotateNode: %s", node.Name) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 5b1d5933..c7c64105 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -140,3 +140,14 @@ func (l *LB) convertToPools(svc *v1.Service, nodes []*v1.Node) infrastructure.Po return pools } +func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { + var lbList []string + + lbCollection, err := l.manager.GetLoadBalancers(ctx) + + for _, lb := range lbCollection.Loadbalancers { + lbList = append(lbList, lb.Name) + } + + return lbList, err +} diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index 326972b8..0451400b 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -49,13 +49,15 @@ func (m *Manager) GetMetro() string { return m.metro } -func (m *Manager) GetLoadBalancer(ctx context.Context, id string) (*lbaas.LoadBalancerCollection, error) { +// Returns a Load Balancer object given an id +func (m *Manager) GetLoadBalancer(ctx context.Context, id string) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - LoadBalancers, _, err := m.client.ProjectsApi.ListLoadBalancers(ctx, m.projectID).Execute() - return LoadBalancers, err + LoadBalancer, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() + return LoadBalancer, err } +// Returns a list of Load Balancer objects in the project func (m *Manager) GetLoadBalancers(ctx context.Context) (*lbaas.LoadBalancerCollection, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index 192bb8ad..5249ba38 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -28,3 +28,6 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { return nil } +func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { + return []string{}, nil +} diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index fb5fe2e2..2bae66d0 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -13,4 +13,6 @@ type LB interface { RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error // UpdateService ensure that the nodes handled by the service are correct UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []Node) error + // GetLoadBalancerList returns the load balancer objects + GetLoadBalancerList(ctx context.Context) ([]string, error) } diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index cbb230ff..67c29780 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -28,3 +28,6 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { return nil } +func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { + return []string{}, nil +} diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index d8feef63..93500fb2 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -165,6 +165,10 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no return nil } +func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { + return []string{}, nil +} + // updateNodes add/delete one or more nodes with the provided name, srcIP, and bgp information func (l *LB) updateNodes(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { config := l.configurer From 3b137eea55eb5d40cfc250501b1c5fd8b0da165b Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 10 Oct 2023 11:39:54 -0500 Subject: [PATCH 33/48] refactor: defer implementation of GetLoadBalancer to EMLB To avoid adding knowledge of specific load balancer implementers in `loadbalancers.go`, this moves the `GetLoadBalancer` logic for EMLB into `emlb.go`. In the future we can refactor the pre-existing `GetLoadBalancer` code in order to push that down to `metallb`, `kubevip`, `empty`, etc., keeping `loadbalancers.go` focused on the work of deciding which implementer to use and how to translate from the cloud-provider interface to our internal implementer interface. --- metal/loadbalancers.go | 12 ++---------- metal/loadbalancers/emlb/emlb.go | 20 ++++++++++++++------ metal/loadbalancers/empty/empty.go | 6 ++++-- metal/loadbalancers/interface.go | 4 ++-- metal/loadbalancers/kubevip/kubevip.go | 6 ++++-- metal/loadbalancers/metallb/metallb.go | 5 +++-- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index ef6a1e18..88807e55 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -149,17 +149,9 @@ func (l *loadBalancers) GetLoadBalancer(ctx context.Context, clusterName string, }, }, true, nil } else { - // TODO: Actually check if the load balancer exists - lbList, err := l.implementor.GetLoadBalancerList(ctx) - if err != nil { - return nil, false, fmt.Errorf("unable to retrieve load balancers for project %s: %w", l.project, err) - } - for i := range lbList { - if lbList[i] == l.GetLoadBalancerName(ctx, clusterName, service) { - return &service.Status.LoadBalancer, true, nil - } - } + return l.implementor.GetLoadBalancer(ctx, clusterName, service) } + return nil, false, nil } diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index c7c64105..8d5327ec 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -4,6 +4,7 @@ package emlb import ( "context" "errors" + "fmt" "strings" "github.com/equinix/cloud-provider-equinix-metal/metal/loadbalancers" @@ -140,14 +141,21 @@ func (l *LB) convertToPools(svc *v1.Service, nodes []*v1.Node) infrastructure.Po return pools } -func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { - var lbList []string +func (l *LB) GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) { + loadBalancerId := svc.Annotations[LoadBalancerIDAnnotation] + + if loadBalancerId != "" { + // TODO probably need to check if err is 404, maybe others? + lb, err := l.manager.GetLoadBalancer(ctx, loadBalancerId) - lbCollection, err := l.manager.GetLoadBalancers(ctx) + if err != nil { + return nil, false, fmt.Errorf("unable to retrieve load balancer: %w", err) + } - for _, lb := range lbCollection.Loadbalancers { - lbList = append(lbList, lb.Name) + if lb != nil { + return &svc.Status.LoadBalancer, true, nil + } } - return lbList, err + return nil, false, nil } diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index 5249ba38..0790bee0 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -28,6 +28,8 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { return nil } -func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { - return []string{}, nil + +func (l *LB) GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) { + // TODO + return nil, false, nil } diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index 2bae66d0..02f98046 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -13,6 +13,6 @@ type LB interface { RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error // UpdateService ensure that the nodes handled by the service are correct UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []Node) error - // GetLoadBalancerList returns the load balancer objects - GetLoadBalancerList(ctx context.Context) ([]string, error) + // GetLoadBalancer implements cloudprovider.GetLoadBalancer + GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) } diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index 67c29780..3642e2c5 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -28,6 +28,8 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { return nil } -func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { - return []string{}, nil + +func (l *LB) GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) { + // TODO + return nil, false, nil } diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index 93500fb2..e16a0770 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -165,8 +165,9 @@ func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, no return nil } -func (l *LB) GetLoadBalancerList(ctx context.Context) ([]string, error) { - return []string{}, nil +func (l *LB) GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) { + // TODO + return nil, false, nil } // updateNodes add/delete one or more nodes with the provided name, srcIP, and bgp information From adcfa0050aed96a2303f24706feea1c8661c4cb2 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 10 Oct 2023 11:04:03 -0500 Subject: [PATCH 34/48] fix: delete pools when load balancer is deleted --- .../emlb/infrastructure/manager.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index 0451400b..6e310711 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -141,10 +141,24 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, config map[ func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - // TODO delete other resources + lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() + + if err != nil { + return err + } + + for _, poolGroups := range lb.Pools { + for _, pool := range poolGroups { + _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, pool.GetId().(string)).Execute() + if err != nil { + return err + } + } + + } // TODO lb, resp, err := - _, err := m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + _, err = m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() return err } From a442432decc7bc4e74c08b0cdc6865cbbe251f08 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 10 Oct 2023 11:04:09 -0500 Subject: [PATCH 35/48] fix: patch service object instead of updating When deploying a new LoadBalancer service, we would see a "Failed to update..." error message in the logs, and 2 load balancers would be created on the Equinix side, with only 1 of them reflected in the service annotations. This changes our `Update` call to update the k8s service object with a `Patch` call; by doing this we focus our k8s API calls on changing only the properties that our provider has modified, which reduces the chance that something else has concurrently made conflicting changes. --- metal/loadbalancers/emlb/emlb.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 8d5327ec..d80b09b2 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -85,13 +85,15 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n svc.Spec.LoadBalancerIP = ip } - // TODO: THIS DOES NOT ACTUALLY UPDATE THE SERVICE! - svc.Status.LoadBalancer.Ingress = ingress + // Per cloud-provider docs, we have to treat `svc` as read-only + updatedSvc := svc.DeepCopy() + updatedSvc.Status.LoadBalancer.Ingress = ingress - svc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() - svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() + updatedSvc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() + updatedSvc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - return l.client.Update(ctx, svc) + patch := client.MergeFrom(svc) + return l.client.Patch(ctx, updatedSvc, patch) } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { From 590d80b4c393bc54086343af97e0d3c8d3caf2ab Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:53:11 -0500 Subject: [PATCH 36/48] feat: report status correctly for new LB Co-authored-by: Charles Treatman Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- metal/loadbalancers.go | 26 ++++++++++++----------- metal/loadbalancers/emlb/emlb.go | 36 +++++++++++++++----------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index 88807e55..da44d949 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -151,8 +151,6 @@ func (l *loadBalancers) GetLoadBalancer(ctx context.Context, clusterName string, } else { return l.implementor.GetLoadBalancer(ctx, clusterName, service) } - - return nil, false, nil } // GetLoadBalancerName returns the name of the load balancer. Implementations must treat the @@ -167,25 +165,22 @@ func (l *loadBalancers) GetLoadBalancerName(ctx context.Context, clusterName str // Implementations must treat the *v1.Service and *v1.Node // parameters as read-only and not modify them. // Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager -func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { +func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName string, readOnlyService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { + service := readOnlyService.DeepCopy() klog.V(2).Infof("EnsureLoadBalancer(): add: service %s/%s", service.Namespace, service.Name) var ipCidr string var err error + + // TODO: Split out most of this to "reconcileLoadBalancer" + // TODO: Split out status checking to a separate function that reconcileLoadBalancer calls + // handling is completely different if it is the control plane vs a regular service of type=LoadBalancer if service.Name == externalServiceName && service.Namespace == externalServiceNamespace { ipCidr, err = l.retrieveIPByTag(ctx, service, l.eipTag) if err != nil { return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) } - } else { - loadBalancerName := l.GetLoadBalancerName(ctx, clusterName, service) - ipCidr, err = l.addService(ctx, service, filterNodes(nodes, l.nodeSelector), loadBalancerName) - if err != nil { - return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) - } - } - if l.usesBGP { // get the IP only ip := strings.SplitN(ipCidr, "/", 2) @@ -195,8 +190,15 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri }, }, nil } else { - return &service.Status.LoadBalancer, nil + loadBalancerName := l.GetLoadBalancerName(ctx, clusterName, service) + _, err = l.addService(ctx, service, filterNodes(nodes, l.nodeSelector), loadBalancerName) + if err != nil { + return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) + } } + + status, _, err := l.GetLoadBalancer(ctx, clusterName, service) + return status, err } // UpdateLoadBalancer updates hosts under the specified load balancer. diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index d80b09b2..b8a45b27 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -76,24 +76,12 @@ func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, n return err } - var ingress []v1.LoadBalancerIngress - for _, ip := range loadBalancer.GetIps() { - ingress = append(ingress, v1.LoadBalancerIngress{ - IP: ip, - }) - // TODO: this is here for backwards compatibility and should be removed ASAP - svc.Spec.LoadBalancerIP = ip - } - - // Per cloud-provider docs, we have to treat `svc` as read-only - updatedSvc := svc.DeepCopy() - updatedSvc.Status.LoadBalancer.Ingress = ingress + patch := client.MergeFrom(svc.DeepCopy()) - updatedSvc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() - updatedSvc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() + svc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() + svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - patch := client.MergeFrom(svc) - return l.client.Patch(ctx, updatedSvc, patch) + return l.client.Patch(ctx, svc, patch) } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { @@ -148,14 +136,24 @@ func (l *LB) GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Se if loadBalancerId != "" { // TODO probably need to check if err is 404, maybe others? - lb, err := l.manager.GetLoadBalancer(ctx, loadBalancerId) + loadBalancer, err := l.manager.GetLoadBalancer(ctx, loadBalancerId) if err != nil { return nil, false, fmt.Errorf("unable to retrieve load balancer: %w", err) } - if lb != nil { - return &svc.Status.LoadBalancer, true, nil + if loadBalancer != nil { + var ingress []v1.LoadBalancerIngress + for _, ip := range loadBalancer.GetIps() { + ingress = append(ingress, v1.LoadBalancerIngress{ + IP: ip, + }) + } + + loadBalancerStatus := v1.LoadBalancerStatus{ + Ingress: ingress, + } + return &loadBalancerStatus, true, nil } } From d4507a3a6a08c0fa48de0b8c70062cb504df3b52 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Wed, 11 Oct 2023 15:57:39 -0500 Subject: [PATCH 37/48] Wire up UpdateService for EMLB --- metal/loadbalancers.go | 2 +- metal/loadbalancers/emlb/emlb.go | 43 +++--- .../emlb/infrastructure/manager.go | 124 ++++++++++++++---- metal/loadbalancers/empty/empty.go | 2 +- metal/loadbalancers/interface.go | 2 +- metal/loadbalancers/kubevip/kubevip.go | 2 +- metal/loadbalancers/metallb/metallb.go | 2 +- 7 files changed, 133 insertions(+), 44 deletions(-) diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index da44d949..e8fa0974 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -247,7 +247,7 @@ func (l *loadBalancers) UpdateLoadBalancer(ctx context.Context, clusterName stri } } - return l.implementor.UpdateService(ctx, service.Namespace, service.Name, n) + return l.implementor.UpdateService(ctx, service.Namespace, service.Name, n, service, nodes) } // EnsureLoadBalancerDeleted deletes the specified load balancer if it diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index b8a45b27..6ba9d371 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -61,8 +61,7 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { if svc.Annotations[LoadBalancerIDAnnotation] != "" { - // TODO UpdateService will be updated later to accept *v1.Service and []*v1.Nodes - return l.UpdateService(ctx, svcNamespace, svcName, nodes) + return l.UpdateService(ctx, svcNamespace, svcName, nodes, svc, n) } if len(svc.Spec.Ports) < 1 { return errors.New("cannot add loadbalancer service; no ports assigned") @@ -94,22 +93,36 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string return err } -func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { - /* - 1. Gather the properties we need: - - load balancer ID - - NodePort - - Public IP addresses of the nodes on which the target pods are running - */ +func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { + // 1. Gather the properties we need: ID of load balancer + loadBalancerId := svc.Annotations[LoadBalancerIDAnnotation] + + pools := l.convertToPools(svc, n) + + loadBalancer, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, pools) + + if err != nil { + return err + } + + var ingress []v1.LoadBalancerIngress + for _, ip := range loadBalancer.GetIps() { + ingress = append(ingress, v1.LoadBalancerIngress{ + IP: ip, + }) + // TODO: this is here for backwards compatibility and should be removed ASAP + svc.Spec.LoadBalancerIP = ip + } - // 2. Update infrastructure change (do we need to return anything here? or are all changes reflected by properties from [1]?) + // Per cloud-provider docs, we have to treat `svc` as read-only + updatedSvc := svc.DeepCopy() + updatedSvc.Status.LoadBalancer.Ingress = ingress - /* - 3. Update the annotations - - Listener port that this service is using - */ + updatedSvc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() + updatedSvc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - return nil + patch := client.MergeFrom(svc) + return l.client.Patch(ctx, updatedSvc, patch) } func (l *LB) convertToPools(svc *v1.Service, nodes []*v1.Node) infrastructure.Pools { diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index 6e310711..a847236c 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -3,6 +3,7 @@ package infrastructure import ( "context" "fmt" + "net/http" "reflect" lbaas "github.com/equinix/cloud-provider-equinix-metal/internal/lbaas/v1" @@ -95,14 +96,14 @@ func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, pools Poo loadBalancerID := lbCreated.GetId() for externalPort, pool := range pools { - poolName := fmt.Sprintf("%v-pool-%v", name, externalPort) + poolName := getResourceName(name, "pool", externalPort) poolID, err := m.createPool(ctx, poolName, pool) if err != nil { return nil, err } createPortRequest := lbaas.LoadBalancerPortCreate{ - Name: fmt.Sprintf("%v-port-%v", name, externalPort), + Name: getResourceName(name, "port", externalPort), Number: externalPort, PoolIds: []string{poolID}, } @@ -115,27 +116,93 @@ func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, pools Poo } lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() - if err != nil { - return nil, err - } - return lb, nil + return lb, err } -func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, config map[string]string) (map[string]string, error) { - outputProperties := map[string]string{} - +func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools) (*lbaas.LoadBalancer, error) { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - // TODO delete other resources - - // TODO lb, resp, err := - _, _, err := m.client.LoadBalancersApi.UpdateLoadBalancer(ctx, id).Execute() + lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() if err != nil { return nil, err } - return outputProperties, nil + existingPorts := map[int32]struct{}{} + + // Update or delete existing targets + for _, port := range lb.GetPorts() { + portNumber := port.GetNumber() + existingPoolIds := port.GetPoolIds() + targets, wanted := pools[portNumber] + if wanted { + // We have a pool for this port and we want to keep it + existingPorts[portNumber] = struct{}{} + + for _, poolID := range existingPoolIds { + existingPool, _, err := m.client.PoolsApi.GetLoadBalancerPool(ctx, poolID).Execute() + if err != nil { + return nil, err + } + + // TODO: can/should we be more granular here? figure out which to add and which to update? + + // Create new origins for all targets + for i, target := range targets { + _, _, err := m.createOrigin(ctx, poolID, existingPool.GetName(), int32(i), target) + if err != nil { + return nil, err + } + } + + // Delete old origins (some of which may be duplicates of the new ones) + for _, origin := range existingPool.Origins { + _, err := m.client.OriginsApi.DeleteLoadBalancerOrigin(ctx, origin.GetId()).Execute() + if err != nil { + return nil, err + } + } + } + } else { + // We have a pool for this port and we want to get rid of it + for _, poolID := range existingPoolIds { + _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, poolID).Execute() + if err != nil { + return nil, err + } + } + _, err := m.client.PortsApi.DeleteLoadBalancerPort(ctx, port.GetId()).Execute() + if err != nil { + return nil, err + } + } + } + + // Create ports & pools for new targets + for externalPort, pool := range pools { + if _, exists := existingPorts[externalPort]; !exists { + poolID, err := m.createPool(ctx, getResourceName(lb.GetName(), "pool", externalPort), pool) + if err != nil { + return nil, err + } + + createPortRequest := lbaas.LoadBalancerPortCreate{ + Name: getResourceName(lb.GetName(), "port", externalPort), + Number: externalPort, + PoolIds: []string{poolID}, + } + + // TODO do we need the port ID for something? + _, _, err = m.client.PortsApi.CreateLoadBalancerPort(ctx, id).LoadBalancerPortCreate(createPortRequest).Execute() + if err != nil { + return nil, err + } + } + } + + lb, _, err = m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() + + return lb, err } func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { @@ -179,17 +246,8 @@ func (m *Manager) createPool(ctx context.Context, name string, targets []Target) poolID := poolCreated.GetId() for i, target := range targets { - createOriginRequest := lbaas.LoadBalancerPoolOriginCreate{ - Name: fmt.Sprintf("%v-origin-%v", name, i), - Target: target.IP, - PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ - Int32: &target.Port, - }, - Active: true, - PoolId: poolID, - } // TODO do we need the origin IDs for something? - _, _, err := m.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + _, _, err := m.createOrigin(ctx, poolID, name, int32(i), target) if err != nil { return "", err } @@ -197,3 +255,21 @@ func (m *Manager) createPool(ctx context.Context, name string, targets []Target) return poolID, nil } + +func (m *Manager) createOrigin(ctx context.Context, poolID, poolName string, number int32, target Target) (*lbaas.ResourceCreatedResponse, *http.Response, error) { + createOriginRequest := lbaas.LoadBalancerPoolOriginCreate{ + Name: getResourceName(poolName, "origin", number), + Target: target.IP, + PortNumber: lbaas.LoadBalancerPoolOriginPortNumber{ + Int32: &target.Port, + }, + Active: true, + PoolId: poolID, + } + return m.client.PoolsApi.CreateLoadBalancerPoolOrigin(ctx, poolID).LoadBalancerPoolOriginCreate(createOriginRequest).Execute() + +} + +func getResourceName(loadBalancerName, resourceType string, number int32) string { + return fmt.Sprintf("%v-%v-%v", loadBalancerName, resourceType, number) +} diff --git a/metal/loadbalancers/empty/empty.go b/metal/loadbalancers/empty/empty.go index 0790bee0..ccc83492 100644 --- a/metal/loadbalancers/empty/empty.go +++ b/metal/loadbalancers/empty/empty.go @@ -25,7 +25,7 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string return nil } -func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { +func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { return nil } diff --git a/metal/loadbalancers/interface.go b/metal/loadbalancers/interface.go index 02f98046..ea51b6a9 100644 --- a/metal/loadbalancers/interface.go +++ b/metal/loadbalancers/interface.go @@ -12,7 +12,7 @@ type LB interface { // RemoveService remove service with the given IP RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error // UpdateService ensure that the nodes handled by the service are correct - UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []Node) error + UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []Node, svc *v1.Service, n []*v1.Node) error // GetLoadBalancer implements cloudprovider.GetLoadBalancer GetLoadBalancer(ctx context.Context, clusterName string, svc *v1.Service) (*v1.LoadBalancerStatus, bool, error) } diff --git a/metal/loadbalancers/kubevip/kubevip.go b/metal/loadbalancers/kubevip/kubevip.go index 3642e2c5..252f2c40 100644 --- a/metal/loadbalancers/kubevip/kubevip.go +++ b/metal/loadbalancers/kubevip/kubevip.go @@ -25,7 +25,7 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string return nil } -func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { +func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { return nil } diff --git a/metal/loadbalancers/metallb/metallb.go b/metal/loadbalancers/metallb/metallb.go index e16a0770..eb03bb64 100644 --- a/metal/loadbalancers/metallb/metallb.go +++ b/metal/loadbalancers/metallb/metallb.go @@ -157,7 +157,7 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string return nil } -func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node) error { +func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { // ensure nodes are correct if err := l.updateNodes(ctx, svcNamespace, svcName, nodes); err != nil { return fmt.Errorf("failed to add nodes: %w", err) From d35cae53e45d4cc5f20dfc60fd2522e00cf4dafc Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Thu, 12 Oct 2023 16:07:02 -0500 Subject: [PATCH 38/48] change ID properties to strings --- metal/loadbalancers/emlb/infrastructure/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index a847236c..b03021a7 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -216,7 +216,7 @@ func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { for _, poolGroups := range lb.Pools { for _, pool := range poolGroups { - _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, pool.GetId().(string)).Execute() + _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, pool.GetId()).Execute() if err != nil { return err } From 279d0fa7df00860b5c3079a98dba8aacf17884f2 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Fri, 13 Oct 2023 10:10:33 -0500 Subject: [PATCH 39/48] fix: actually delete old resources when updating a service After inspecting API responses in the Equinix Metal console, I saw that origin pool data is not nested under the corresponding port. Instead, `ports` is an array of ports, and `pools` is an array of arrays of pools. The top-level index of `pools` corresponds to the top-level index of `ports`, so `pools[i]` is an array of all of the origin pools assigned to `ports[i]`. The old code was looking for origin data in the wrong place, so it was failing to delete origins that are no longer needed. Now it is looking in the right place and successfully deleting old origins. --- .../emlb/infrastructure/manager.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index b03021a7..0fad4838 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -129,18 +129,17 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools } existingPorts := map[int32]struct{}{} - + existingPools := lb.GetPools() // Update or delete existing targets - for _, port := range lb.GetPorts() { + for i, port := range lb.GetPorts() { portNumber := port.GetNumber() - existingPoolIds := port.GetPoolIds() targets, wanted := pools[portNumber] if wanted { // We have a pool for this port and we want to keep it existingPorts[portNumber] = struct{}{} - for _, poolID := range existingPoolIds { - existingPool, _, err := m.client.PoolsApi.GetLoadBalancerPool(ctx, poolID).Execute() + for _, existingPool := range existingPools[i] { + existingOrigins, _, err := m.client.PoolsApi.ListLoadBalancerPoolOrigins(ctx, existingPool.GetId()).Execute() if err != nil { return nil, err } @@ -148,15 +147,15 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools // TODO: can/should we be more granular here? figure out which to add and which to update? // Create new origins for all targets - for i, target := range targets { - _, _, err := m.createOrigin(ctx, poolID, existingPool.GetName(), int32(i), target) + for j, target := range targets { + _, _, err := m.createOrigin(ctx, existingPool.GetId(), existingPool.GetName(), int32(j), target) if err != nil { return nil, err } } // Delete old origins (some of which may be duplicates of the new ones) - for _, origin := range existingPool.Origins { + for _, origin := range existingOrigins.GetOrigins() { _, err := m.client.OriginsApi.DeleteLoadBalancerOrigin(ctx, origin.GetId()).Execute() if err != nil { return nil, err @@ -165,8 +164,8 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools } } else { // We have a pool for this port and we want to get rid of it - for _, poolID := range existingPoolIds { - _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, poolID).Execute() + for _, existingPool := range existingPools[i] { + _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, existingPool.GetId()).Execute() if err != nil { return nil, err } From 540c751496a2946f53b0601bee0118f6eeabc54f Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:37:46 -0500 Subject: [PATCH 40/48] chore: bump go mods and version Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- Dockerfile | 2 +- go.mod | 131 ++++++++-------- go.sum | 452 ++++++++++++++++++++++------------------------------- 3 files changed, 251 insertions(+), 334 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bb8f599..2addd3cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # Build the manager binary -ARG GOVER=1.19 +ARG GOVER=1.21 FROM --platform=$BUILDPLATFORM golang:${GOVER} as builder ARG TARGETPLATFORM diff --git a/go.mod b/go.mod index 6a8aed26..593d01ad 100644 --- a/go.mod +++ b/go.mod @@ -1,84 +1,84 @@ module github.com/equinix/cloud-provider-equinix-metal -go 1.19 +go 1.21 require ( - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/hashicorp/go-retryablehttp v0.7.4 github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb github.com/packethost/packngo v0.30.0 github.com/pallinder/go-randomdata v1.2.0 - go.universe.tf/metallb v0.13.7 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b - k8s.io/api v0.26.4 - k8s.io/apimachinery v0.26.4 - k8s.io/client-go v0.26.4 - k8s.io/cloud-provider v0.26.4 - k8s.io/component-base v0.26.4 + github.com/spf13/pflag v1.0.5 + go.universe.tf/metallb v0.13.11 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/oauth2 v0.13.0 + k8s.io/api v0.28.2 + k8s.io/apimachinery v0.28.2 + k8s.io/client-go v0.28.2 + k8s.io/cloud-provider v0.28.2 + k8s.io/component-base v0.28.2 k8s.io/klog/v2 v2.100.1 - sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/controller-runtime v0.16.2 sigs.k8s.io/yaml v1.3.0 ) require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/cel-go v0.12.6 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/cel-go v0.16.1 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect - github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/spf13/cobra v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.9 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect + go.etcd.io/etcd/client/v3 v3.5.9 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect go.opentelemetry.io/otel v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect @@ -87,33 +87,34 @@ require ( go.opentelemetry.io/otel/sdk v1.10.0 // indirect go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.25.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.16.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.1 // indirect - k8s.io/apiserver v0.26.4 // indirect - k8s.io/component-helpers v0.26.4 // indirect - k8s.io/controller-manager v0.26.4 // indirect - k8s.io/kms v0.26.4 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/apiextensions-apiserver v0.28.0 // indirect + k8s.io/apiserver v0.28.2 // indirect + k8s.io/component-helpers v0.28.2 // indirect + k8s.io/controller-manager v0.28.2 // indirect + k8s.io/kms v0.28.2 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 7806c7d5..1a8dbe18 100644 --- a/go.sum +++ b/go.sum @@ -13,13 +13,17 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -34,35 +38,29 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -73,81 +71,68 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -178,15 +163,17 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= -github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.16.1 h1:3hZfSNiAU3KOiNtxuFXVp5WFy4hf/Ly3Sa4/7F8SXNo= +github.com/google/cel-go v0.16.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -194,14 +181,13 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -211,16 +197,20 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -236,148 +226,120 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb h1:Tq95IyiReWp+eWNco4KoDNHIALQbNHFg5dD4TELEg9o= github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb/go.mod h1:xX9d7NVrCzFoIFdy5hav33pCMZwiE37ZKgd1XM9qMEY= github.com/packethost/packngo v0.30.0 h1:JVeTwbXXETsLTDQncUbYwIFpkOp/xevXrffM2HrFECI= github.com/packethost/packngo v0.30.0/go.mod h1:BT/XcdwLVmeMtGPbovnxCpnI1s9ylSE1cs/7pq007NE= github.com/pallinder/go-randomdata v1.2.0 h1:EJPxw+sgM1mbMW8RBu5zkG4FbloJpDOCCqPccdWto8A= github.com/pallinder/go-randomdata v1.2.0/go.mod h1:p8CasZQiWDLuaKy5ihVvdshqc7LlL6ovmRxAuNXgi3U= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= +go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= +go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= +go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= +go.etcd.io/etcd/client/v2 v2.305.9 h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo= +go.etcd.io/etcd/client/v2 v2.305.9/go.mod h1:0NBdNx9wbxtEQLwAQtrDHwx58m02vXpDcgSYI2seohQ= +go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= +go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= +go.etcd.io/etcd/pkg/v3 v3.5.9 h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ= +go.etcd.io/etcd/pkg/v3 v3.5.9/go.mod h1:BZl0SAShQFk0IpLWR78T/+pyt8AruMHhTNNX73hkNVY= +go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= +go.etcd.io/etcd/raft/v3 v3.5.9/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg= +go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= +go.etcd.io/etcd/server/v3 v3.5.9/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -385,8 +347,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 h1:sxoY9kG1s1WpSYNyzm24rlwH4lnRYFXUVVBmKMBfRgw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= @@ -404,29 +366,22 @@ go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/A go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.universe.tf/metallb v0.13.7 h1:K/FS/ecpE/a0l5di8ZbFAXB7UP37S8vnrdLGnleiVGk= -go.universe.tf/metallb v0.13.7/go.mod h1:VWu3zLDSHfT3+U9qfV2vr3SJJmIRy133ttd1TTVGyUA= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.universe.tf/metallb v0.13.11 h1:NkUP2sEWjr3KSIWabRRg66vQ3VXh/O/nk5Q3vH807D0= +go.universe.tf/metallb v0.13.11/go.mod h1:gWM27UgpVVS5OevgfBcGeyKVX6Cvezza34z5iKaArvE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -437,8 +392,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -451,7 +406,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -460,10 +414,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -471,7 +423,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -491,20 +442,16 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -514,17 +461,12 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -533,7 +475,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -546,39 +487,26 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -595,7 +523,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -628,13 +555,14 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -689,11 +617,13 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -708,13 +638,10 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -728,36 +655,26 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -765,43 +682,42 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.4 h1:qSG2PmtcD23BkYiWfoYAcak870eF/hE7NNYBYavTT94= -k8s.io/api v0.26.4/go.mod h1:WwKEXU3R1rgCZ77AYa7DFksd9/BAIKyOmRlbVxgvjCk= -k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.4 h1:rZccKdBLg9vP6J09JD+z8Yr99Ce8gk3Lbi9TCx05Jzs= -k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/apiserver v0.26.4 h1:3Oq4mnJv0mzVX7BR/Nod+8KjlELf/3Ljvu9ZWDyLUoA= -k8s.io/apiserver v0.26.4/go.mod h1:yAY3O1vBM4/0OIGAGeWcdfzQvgdwJ188VirLcuSAVnw= -k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= -k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= -k8s.io/cloud-provider v0.26.4 h1:mqN4vhC4mRoMi+ujI92ImkIOuYS7ZS55FvXB10d6Wp4= -k8s.io/cloud-provider v0.26.4/go.mod h1:F9xY0PvBuZDuGIHOM28dNiPLHxQnWfsiUuCSUikHevo= -k8s.io/component-base v0.26.4 h1:Bg2xzyXNKL3eAuiTEu3XE198d6z22ENgFgGQv2GGOUk= -k8s.io/component-base v0.26.4/go.mod h1:lTuWL1Xz/a4e80gmIC3YZG2JCO4xNwtKWHJWeJmsq20= -k8s.io/component-helpers v0.26.4 h1:qbZrh8QmfL+Yn7lWEI/BPrvITGgkBy33djP5Tzsu2hA= -k8s.io/component-helpers v0.26.4/go.mod h1:2Siz5eWmaKu0khASXMTCfJuASZAbCPX9mtjlCe5IWRs= -k8s.io/controller-manager v0.26.4 h1:SeOHV55WKqCa5HQfPHjMpfSPzJNblDvVDzfNgbQlSdQ= -k8s.io/controller-manager v0.26.4/go.mod h1:HJPU8OKTI8YhrtnvpuFdllK1QCQfibhJXDToDzCEsnQ= +k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= +k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= +k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= +k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= +k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= +k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= +k8s.io/apiserver v0.28.2 h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI= +k8s.io/apiserver v0.28.2/go.mod h1:f7D5e8wH8MWcKD7azq6Csw9UN+CjdtXIVQUyUhrtb+E= +k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= +k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/cloud-provider v0.28.2 h1:9qsYm86hm4bnPgZbl9LE29Zfgjuq3NZR2dgtPioJ40s= +k8s.io/cloud-provider v0.28.2/go.mod h1:40fqf6MtgYho5Eu4gkyLgh5abxU/QKTMTIwBxt4ILyU= +k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= +k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= +k8s.io/component-helpers v0.28.2 h1:r/XJ265PMirW9EcGXr/F+2yWrLPo2I69KdvcY/h9HAo= +k8s.io/component-helpers v0.28.2/go.mod h1:pF1R5YWQ+sgf0i6EbVm+MQCzkYuqutDUibdrkvAa6aI= +k8s.io/controller-manager v0.28.2 h1:C2RKx+NH3Iw+4yLdTGNJlYUd4cRV1N8tKl4XfqMwuTk= +k8s.io/controller-manager v0.28.2/go.mod h1:7bT6FlTE96Co7QevCtvcVnZZIJSaGj6F7EmyT2Rf3GY= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.26.4 h1:mQ+DeOvgAHC6+heZcozPkEd3rWtP4DVVjo1hLSih9w4= -k8s.io/kms v0.26.4/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kms v0.28.2 h1:KhG63LHopCdzs1oKA1j+NWleuIXudgOyCqJo4yi3GaM= +k8s.io/kms v0.28.2/go.mod h1:iAjgIqBrV2+8kmsjbbgUkAyKSuYq5g1dW9knpt6OhaE= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 h1:PUuX1qIFv309AT8hF/CdPKDmsG/hn/L8zRX7VvISM3A= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From d7eb27337d6d93ed882c0b296a24eb7c6e5c3657 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:48:17 -0500 Subject: [PATCH 41/48] fix: support latest k8s code new flags Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- go.mod | 3 ++- go.sum | 10 +++++++++- main.go | 4 +++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 593d01ad..4bbfd79f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb github.com/packethost/packngo v0.30.0 github.com/pallinder/go-randomdata v1.2.0 - github.com/spf13/pflag v1.0.5 go.universe.tf/metallb v0.13.11 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/oauth2 v0.13.0 @@ -43,6 +42,7 @@ require ( github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -73,6 +73,7 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect go.etcd.io/etcd/api/v3 v3.5.9 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect diff --git a/go.sum b/go.sum index 1a8dbe18..e1643852 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,7 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -324,6 +325,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= @@ -366,10 +368,14 @@ go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/A go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go.universe.tf/metallb v0.13.11 h1:NkUP2sEWjr3KSIWabRRg66vQ3VXh/O/nk5Q3vH807D0= @@ -414,6 +420,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -461,6 +468,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -527,7 +535,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -555,6 +562,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index bc35f403..4d1d5bce 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + "k8s.io/cloud-provider/names" "k8s.io/cloud-provider/options" "k8s.io/component-base/cli" cliflag "k8s.io/component-base/cli/flag" @@ -25,6 +26,7 @@ func main() { } controllerInitializers := app.DefaultInitFuncConstructors + controllerAliases := names.CCMControllerAliases() // Remove the route controller which cloud provider equinix metal does not use. delete(controllerInitializers, "route") @@ -33,7 +35,7 @@ func main() { ccmOptions.KubeCloudShared.CloudProvider.Name = metal.ProviderName fss := cliflag.NamedFlagSets{} - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, controllerAliases, fss, wait.NeverStop) code := cli.Run(command) os.Exit(code) } From bac1e6cba1300d76b4b19152767cb8a24f2f4f87 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:50:53 -0500 Subject: [PATCH 42/48] fix: Tiltfile support new cpem controller name Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- dev/Tiltfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Tiltfile b/dev/Tiltfile index 9753a34f..4976a1b0 100644 --- a/dev/Tiltfile +++ b/dev/Tiltfile @@ -18,7 +18,7 @@ deployment[0]['spec']['template']['spec']['containers'][0]['env']=[] deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_METRO_NAME","value":"YOUR_METRO_HERE"}) deployment[0]['spec']['template']['spec']['containers'][0]['env'].append({"name": "METAL_LOAD_BALANCER","value":"YOUR_LOAD_BALANCER_HERE"}) k8s_yaml(encode_yaml_stream(deployment)) -k8s_resource(workload='cloud-provider-equinix-metal',objects=['cloud-controller-manager:ServiceAccount:kube-system','system\\:cloud-controller-manager:ClusterRole:default','system\\:cloud-controller-manager:ClusterRoleBinding:default']) +k8s_resource(workload='cloud-provider-equinix-metal',objects=['cloud-provider-equinix-metal:ServiceAccount:kube-system','cloud-provider-equinix-metal:ClusterRole:default','cloud-provider-equinix-metal:ClusterRoleBinding:default']) k8s_resource(new_name='metal-cloud-config',objects=['metal-cloud-config:Secret:kube-system']) # Load the secret extension From 7f2cfe67addd75d4033e01464b6dcb5ff02fc172 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Fri, 13 Oct 2023 13:18:36 -0500 Subject: [PATCH 43/48] fix: revert k8s and metallb updates Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- go.mod | 109 ++++++++------- go.sum | 412 +++++++++++++++++++++++++++++++++++--------------------- main.go | 4 +- 3 files changed, 311 insertions(+), 214 deletions(-) diff --git a/go.mod b/go.mod index 4bbfd79f..c2121745 100644 --- a/go.mod +++ b/go.mod @@ -8,78 +8,78 @@ require ( github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb github.com/packethost/packngo v0.30.0 github.com/pallinder/go-randomdata v1.2.0 - go.universe.tf/metallb v0.13.11 + go.universe.tf/metallb v0.13.7 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/oauth2 v0.13.0 - k8s.io/api v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/client-go v0.28.2 - k8s.io/cloud-provider v0.28.2 - k8s.io/component-base v0.28.2 + k8s.io/api v0.26.1 + k8s.io/apimachinery v0.26.1 + k8s.io/client-go v0.26.1 + k8s.io/cloud-provider v0.26.1 + k8s.io/component-base v0.26.1 k8s.io/klog/v2 v2.100.1 - sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/yaml v1.3.0 ) require ( + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cel-go v0.16.1 // indirect - github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/cel-go v0.12.6 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/google/gofuzz v1.1.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/spf13/cobra v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/v3 v3.5.9 // indirect + go.etcd.io/etcd/api/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/v3 v3.5.5 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect go.opentelemetry.io/otel v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect @@ -88,34 +88,33 @@ require ( go.opentelemetry.io/otel/sdk v1.10.0 // indirect go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.25.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.7.0 // indirect + go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.16.0 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.54.0 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/grpc v1.49.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.0 // indirect - k8s.io/apiserver v0.28.2 // indirect - k8s.io/component-helpers v0.28.2 // indirect - k8s.io/controller-manager v0.28.2 // indirect - k8s.io/kms v0.28.2 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + k8s.io/apiextensions-apiserver v0.26.1 // indirect + k8s.io/apiserver v0.26.1 // indirect + k8s.io/component-helpers v0.26.1 // indirect + k8s.io/controller-manager v0.26.1 // indirect + k8s.io/kms v0.26.1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index e1643852..83373b39 100644 --- a/go.sum +++ b/go.sum @@ -13,7 +13,7 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -38,30 +38,35 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -72,68 +77,82 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -171,10 +190,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.16.1 h1:3hZfSNiAU3KOiNtxuFXVp5WFy4hf/Ly3Sa4/7F8SXNo= -github.com/google/cel-go v0.16.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= +github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -182,13 +201,14 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -198,8 +218,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= @@ -227,98 +245,137 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= +github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb h1:Tq95IyiReWp+eWNco4KoDNHIALQbNHFg5dD4TELEg9o= github.com/packethost/packet-api-server v0.0.0-20230223042617-bc7d1539adbb/go.mod h1:xX9d7NVrCzFoIFdy5hav33pCMZwiE37ZKgd1XM9qMEY= github.com/packethost/packngo v0.30.0 h1:JVeTwbXXETsLTDQncUbYwIFpkOp/xevXrffM2HrFECI= github.com/packethost/packngo v0.30.0/go.mod h1:BT/XcdwLVmeMtGPbovnxCpnI1s9ylSE1cs/7pq007NE= github.com/pallinder/go-randomdata v1.2.0 h1:EJPxw+sgM1mbMW8RBu5zkG4FbloJpDOCCqPccdWto8A= github.com/pallinder/go-randomdata v1.2.0/go.mod h1:p8CasZQiWDLuaKy5ihVvdshqc7LlL6ovmRxAuNXgi3U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -326,22 +383,22 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v2 v2.305.9 h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo= -go.etcd.io/etcd/client/v2 v2.305.9/go.mod h1:0NBdNx9wbxtEQLwAQtrDHwx58m02vXpDcgSYI2seohQ= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -go.etcd.io/etcd/pkg/v3 v3.5.9 h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ= -go.etcd.io/etcd/pkg/v3 v3.5.9/go.mod h1:BZl0SAShQFk0IpLWR78T/+pyt8AruMHhTNNX73hkNVY= -go.etcd.io/etcd/raft/v3 v3.5.9 h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI= -go.etcd.io/etcd/raft/v3 v3.5.9/go.mod h1:WnFkqzFdZua4LVlVXQEGhmooLeyS7mqzS4Pf4BCVqXg= -go.etcd.io/etcd/server/v3 v3.5.9 h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0= -go.etcd.io/etcd/server/v3 v3.5.9/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= +go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= +go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= +go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= +go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= +go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= +go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= +go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -349,8 +406,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 h1:sxoY9kG1s1WpSYNyzm24rlwH4lnRYFXUVVBmKMBfRgw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= @@ -369,17 +426,21 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= -go.universe.tf/metallb v0.13.11 h1:NkUP2sEWjr3KSIWabRRg66vQ3VXh/O/nk5Q3vH807D0= -go.universe.tf/metallb v0.13.11/go.mod h1:gWM27UgpVVS5OevgfBcGeyKVX6Cvezza34z5iKaArvE= +go.universe.tf/metallb v0.13.7 h1:K/FS/ecpE/a0l5di8ZbFAXB7UP37S8vnrdLGnleiVGk= +go.universe.tf/metallb v0.13.7/go.mod h1:VWu3zLDSHfT3+U9qfV2vr3SJJmIRy133ttd1TTVGyUA= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -412,6 +473,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -423,6 +485,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -430,6 +493,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -449,6 +513,9 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -456,7 +523,9 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -468,13 +537,17 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -483,6 +556,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -495,16 +569,27 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -513,6 +598,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -531,10 +618,12 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -562,15 +651,13 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -625,13 +712,11 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -646,10 +731,13 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -663,26 +751,37 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -690,42 +789,43 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= -k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/apiserver v0.28.2 h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI= -k8s.io/apiserver v0.28.2/go.mod h1:f7D5e8wH8MWcKD7azq6Csw9UN+CjdtXIVQUyUhrtb+E= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/cloud-provider v0.28.2 h1:9qsYm86hm4bnPgZbl9LE29Zfgjuq3NZR2dgtPioJ40s= -k8s.io/cloud-provider v0.28.2/go.mod h1:40fqf6MtgYho5Eu4gkyLgh5abxU/QKTMTIwBxt4ILyU= -k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= -k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= -k8s.io/component-helpers v0.28.2 h1:r/XJ265PMirW9EcGXr/F+2yWrLPo2I69KdvcY/h9HAo= -k8s.io/component-helpers v0.28.2/go.mod h1:pF1R5YWQ+sgf0i6EbVm+MQCzkYuqutDUibdrkvAa6aI= -k8s.io/controller-manager v0.28.2 h1:C2RKx+NH3Iw+4yLdTGNJlYUd4cRV1N8tKl4XfqMwuTk= -k8s.io/controller-manager v0.28.2/go.mod h1:7bT6FlTE96Co7QevCtvcVnZZIJSaGj6F7EmyT2Rf3GY= +k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= +k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= +k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= +k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apiserver v0.26.1 h1:6vmnAqCDO194SVCPU3MU8NcDgSqsUA62tBUSWrFXhsc= +k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= +k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= +k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= +k8s.io/cloud-provider v0.26.1 h1:qEZmsGWGptOtVSpeMdTsapHX2BEqIk7rc5MA4caBqE0= +k8s.io/cloud-provider v0.26.1/go.mod h1:6PheIxRySYuRBBxtTUADya8S2rbr18xKi+fhGbLkduc= +k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= +k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/component-helpers v0.26.1 h1:Y5h1OYUJTGyHZlSAsc7mcfNsWF08S/MlrQyF/vn93mU= +k8s.io/component-helpers v0.26.1/go.mod h1:jxNTnHb1axLe93MyVuvKj9T/+f4nxBVrj/xf01/UNFk= +k8s.io/controller-manager v0.26.1 h1:KmwVTmZ61dxUoHI1TQXlfsbmmk1NVZPUTKjtRowRD30= +k8s.io/controller-manager v0.26.1/go.mod h1:2K95SC0wv5qVbXuC5dJnSgU6vM9J+YBgAaJdVijIy3E= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.28.2 h1:KhG63LHopCdzs1oKA1j+NWleuIXudgOyCqJo4yi3GaM= -k8s.io/kms v0.28.2/go.mod h1:iAjgIqBrV2+8kmsjbbgUkAyKSuYq5g1dW9knpt6OhaE= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kms v0.26.1 h1:JE0n4J4+8/Z+egvXz2BTJeJ9ecsm4ZSLKF7ttVXXm/4= +k8s.io/kms v0.26.1/go.mod h1:ReC1IEGuxgfN+PDCIpR6w8+XMmDE7uJhxcCwMZFdIYc= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= -sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= -sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= +sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= +sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/main.go b/main.go index 4d1d5bce..bc35f403 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,6 @@ import ( cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - "k8s.io/cloud-provider/names" "k8s.io/cloud-provider/options" "k8s.io/component-base/cli" cliflag "k8s.io/component-base/cli/flag" @@ -26,7 +25,6 @@ func main() { } controllerInitializers := app.DefaultInitFuncConstructors - controllerAliases := names.CCMControllerAliases() // Remove the route controller which cloud provider equinix metal does not use. delete(controllerInitializers, "route") @@ -35,7 +33,7 @@ func main() { ccmOptions.KubeCloudShared.CloudProvider.Name = metal.ProviderName fss := cliflag.NamedFlagSets{} - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, controllerAliases, fss, wait.NeverStop) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, fss, wait.NeverStop) code := cli.Run(command) os.Exit(code) } From 69c4c2a620c4084a6b9dabc07d6441f3f6ea3583 Mon Sep 17 00:00:00 2001 From: Chris Privitere <23177737+cprivitere@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:12:31 -0500 Subject: [PATCH 44/48] fix: remove cloud-sa.json from root Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com> --- cloud-sa.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 cloud-sa.json diff --git a/cloud-sa.json b/cloud-sa.json deleted file mode 100644 index 6b923941..00000000 --- a/cloud-sa.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "apiKey":"HDNdRFiLyhv54G6YajToD34X2d5XvGmq", - "projectID": "587aeda9-ee51-447d-87ce-068c57846508" -} \ No newline at end of file From 36cf224aa22eb515a28e9a0001b1e07319794404 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Fri, 20 Oct 2023 08:28:43 -0500 Subject: [PATCH 45/48] chore: refactor manager to replace add/update functions with reconcile --- metal/loadbalancers/emlb/emlb.go | 51 +++-------- .../emlb/infrastructure/manager.go | 85 +++++++------------ 2 files changed, 40 insertions(+), 96 deletions(-) diff --git a/metal/loadbalancers/emlb/emlb.go b/metal/loadbalancers/emlb/emlb.go index 6ba9d371..4b4e3ade 100644 --- a/metal/loadbalancers/emlb/emlb.go +++ b/metal/loadbalancers/emlb/emlb.go @@ -3,7 +3,6 @@ package emlb import ( "context" - "errors" "fmt" "strings" @@ -60,27 +59,7 @@ func NewLB(k8sclient kubernetes.Interface, config, metalAPIKey, projectID string } func (l *LB) AddService(ctx context.Context, svcNamespace, svcName, ip string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { - if svc.Annotations[LoadBalancerIDAnnotation] != "" { - return l.UpdateService(ctx, svcNamespace, svcName, nodes, svc, n) - } - if len(svc.Spec.Ports) < 1 { - return errors.New("cannot add loadbalancer service; no ports assigned") - } - - pools := l.convertToPools(svc, n) - - loadBalancer, err := l.manager.CreateLoadBalancer(ctx, loadBalancerName, pools) - - if err != nil { - return err - } - - patch := client.MergeFrom(svc.DeepCopy()) - - svc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() - svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - - return l.client.Patch(ctx, svc, patch) + return l.reconcileService(ctx, svc, n, loadBalancerName) } func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string, svc *v1.Service) error { @@ -94,35 +73,27 @@ func (l *LB) RemoveService(ctx context.Context, svcNamespace, svcName, ip string } func (l *LB) UpdateService(ctx context.Context, svcNamespace, svcName string, nodes []loadbalancers.Node, svc *v1.Service, n []*v1.Node) error { - // 1. Gather the properties we need: ID of load balancer + loadBalancerName := "" // TODO should UpdateService accept the load balancer name? + return l.reconcileService(ctx, svc, n, loadBalancerName) +} + +func (l *LB) reconcileService(ctx context.Context, svc *v1.Service, n []*v1.Node, loadBalancerName string) error { loadBalancerId := svc.Annotations[LoadBalancerIDAnnotation] pools := l.convertToPools(svc, n) - loadBalancer, err := l.manager.UpdateLoadBalancer(ctx, loadBalancerId, pools) + loadBalancer, err := l.manager.ReconcileLoadBalancer(ctx, loadBalancerId, loadBalancerName, pools) if err != nil { return err } - var ingress []v1.LoadBalancerIngress - for _, ip := range loadBalancer.GetIps() { - ingress = append(ingress, v1.LoadBalancerIngress{ - IP: ip, - }) - // TODO: this is here for backwards compatibility and should be removed ASAP - svc.Spec.LoadBalancerIP = ip - } - - // Per cloud-provider docs, we have to treat `svc` as read-only - updatedSvc := svc.DeepCopy() - updatedSvc.Status.LoadBalancer.Ingress = ingress + patch := client.MergeFrom(svc.DeepCopy()) - updatedSvc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() - updatedSvc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() + svc.Annotations[LoadBalancerIDAnnotation] = loadBalancer.GetId() + svc.Annotations["equinix.com/loadbalancerMetro"] = l.manager.GetMetro() - patch := client.MergeFrom(svc) - return l.client.Patch(ctx, updatedSvc, patch) + return l.client.Patch(ctx, svc, patch) } func (l *LB) convertToPools(svc *v1.Service, nodes []*v1.Node) infrastructure.Pools { diff --git a/metal/loadbalancers/emlb/infrastructure/manager.go b/metal/loadbalancers/emlb/infrastructure/manager.go index 0fad4838..93a52edc 100644 --- a/metal/loadbalancers/emlb/infrastructure/manager.go +++ b/metal/loadbalancers/emlb/infrastructure/manager.go @@ -73,55 +73,52 @@ func (m *Manager) GetPools(ctx context.Context) (*lbaas.LoadBalancerPoolCollecti return LoadBalancerPools, err } -func (m *Manager) CreateLoadBalancer(ctx context.Context, name string, pools Pools) (*lbaas.LoadBalancer, error) { +func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - locationId, ok := LBMetros[m.metro] - if !ok { - return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", m.metro, reflect.ValueOf(LBMetros).MapKeys()) + lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() + + if err != nil { + return err } - lbCreateRequest := lbaas.LoadBalancerCreate{ - Name: name, - LocationId: locationId, - ProviderId: ProviderID, + for _, poolGroups := range lb.Pools { + for _, pool := range poolGroups { + _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, pool.GetId()).Execute() + if err != nil { + return err + } + } + } // TODO lb, resp, err := - lbCreated, _, err := m.client.ProjectsApi.CreateLoadBalancer(ctx, m.projectID).LoadBalancerCreate(lbCreateRequest).Execute() - if err != nil { - return nil, err - } + _, err = m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() + return err +} - loadBalancerID := lbCreated.GetId() +func (m *Manager) ReconcileLoadBalancer(ctx context.Context, id, name string, pools Pools) (*lbaas.LoadBalancer, error) { + ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - for externalPort, pool := range pools { - poolName := getResourceName(name, "pool", externalPort) - poolID, err := m.createPool(ctx, poolName, pool) - if err != nil { - return nil, err + if id == "" { + locationId, ok := LBMetros[m.metro] + if !ok { + return nil, fmt.Errorf("could not determine load balancer location for metro %v; valid values are %v", m.metro, reflect.ValueOf(LBMetros).MapKeys()) } - createPortRequest := lbaas.LoadBalancerPortCreate{ - Name: getResourceName(name, "port", externalPort), - Number: externalPort, - PoolIds: []string{poolID}, + lbCreateRequest := lbaas.LoadBalancerCreate{ + Name: name, + LocationId: locationId, + ProviderId: ProviderID, } - // TODO do we need the port ID for something? - _, _, err = m.client.PortsApi.CreateLoadBalancerPort(ctx, loadBalancerID).LoadBalancerPortCreate(createPortRequest).Execute() + lbCreated, _, err := m.client.ProjectsApi.CreateLoadBalancer(ctx, m.projectID).LoadBalancerCreate(lbCreateRequest).Execute() if err != nil { return nil, err } - } - - lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, loadBalancerID).Execute() - - return lb, err -} -func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools) (*lbaas.LoadBalancer, error) { - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) + id = lbCreated.GetId() + } lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() if err != nil { @@ -204,30 +201,6 @@ func (m *Manager) UpdateLoadBalancer(ctx context.Context, id string, pools Pools return lb, err } -func (m *Manager) DeleteLoadBalancer(ctx context.Context, id string) error { - ctx = context.WithValue(ctx, lbaas.ContextOAuth2, m.tokenExchanger) - - lb, _, err := m.client.LoadBalancersApi.GetLoadBalancer(ctx, id).Execute() - - if err != nil { - return err - } - - for _, poolGroups := range lb.Pools { - for _, pool := range poolGroups { - _, err := m.client.PoolsApi.DeleteLoadBalancerPool(ctx, pool.GetId()).Execute() - if err != nil { - return err - } - } - - } - - // TODO lb, resp, err := - _, err = m.client.LoadBalancersApi.DeleteLoadBalancer(ctx, id).Execute() - return err -} - func (m *Manager) createPool(ctx context.Context, name string, targets []Target) (string, error) { createPoolRequest := lbaas.LoadBalancerPoolCreate{ Name: name, From e81f2c1299c3b61d20c6aad737d2d4671d0d83a4 Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Mon, 23 Oct 2023 16:12:10 -0500 Subject: [PATCH 46/48] feat: add control plane load balancing with LBaaS - Allow the control plane LoadBalancer service to be managed by the provider if `usesBGP` is false for the selected load balancer implementation - Add a reconciler for LBaaS-based control plane LoadBalancer - Add a `LoadBalancerID` config option to specify the ID of the load balancer to use for the control plane --- metal/cloud.go | 16 +- metal/config.go | 5 + metal/controlplane_load_balancer_manager.go | 217 ++++++++++++++++++++ metal/loadbalancers.go | 4 +- 4 files changed, 235 insertions(+), 7 deletions(-) create mode 100644 metal/controlplane_load_balancer_manager.go diff --git a/metal/cloud.go b/metal/cloud.go index 15284f3f..d48e5045 100644 --- a/metal/cloud.go +++ b/metal/cloud.go @@ -27,11 +27,12 @@ const ( // cloud implements cloudprovider.Interface type cloud struct { - client *packngo.Client - config Config - instances *instances - loadBalancer *loadBalancers - controlPlaneEndpointManager *controlPlaneEndpointManager + client *packngo.Client + config Config + instances *instances + loadBalancer *loadBalancers + controlPlaneEndpointManager *controlPlaneEndpointManager + controlPlaneLoadBalancerManager *controlPlaneLoadBalancerManager // holds our bgp service handler bgp *bgp } @@ -80,6 +81,10 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, if err != nil { klog.Fatalf("could not initialize ControlPlaneEndpointManager: %v", err) } + lbm, err := newControlPlaneLoadBalancerManager(clientset, stop, c.config.ProjectID, c.config.LoadBalancerID, c.config.APIServerPort, c.config.EIPHealthCheckUseHostIP) + if err != nil { + klog.Fatalf("could not initialize ControlPlaneEndpointManager: %v", err) + } bgp, err := newBGP(c.client, clientset, c.config) if err != nil { klog.Fatalf("could not initialize BGP: %v", err) @@ -93,6 +98,7 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, c.bgp = bgp c.instances = newInstances(c.client, c.config.ProjectID) c.controlPlaneEndpointManager = epm + c.controlPlaneLoadBalancerManager = lbm klog.Info("Initialize of cloud provider complete") } diff --git a/metal/config.go b/metal/config.go index 5826a058..3642a548 100644 --- a/metal/config.go +++ b/metal/config.go @@ -31,6 +31,7 @@ const ( envVarAPIServerPort = "METAL_API_SERVER_PORT" envVarBGPNodeSelector = "METAL_BGP_NODE_SELECTOR" envVarEIPHealthCheckUseHostIP = "METAL_EIP_HEALTH_CHECK_USE_HOST_IP" + envVarLoadBalancerID = "METAL_LOAD_BALANCER_ID" ) // Config configuration for a provider, includes authentication token, project ID ID, and optional override URL to talk to a different Equinix Metal API endpoint @@ -55,6 +56,7 @@ type Config struct { APIServerPort int32 `json:"apiServerPort,omitempty"` BGPNodeSelector string `json:"bgpNodeSelector,omitempty"` EIPHealthCheckUseHostIP bool `json:"eipHealthCheckUseHostIP,omitempty"` + LoadBalancerID string `json:"loadBalancerID,omitempty"` } // String converts the Config structure to a string, while masking hidden fields. @@ -79,6 +81,7 @@ func (c Config) Strings() []string { ret = append(ret, fmt.Sprintf("Elastic IP Tag: '%s'", c.EIPTag)) ret = append(ret, fmt.Sprintf("API Server Port: '%d'", c.APIServerPort)) ret = append(ret, fmt.Sprintf("BGP Node Selector: '%s'", c.BGPNodeSelector)) + ret = append(ret, fmt.Sprintf("Load Balancer ID: '%s'", c.LoadBalancerID)) return ret } @@ -165,6 +168,8 @@ func getMetalConfig(providerConfig io.Reader) (Config, error) { config.EIPTag = override(os.Getenv(envVarEIPTag), rawConfig.EIPTag) + config.LoadBalancerID = override(os.Getenv(envVarLoadBalancerID), rawConfig.LoadBalancerID) + apiServer := os.Getenv(envVarAPIServerPort) switch { case apiServer != "": diff --git a/metal/controlplane_load_balancer_manager.go b/metal/controlplane_load_balancer_manager.go new file mode 100644 index 00000000..a20430c4 --- /dev/null +++ b/metal/controlplane_load_balancer_manager.go @@ -0,0 +1,217 @@ +package metal + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/packethost/packngo" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + v1applyconfig "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +type controlPlaneLoadBalancerManager struct { + apiServerPort int32 // node on which the external load balancer should listen + nodeAPIServerPort int32 // port on which the api server is listening on the control plane nodes + projectID string + loadBalancerID string + httpClient *http.Client + k8sclient kubernetes.Interface + assignmentMutex sync.Mutex + serviceMutex sync.Mutex + endpointsMutex sync.Mutex + controlPlaneSelectors []labels.Selector + useHostIP bool +} + +func newControlPlaneLoadBalancerManager(k8sclient kubernetes.Interface, stop <-chan struct{}, projectID string, loadBalancerID string, apiServerPort int32, useHostIP bool) (*controlPlaneLoadBalancerManager, error) { + klog.V(2).Info("newControlPlaneLoadBalancerManager()") + + if loadBalancerID == "" { + klog.Info("Load balancer ID is not configured, skipping control plane load balancer management") + return nil, nil + } + + m := &controlPlaneLoadBalancerManager{ + httpClient: &http.Client{ + Timeout: time.Second * 5, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, + apiServerPort: apiServerPort, + projectID: projectID, + loadBalancerID: loadBalancerID, + k8sclient: k8sclient, + useHostIP: useHostIP, + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-stop + cancel() + }() + + for _, label := range controlPlaneLabels { + req, err := labels.NewRequirement(label, selection.Exists, nil) + if err != nil { + return m, err + } + + m.controlPlaneSelectors = append(m.controlPlaneSelectors, labels.NewSelector().Add(*req)) + } + + sharedInformer := informers.NewSharedInformerFactory(k8sclient, checkLoopTimerSeconds*time.Second) + + if _, err := sharedInformer.Core().V1().Endpoints().Informer().AddEventHandler( + cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + e, _ := obj.(*v1.Endpoints) + if e.Namespace != metav1.NamespaceDefault && e.Name != "kubernetes" { + return false + } + + return true + }, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + k8sEndpoints, _ := obj.(*v1.Endpoints) + klog.Infof("handling add, endpoints: %s/%s", k8sEndpoints.Namespace, k8sEndpoints.Name) + + if err := m.syncEndpoints(ctx, k8sEndpoints); err != nil { + klog.Errorf("failed to sync endpoints from default/kubernetes to %s/%s: %v", externalServiceNamespace, externalServiceName, err) + return + } + }, + UpdateFunc: func(_, obj interface{}) { + k8sEndpoints, _ := obj.(*v1.Endpoints) + klog.Infof("handling update, endpoints: %s/%s", k8sEndpoints.Namespace, k8sEndpoints.Name) + + if err := m.syncEndpoints(ctx, k8sEndpoints); err != nil { + klog.Errorf("failed to sync endpoints from default/kubernetes to %s/%s: %v", externalServiceNamespace, externalServiceName, err) + return + } + }, + }, + }, + ); err != nil { + return m, err + } + + if _, err := sharedInformer.Core().V1().Services().Informer().AddEventHandler( + cache.FilteringResourceEventHandler{ + FilterFunc: func(obj interface{}) bool { + s, _ := obj.(*v1.Service) + // Filter only service default/kubernetes + if s.Namespace == metav1.NamespaceDefault && s.Name == "kubernetes" { + return true + } + //else + return false + }, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + k8sService, _ := obj.(*v1.Service) + klog.Infof("handling add, service: %s/%s", k8sService.Namespace, k8sService.Name) + + if err := m.syncService(ctx, k8sService); err != nil { + klog.Errorf("failed to sync service from default/kubernetes to %s/%s: %v", externalServiceNamespace, externalServiceName, err) + return + } + }, + UpdateFunc: func(_, obj interface{}) { + k8sService, _ := obj.(*v1.Service) + klog.Infof("handling update, service: %s/%s", k8sService.Namespace, k8sService.Name) + + if err := m.syncService(ctx, k8sService); err != nil { + klog.Errorf("failed to sync service from default/kubernetes to %s/%s: %v", externalServiceNamespace, externalServiceName, err) + return + } + }, + }, + }, + ); err != nil { + return m, err + } + + sharedInformer.Start(stop) + sharedInformer.WaitForCacheSync(stop) + + return m, nil +} + +func (m *controlPlaneLoadBalancerManager) healthURLFromControlPlaneLoadBalancer(controlPlaneLoadBalancer *packngo.IPAddressReservation) string { + return fmt.Sprintf("https://%s:%d/healthz", controlPlaneLoadBalancer.Address, m.apiServerPort) +} + +func (m *controlPlaneLoadBalancerManager) syncEndpoints(ctx context.Context, k8sEndpoints *v1.Endpoints) error { + m.endpointsMutex.Lock() + defer m.endpointsMutex.Unlock() + + applyConfig := v1applyconfig.Endpoints(externalServiceName, externalServiceNamespace) + for _, subset := range k8sEndpoints.Subsets { + applyConfig = applyConfig.WithSubsets(EndpointSubsetApplyConfig(subset)) + } + + if _, err := m.k8sclient.CoreV1().Endpoints(externalServiceNamespace).Apply( + ctx, + applyConfig, + metav1.ApplyOptions{FieldManager: emIdentifier}, + ); err != nil { + return fmt.Errorf("failed to apply endpoint %s/%s: %w", externalServiceNamespace, externalServiceName, err) + } + + return nil +} + +func (m *controlPlaneLoadBalancerManager) syncService(ctx context.Context, k8sService *v1.Service) error { + m.serviceMutex.Lock() + defer m.serviceMutex.Unlock() + + // get the target port + existingPorts := k8sService.Spec.Ports + if len(existingPorts) < 1 { + return errors.New("default/kubernetes service does not have any ports defined") + } + + // track which port the kube-apiserver actually is listening on + m.nodeAPIServerPort = existingPorts[0].TargetPort.IntVal + // did we set a specific port, or did we request that it just be left as is? + if m.apiServerPort == 0 { + m.apiServerPort = m.nodeAPIServerPort + } + + annotations := map[string]string{} + annotations["equinix.com/loadbalancerID"] = m.loadBalancerID + + specApplyConfig := v1applyconfig.ServiceSpec().WithType(v1.ServiceTypeLoadBalancer) + + for _, port := range existingPorts { + specApplyConfig = specApplyConfig.WithPorts(ServicePortApplyConfig(port)) + } + + applyConfig := v1applyconfig.Service(externalServiceName, externalServiceNamespace). + WithAnnotations(annotations). + WithSpec(specApplyConfig) + + if _, err := m.k8sclient.CoreV1().Services(externalServiceNamespace).Apply( + ctx, + applyConfig, + metav1.ApplyOptions{FieldManager: emIdentifier}, + ); err != nil { + return fmt.Errorf("failed to apply service %s/%s: %w", externalServiceNamespace, externalServiceName, err) + } + + return nil +} diff --git a/metal/loadbalancers.go b/metal/loadbalancers.go index e8fa0974..bd726039 100644 --- a/metal/loadbalancers.go +++ b/metal/loadbalancers.go @@ -174,8 +174,8 @@ func (l *loadBalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri // TODO: Split out most of this to "reconcileLoadBalancer" // TODO: Split out status checking to a separate function that reconcileLoadBalancer calls - // handling is completely different if it is the control plane vs a regular service of type=LoadBalancer - if service.Name == externalServiceName && service.Namespace == externalServiceNamespace { + // For EIP-based (BGP) load balancers, handling is completely different if it is the control plane vs a regular service of type=LoadBalancer + if l.usesBGP && service.Name == externalServiceName && service.Namespace == externalServiceNamespace { ipCidr, err = l.retrieveIPByTag(ctx, service, l.eipTag) if err != nil { return nil, fmt.Errorf("failed to add service %s: %w", service.Name, err) From 2a3dc23ba5068f8ed75398b055ced0e3b6909b4c Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 24 Oct 2023 15:06:36 -0500 Subject: [PATCH 47/48] chore: address lint failures --- metal/controlplane_load_balancer_manager.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/metal/controlplane_load_balancer_manager.go b/metal/controlplane_load_balancer_manager.go index a20430c4..8bde160b 100644 --- a/metal/controlplane_load_balancer_manager.go +++ b/metal/controlplane_load_balancer_manager.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/packethost/packngo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -28,7 +27,6 @@ type controlPlaneLoadBalancerManager struct { loadBalancerID string httpClient *http.Client k8sclient kubernetes.Interface - assignmentMutex sync.Mutex serviceMutex sync.Mutex endpointsMutex sync.Mutex controlPlaneSelectors []labels.Selector @@ -151,10 +149,6 @@ func newControlPlaneLoadBalancerManager(k8sclient kubernetes.Interface, stop <-c return m, nil } -func (m *controlPlaneLoadBalancerManager) healthURLFromControlPlaneLoadBalancer(controlPlaneLoadBalancer *packngo.IPAddressReservation) string { - return fmt.Sprintf("https://%s:%d/healthz", controlPlaneLoadBalancer.Address, m.apiServerPort) -} - func (m *controlPlaneLoadBalancerManager) syncEndpoints(ctx context.Context, k8sEndpoints *v1.Endpoints) error { m.endpointsMutex.Lock() defer m.endpointsMutex.Unlock() From 8a048dc5b3f9b735b49877bf41b8f15f265dfc7c Mon Sep 17 00:00:00 2001 From: Charles Treatman Date: Tue, 24 Oct 2023 15:46:25 -0500 Subject: [PATCH 48/48] docs: update README with EMLB support and config --- README.md | 95 +++++++++++++++++++++++++++---------------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 0e6699f3..c6639313 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,7 @@ This section lists each configuration option, and whether it can be set by each | Kubernetes Service annotation to set EIP metro | | `METAL_ANNOTATION_EIP_METRO` | `annotationEIPMetro` | `"metal.equinix.com/eip-metro"` | | Kubernetes Service annotation to set EIP facility | | `METAL_ANNOTATION_EIP_FACILITY` | `annotationEIPFacility` | `"metal.equinix.com/eip-facility"` | | Tag for control plane Elastic IP | | `METAL_EIP_TAG` | `eipTag` | No control plane Elastic IP | +| ID for control plane Equinix Metal Load Balancer | | `METAL_LOAD_BALANCER_ID` | `loadBalancerID` | No control plane Equinix Metal Load Balancer | | Kubernetes API server port for Elastic IP | | `METAL_API_SERVER_PORT` | `apiServerPort` | Same as `kube-apiserver` on control plane nodes, same as `0` | | Filter for cluster nodes on which to enable BGP | | `METAL_BGP_NODE_SELECTOR` | `bgpNodeSelector` | All nodes | | Use host IP for Control Plane endpoint health checks | | `METAL_EIP_HEALTH_CHECK_USE_HOST_IP` | `eipHealthCheckUseHostIP` | false | @@ -253,50 +254,20 @@ The Kubernetes CCM for Equinix Metal deploys as a `Deployment` into your cluster ### Service Load Balancers -~~Equinix Metal does not offer managed load balancers like [AWS ELB](https://aws.amazon.com/elasticloadbalancing/) -or [GCP Load Balancers](https://cloud.google.com/load-balancing/). Instead, if configured to do so, -Equinix Metal CCM will interface with and configure external bare-metal loadbalancers.~~ +Equinix CCM supports two approaches to load balancing: -When a load balancer is enabled, the CCM does the following: +1. If configured to do so, Equinix Metal CCM will interface with and configure external bare-metal load balancers +2. If configured to do so, and if the feature is available on your Equinix Metal account, Equinix Metal CCM will interface with and configure external, managed Equinix Metal Load Balancers (EMLB) + +When any load balancer is enabled, the CCM does the following: 1. Enable BGP for the project 1. Enable BGP on each node as it comes up 1. Sets ASNs based on configuration or default -1. Unless you are using EMLB, for each `Service` of `type=LoadBalancer`: +1. If you are using bare-metal load balancers, then for each `Service` of `type=LoadBalancer`: - If you have specified a load balancer IP on `Service.Spec.LoadBalancerIP` (bring your own IP, or BYOIP), do nothing - If you have not specified a load balancer IP on `Service.Spec.LoadBalancerIP`, get an Equinix Metal Elastic IP and set it on `Service.Spec.LoadBalancerIP`, see below -1. Then, depending on the load balancer implementation: - - 1. If MetalLB, set up the CRDs necessary for MetalLB to create the load balancer and configure it on the relevant nodes - 1. If EMLB - - 1. Ensure preferred load balancer metro is configured (make it part of the schema?) - 1. Create an origin pool per service - 1. Pool Name, based on service - 1. Node IP - 1. Port (the nodeport) - chris's guess - 1. Reuse or Create load balancer - 1. Check annotation for load balancer name/metro - 1. Already exists? - 1. Yes - 1. Get the load balancer - 1. Make sure public port is free - 1. If not, do we enable creating new load balancers? $$$$$ - 1. Specify Origin Pool created above - 1. No - 1. Create it - 1. Listener Port (public port requested in Service.Spec.Port) - 1. Specify Origin Pool created above - 1. Delete Load Balancer - 1. Remove origin pool - 1. Remove listener Port - 1. If no more listeners, delete load balancer ? - 1. Update Load Balancer - 1. Get Origin Pool - 1. Diff - 1. Update if needed - - 1. If Kube-vip or empty, do nothing +1. Pass control to the specific load balancer implementation #### Service Load Balancer IP @@ -354,12 +325,6 @@ are created at a system-wide level, ignoring the annotations. Using these flags and annotations, you can run the CCM on a node in a different metro or facility, or even outside of Equinix Metal entirely. -#### Control Plane LoadBalancer Implementation - -For the control plane nodes, the Equinix Metal CCM uses static Elastic IP assignment, via the Equinix Metal API, to tell the -Equinix Metal network which control plane node should receive the traffic. For more details on the control plane -load-balancer, see [this section](#control-plane-load-balancing). - #### Service LoadBalancer Implementations Loadbalancing is enabled as follows. @@ -375,6 +340,8 @@ The value of the loadbalancing configuration is `:///` where: For loadbalancing for Kubernetes `Service` of `type=LoadBalancer`, the following implementations are supported: + +- [Equinix Metal Load Balancer](#EquinixMetalLoadBalancer) - [kube-vip](#kube-vip) - [MetalLB](#metallb) - [empty](#empty) @@ -382,6 +349,23 @@ For loadbalancing for Kubernetes `Service` of `type=LoadBalancer`, the following CCM does **not** deploy _any_ load balancers for you. It limits itself to managing the Equinix Metal-specific API calls to support a load balancer, and providing configuration for supported load balancers. +##### Equinix Metal Load Balancer + +Equinix Metal Load Balancer (EMLB) is a beta service that is available to a limited number of Equinix Metal customers that provides managed layer 4 load balancers. + +When the EMLB option is enabled, for user-deployed Kubernetes `Service` of `type=LoadBalancer`, the Equinix Metal CCM: +- creates an Equinix Metal Load Balancer for the service +- creates listener ports on the Equinix Metal Load Balancer for each port on the service +- creates origin pools for each listener port that send traffic to the corresponding NodePorts in your cluster + +To enable EMLB, set the configuration `METAL_LOAD_BALANCER` or config `loadbalancer` to: + +``` +emlb:// +``` + +Where `` is the Equinix metro in which you want CCM to deploy your external load balancers. For example, to deploy your load balancers in Silicon Valley, you would set the configuration to `emlb://sv`. Note that EMLB is available in a limited number of Equinix metros (as of this writing, `sv`, `da`, and `ny`). + ##### kube-vip **Supported Versions**: @@ -602,8 +586,10 @@ load balancer to work, the IP address needs to be all of: Reserved, Assigned, Ma ## Control Plane Load Balancing -CCM implements an optional control plane load balancer using an Equinix Metal Elastic IP (EIP) and the Equinix Metal API's -ability to assign that EIP to different devices. +CCM implements an optional control plane load balancer using one of two approaches: + +1. an Equinix Metal Load Balancer +1. an Equinix Metal Elastic IP (EIP) and the Equinix Metal API's ability to assign that EIP to different devices. You have several options for control plane load-balancing: @@ -613,6 +599,19 @@ You have several options for control plane load-balancing: ### CCM Managed +#### Equinix Metal Load Balancer + +If you have configured the CCM to use Equinix Metal Load Balancers (EMLB) for service load balancing, you can also choose to use EMLB for control plane load balancing. To enable control plane load balancing with EMLB: + +1. Create a Load Balancer using the Equinix Metal API or Web UI +1. When starting the CCM + - set the [configuration](#Configuration) for load balancing with EMLB, e.g. env var `METAL_LOAD_BALANCER=emlb://`, where `` is the metro in which you want the CCM to create your load balancers + - set the [configuration](#Configuration) for the control plane EIP tag, e.g. env var `METAL_LOAD_BALANCER_ID=`, where `` is the ID of the Load Balancer you created earlier + +When run with the correct configuration, on startup, CCM will automatically update your Load Balancer to send traffic to your control plane nodes. + +#### Elastic IP Load Balancer + It is a common procedure to use Elastic IP as Control Plane endpoint in order to have a static endpoint that you can use from the outside, or when configuring the advertise address for the kubelet. @@ -622,9 +621,9 @@ To enable CCM to manage the control plane EIP: 1. Create an Elastic IP, using the Equinix Metal API, Web UI or CLI 1. Put an arbitrary but unique tag on the EIP 1. When starting the CCM - - set the [configuration][Configuration] for the control plane EIP tag, e.g. env var `METAL_EIP_TAG=`, where `` is whatever tag you set on the EIP + - set the [configuration](#Configuration) for the control plane EIP tag, e.g. env var `METAL_EIP_TAG=`, where `` is whatever tag you set on the EIP - (optional) set the port that the EIP should listen on; by default, or when set to `0`, it will use the same port as the `kube-apiserver` on the control plane nodes. This port can also be specified with `METAL_API_SERVER_PORT=.` - - (optional) set the [configuration][Configuration] for using the host IP for control plane endpoint health checks. This is + - (optional) set the [configuration](#Configuration) for using the host IP for control plane endpoint health checks. This is needed when the EIP is configured as an loopback IP address, such as the case with [CAPP](https://github.com/kubernetes-sigs/cluster-api-provider-packet) In [CAPP](https://github.com/kubernetes-sigs/cluster-api-provider-packet) we @@ -655,7 +654,7 @@ The logic will circle over all the available control planes looking for an active api server. As soon as it can find one the Elastic IP will be unassigned and reassigned to the working node. -#### How the Elastic IP Traffic is Routed +##### How the Elastic IP Traffic is Routed Of course, even if the router sends traffic for your Elastic IP (EIP) to a given control plane node, that node needs to know to process the traffic. Rather than require you to