diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml index d6ea5daf..5e5af758 100644 --- a/.github/workflows/build-container-image.yml +++ b/.github/workflows/build-container-image.yml @@ -20,23 +20,23 @@ jobs: packages: write steps: - name: Check out repository - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + uses: actions/checkout@v3 - name: Install Go - uses: actions/setup-go@84cbf8094393cdc5fe1fe1671ff2647332956b1a + uses: actions/setup-go@v3 with: - go-version: "1.18" + go-version: "1.20.3" - name: Set up Docker Buildx id: docker-setup - uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 + uses: docker/setup-buildx-action@v2 - name: Build container image run: GCE_PD_CSI_STAGING_IMAGE=ghcr.io/edgelesssys/constellation/gcp-csi-driver GCE_PD_CSI_STAGING_VERSION=${{ inputs.versionTag }} make build-container - name: Log in to the Container registry id: docker-login - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/sanitytest.yaml b/.github/workflows/sanitytest.yaml index 817f442b..ae3e55ff 100644 --- a/.github/workflows/sanitytest.yaml +++ b/.github/workflows/sanitytest.yaml @@ -13,16 +13,16 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: "1.20.3" id: go - name: Install Dependencies run: sudo apt-get update && sudo apt-get install -y libcryptsetup-dev - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build Test run: | diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 72fe7c4b..b5ba33ac 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -12,16 +12,16 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: "1.20.3" id: go - name: Install Dependencies run: sudo apt-get update && sudo apt-get install -y libcryptsetup-dev - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build Test run: | diff --git a/CHANGELOG/CHANGELOG-0.6.md b/CHANGELOG/CHANGELOG-0.6.md deleted file mode 100644 index e421d851..00000000 --- a/CHANGELOG/CHANGELOG-0.6.md +++ /dev/null @@ -1,42 +0,0 @@ -# v0.6.0 - Changelog Since v0.5.0 - -## Breaking Changes - -- Some of the API objects in the deployment specs have changed names/labels/namespaces, please tear down old driver before deploying this version to avoid orphaning old objects. You will also no longer see the driver in the `default` namespace. -- Some error codes have been changed, please see below for details if you rely on specific error codes of the driver - -## New Features - -- Add support for Raw Block devices. ([#283](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/283), [@davidz627](https://github.com/davidz627)) -- Operations in the node driver are now parallelized, except those involving a volume already being operated on now return an error. ([#303](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/303), [@hantaowang](https://github.com/hantaowang)) -- Adds support for ControllerExpandVolume and NodeExpandVolume ([#317](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/317), [@davidz627](https://github.com/davidz627)) -- Operations in the controller driver on a volume already being operated on now return an error. ([#316](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/316), [@hantaowang](https://github.com/hantaowang)) -- Picking up support for inline volume migration and some fixes for backward compatible access modes for migration ([#324](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/324), [@davidz627](https://github.com/davidz627)) - - -## Bug Fixes - -- Reduces node attach limits by 1 since the node boot disk is considered an attachable disk ([#361](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/361), [@davidz627](https://github.com/davidz627)) -- Fixed a bug that causes disks in the same zone/region to be provisioned serially ([#344](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/344), [@hantaowang](https://github.com/hantaowang)) -- Remove cross validation of access modes, multiple access modes can be specified that represent all the capabilities of the volume ([#289](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/289), [@davidz627](https://github.com/davidz627)) -- Driver should check socket parent directory before trying to bind it ([#339](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/339), [@zhucan](https://github.com/zhucan)) -- Updated CSI Attacher to stop ignoring errors from ControllerUnpublish ([#378](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/378), [@davidz627](https://github.com/davidz627)) -- CreateVolume will now fail with NOT_FOUND error when VolumeContentSource SnapshotId does not refer to a snapshot that can be found ([#312](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/312), [@davidz627](https://github.com/davidz627)) -- ControllerUnpublishVolume now returns success when the Node is GCE API NotFound. -Invalid format VolumeID is now GRPC InvalidArgument error instead of GRPC NotFound. -Underspecified disks not found in any zone now return GRPC NotFound. ([#368](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/368), [@davidz627](https://github.com/davidz627)) - - -## Other Notable Changes - -- Deployment spec updates: -The deployment is no longer in namespace `default` -Changed "app" label key to "k8s-app" -csi-snapshotter version has been changed to v1.2.0-gke.0 -The resizer role binding has been renamed to "csi-controller-resizer-binding" -Removed driver-registrar role. ([#364](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/364), [@verult](https://github.com/verult)) -- Updating the following image versions in stable deployment specs: -gcp-compute-persistent-disk-csi-driver: v0.6.0-gke.0 -csi-provisioner: v1.4.0-gke.0 -csi-attacher: v2.0.0-gke.0 -csi-node-driver-registrar: v1.2.0-gke.0 ([#400](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/400), [@verult](https://github.com/verult)) \ No newline at end of file diff --git a/CHANGELOG/CHANGELOG-0.7.md b/CHANGELOG/CHANGELOG-0.7.md deleted file mode 100644 index 2adf2695..00000000 --- a/CHANGELOG/CHANGELOG-0.7.md +++ /dev/null @@ -1,33 +0,0 @@ -# v0.7.0 - Changelog Since v0.6.0 - -## Changes with Action Required - -- Adding `PodSecurityPoliciy` to allow `csi-gce-pd-node` in clusters with policies enabled. -IF LOCAL PSP MANIFEST PATCH IS USED PLEASE BEWARE THAT YOU WILL NEED TO DELETE LOCAL CHANGES AND USE THE UPSTREAM ([#448](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/448), [@ffilippopoulos](https://github.com/ffilippopoulos)) -- BREAKING CHANGE: All deployment objects in setup-cluster.yaml have been renamed. When deleting the deployment using ./delete-driver.sh, make sure to use specs from your previous deployment version to ensure the correct objects are cleaned up. ([#405](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/405), [@verult](https://github.com/verult)) - -## New Features - -- Add GET_VOLUME_STATS Node Service Capability and implementation for getting stats for volume ([#406](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/406), [@davidz627](https://github.com/davidz627)) -- ValidateVolumeCapabilities validates that the given volume conforms to all capabilities in the request. Validation of existing volumes during inserts also improved to check all parameters. ([#467](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/467), [@davidz627](https://github.com/davidz627)) -- It is now possible to disable the controller service by setting `--run-controller-service=false`. Similarly, it is possible to disable the node service by setting `--run-node-service=false`. The latter enables running the controller server of the GCE PD driver separately/outside of the cluster it is serving. Also, if both `project-id` and `zone` are specified in the GCE cloud config then the controller server does no longer try to contact the GCE metadata service. ([#449](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/449), [@rfranzke](https://github.com/rfranzke)) -- Add support for formatting and mounting an XFS filesystem ([#447](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/447), [@davidz627](https://github.com/davidz627)) -- Add a blanket toleration to the Node Daemonset of the driver deployment so that it can be deployed on all nodes ([#417](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/417), [@davidz627](https://github.com/davidz627)) -- Adds LIST_VOLUMES and LIST_VOLUMES_PUBLISHED_NODES capabilities with respective functionality ([#392](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/392), [@davidz627](https://github.com/davidz627)) - - -## Bug Fixes - -- Fixed bug where ControllerExpandVolume was returning incorrect size when disk was already the requested size or larger ([#462](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/462), [@davidz627](https://github.com/davidz627)) -- Set volume limits to 15 only for machine-types: "f1-micro", "g1-small", "e2-micro", "e2-small", "e2-medium". Limit is 127 for all others ([#455](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/455), [@davidz627](https://github.com/davidz627)) -- Changed deployment of Controller and Node components to use hostNetwork for compatibility with GKE Workload Identity ([#436](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/436), [@davidz627](https://github.com/davidz627)) -- During NodeStageVolume run udevadm --trigger to fix device symlinks if device path is not found or device path points to the wrong device ([#459](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/459), [@davidz627](https://github.com/davidz627)) -- Bump external-snapshotter version to v1.2.2 for fix of CVE-2019-11255 ([#434](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/434), [@davidz627](https://github.com/davidz627)) - - -## Other Notable Changes - -- Update driver base image distro to debian-amd64:v2.0.0 and build with go v1.13.4 ([#439](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/439), [@davidz627](https://github.com/davidz627)) -- Mounting an unformatted volume with an fstype as read-only now throws a more descriptive error ([#458](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/458), [@davidz627](https://github.com/davidz627)) -- Remove explicit stripping of secrets from RPC request/response logs since the driver doesn't accept secrets for operations ([#428](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/428), [@davidz627](https://github.com/davidz627)) -- Improve driver logs to log success in all paths as well as logging additional useful information ([#409](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/409), [@davidz627](https://github.com/davidz627)) \ No newline at end of file diff --git a/CHANGELOG/CHANGELOG-1.0.md b/CHANGELOG/CHANGELOG-1.0.md deleted file mode 100644 index ad4c8763..00000000 --- a/CHANGELOG/CHANGELOG-1.0.md +++ /dev/null @@ -1,359 +0,0 @@ -# v1.0.4 - Changelog since v1.0.2 - -### Other - -- Fix build for various go versions, ([#832](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/832), [@mattcary](https://github.com/mattcary)) - -# v1.0.3 - **Bad tag, skipped** - -# v1.0.2 - Changelog since v1.0.1 - -### Bug or Regression - -- Update base image to buster-v.1.9.0 ([#829](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/829), [@mattcary](https://github.com/mattcary)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.0.1 - Changelog since v1.0.0 - -## Changes by Kind - -### Other (Cleanup or Flake) - -- Update GCE PD CSI Driver Docker base image to `k8s.gcr.io/build-image/debian-base-amd64:v2.1.3` (previously `gcr.io/google-containers/debian-base-amd64:v2.0.0`) to address CVEs. ([#598](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/598), [@saad-ali](https://github.com/saad-ali)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.0.0 - Changelog since v0.7.0 - -## Changes by Kind - -### Feature - -- Add support for multi-writer raw block devices. ([#415](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/415), [@sschmitt](https://github.com/sschmitt)) -- Enable CSI Snapshotter (2.x) side car for PD CSI driver ([#500](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/500), [@saikat-royc](https://github.com/saikat-royc)) -- Enable CSI snapshotter side car (beta) for PD CSI driver. ([#507](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/507), [@saikat-royc](https://github.com/saikat-royc)) -- Increase sidecar operation timeout for stable overlay ([#577](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/577), [@saikat-royc](https://github.com/saikat-royc)) -- Provisioned GCE PD Description will contain JSON with information about what PVC/PV disk was created for (similar to volumes provisioned by in-tree GCE PD plugin). This feature requires the `--extra-create-metadata` flag to be set on the CSI external-provisioner. ([#570](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/570), [@saad-ali](https://github.com/saad-ali)) -- The driver deployment now has leader election enabled, and the controller deployment uses a Deployment instead of a StatefulSet. ([#521](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/521), [@verult](https://github.com/verult)) -- Updating the following image versions in stable deployment specs: - - csi-provisioner: v1.6.0-gke.0 - - csi-attacher: v2.2.0-gke.0 - - csi-node-driver-registrar: v1.3.0-gke.0 - - csi-resizer: v0.5.0-gke.0 ([#517](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/517), [@verult](https://github.com/verult)) -- Add Windows support in GCE PD driver with the use of csi-proxy (https://github.com/kubernetes-csi/csi-proxy/). ([#483](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/483), [@jingxu97](https://github.com/jingxu97)) -- Adds new build rules using docker buildx feature for allowing to build windows container image on either windows or linux node. ([#489](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/489), [@jingxu97](https://github.com/jingxu97)) -- Install CSIDriver object as part of driver deployment. ([#575](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/575), [@Jiawei0227](https://github.com/Jiawei0227)) - -### Bug or Regression - -- Increase provisioner and attacher op timeout to reduce retries ([#542](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/542), [@saikat-royc](https://github.com/saikat-royc)) -- Fixed issue where provisioning of GCE PDs with CMEK used to sometimes fails with `disk already exists with same name` error. ([#563](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/563), [@saad-ali](https://github.com/saad-ali)) -- Honor image-type in GKE cluster deploy ([#536](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/536), [@saikat-royc](https://github.com/saikat-royc)) -- In GCE PersistentDisk CSI Driver CreateVolume call, wait for disk to reach a READY status, before returning success to caller. ([#527](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/527), [@saikat-royc](https://github.com/saikat-royc)) -- Skip NodeExpandVolume for block volumes. ([#571](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/571), [@saad-ali](https://github.com/saad-ali)) - -### Failing Test - -- Add autorepair options for GKE release channel ([#538](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/538), [@saikat-royc](https://github.com/saikat-royc)) -- Disable the optional capability to handle volume in user errors for staging-head PD driver. ([#540](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/540), [@saikat-royc](https://github.com/saikat-royc)) -- Add hook to deploy GKE with GCE PD CSI driver, and run kubernetes e2e tests against the managed driver ([#515](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/515), [@saikat-royc](https://github.com/saikat-royc)) -- Enable PD CSI snapshot tests for release-staging-head and release-staging-rc ([#505](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/505), [@saikat-royc](https://github.com/saikat-royc)) - -### Documentation - -- Update documentation and user guides for beta snapshotter ([#508](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/508), [@saikat-royc](https://github.com/saikat-royc)) - - -## Dependencies - -### Added -- bitbucket.org/bertimus9/systemstat: 0eeff89 -- github.com/Azure/go-autorest/autorest/adal: [v0.5.0](https://github.com/Azure/go-autorest/autorest/adal/tree/v0.5.0) -- github.com/Azure/go-autorest/autorest/date: [v0.1.0](https://github.com/Azure/go-autorest/autorest/date/tree/v0.1.0) -- github.com/Azure/go-autorest/autorest/mocks: [v0.2.0](https://github.com/Azure/go-autorest/autorest/mocks/tree/v0.2.0) -- github.com/Azure/go-autorest/autorest/to: [v0.2.0](https://github.com/Azure/go-autorest/autorest/to/tree/v0.2.0) -- github.com/Azure/go-autorest/autorest/validation: [v0.1.0](https://github.com/Azure/go-autorest/autorest/validation/tree/v0.1.0) -- github.com/Azure/go-autorest/autorest: [v0.9.0](https://github.com/Azure/go-autorest/autorest/tree/v0.9.0) -- github.com/Azure/go-autorest/logger: [v0.1.0](https://github.com/Azure/go-autorest/logger/tree/v0.1.0) -- github.com/Azure/go-autorest/tracing: [v0.5.0](https://github.com/Azure/go-autorest/tracing/tree/v0.5.0) -- github.com/JeffAshton/win_pdh: [76bb4ee](https://github.com/JeffAshton/win_pdh/tree/76bb4ee) -- github.com/MakeNowJust/heredoc: [bb23615](https://github.com/MakeNowJust/heredoc/tree/bb23615) -- github.com/Microsoft/hcsshim: [672e52e](https://github.com/Microsoft/hcsshim/tree/672e52e) -- github.com/OpenPeeDeeP/depguard: [v1.0.1](https://github.com/OpenPeeDeeP/depguard/tree/v1.0.1) -- github.com/Rican7/retry: [v0.1.0](https://github.com/Rican7/retry/tree/v0.1.0) -- github.com/StackExchange/wmi: [5d04971](https://github.com/StackExchange/wmi/tree/5d04971) -- github.com/agnivade/levenshtein: [v1.0.1](https://github.com/agnivade/levenshtein/tree/v1.0.1) -- github.com/ajstarks/svgo: [644b8db](https://github.com/ajstarks/svgo/tree/644b8db) -- github.com/andreyvit/diff: [c7f18ee](https://github.com/andreyvit/diff/tree/c7f18ee) -- github.com/anmitsu/go-shlex: [648efa6](https://github.com/anmitsu/go-shlex/tree/648efa6) -- github.com/armon/circbuf: [bbbad09](https://github.com/armon/circbuf/tree/bbbad09) -- github.com/auth0/go-jwt-middleware: [5493cab](https://github.com/auth0/go-jwt-middleware/tree/5493cab) -- github.com/bazelbuild/bazel-gazelle: [70208cb](https://github.com/bazelbuild/bazel-gazelle/tree/70208cb) -- github.com/bazelbuild/rules_go: [6dae44d](https://github.com/bazelbuild/rules_go/tree/6dae44d) -- github.com/bifurcation/mint: [93c51c6](https://github.com/bifurcation/mint/tree/93c51c6) -- github.com/boltdb/bolt: [v1.3.1](https://github.com/boltdb/bolt/tree/v1.3.1) -- github.com/bradfitz/go-smtpd: [deb6d62](https://github.com/bradfitz/go-smtpd/tree/deb6d62) -- github.com/caddyserver/caddy: [v1.0.3](https://github.com/caddyserver/caddy/tree/v1.0.3) -- github.com/cenkalti/backoff: [v2.1.1+incompatible](https://github.com/cenkalti/backoff/tree/v2.1.1) -- github.com/cespare/prettybench: [03b8cfe](https://github.com/cespare/prettybench/tree/03b8cfe) -- github.com/chai2010/gettext-go: [c6fed77](https://github.com/chai2010/gettext-go/tree/c6fed77) -- github.com/checkpoint-restore/go-criu: [17b0214](https://github.com/checkpoint-restore/go-criu/tree/17b0214) -- github.com/cheekybits/genny: [9127e81](https://github.com/cheekybits/genny/tree/9127e81) -- github.com/cilium/ebpf: [95b36a5](https://github.com/cilium/ebpf/tree/95b36a5) -- github.com/clusterhq/flocker-go: [2b8b725](https://github.com/clusterhq/flocker-go/tree/2b8b725) -- github.com/cockroachdb/datadriven: [80d97fb](https://github.com/cockroachdb/datadriven/tree/80d97fb) -- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) -- github.com/containerd/console: [84eeaae](https://github.com/containerd/console/tree/84eeaae) -- github.com/containerd/containerd: [v1.0.2](https://github.com/containerd/containerd/tree/v1.0.2) -- github.com/containerd/typeurl: [2a93cfd](https://github.com/containerd/typeurl/tree/2a93cfd) -- github.com/containernetworking/cni: [v0.7.1](https://github.com/containernetworking/cni/tree/v0.7.1) -- github.com/coredns/corefile-migration: [v1.0.6](https://github.com/coredns/corefile-migration/tree/v1.0.6) -- github.com/creack/pty: [v1.1.7](https://github.com/creack/pty/tree/v1.1.7) -- github.com/cyphar/filepath-securejoin: [v0.2.2](https://github.com/cyphar/filepath-securejoin/tree/v0.2.2) -- github.com/daviddengcn/go-colortext: [511bcaf](https://github.com/daviddengcn/go-colortext/tree/511bcaf) -- github.com/dnaeon/go-vcr: [v1.0.1](https://github.com/dnaeon/go-vcr/tree/v1.0.1) -- github.com/docker/libnetwork: [c8a5fca](https://github.com/docker/libnetwork/tree/c8a5fca) -- github.com/euank/go-kmsg-parser: [v2.0.0+incompatible](https://github.com/euank/go-kmsg-parser/tree/v2.0.0) -- github.com/exponent-io/jsonpath: [d6023ce](https://github.com/exponent-io/jsonpath/tree/d6023ce) -- github.com/fatih/camelcase: [v1.0.0](https://github.com/fatih/camelcase/tree/v1.0.0) -- github.com/flynn/go-shlex: [3f9db97](https://github.com/flynn/go-shlex/tree/3f9db97) -- github.com/fogleman/gg: [0403632](https://github.com/fogleman/gg/tree/0403632) -- github.com/gliderlabs/ssh: [v0.1.1](https://github.com/gliderlabs/ssh/tree/v0.1.1) -- github.com/go-acme/lego: [v2.5.0+incompatible](https://github.com/go-acme/lego/tree/v2.5.0) -- github.com/go-bindata/go-bindata: [v3.1.1+incompatible](https://github.com/go-bindata/go-bindata/tree/v3.1.1) -- github.com/go-critic/go-critic: [1df3008](https://github.com/go-critic/go-critic/tree/1df3008) -- github.com/go-lintpack/lintpack: [v0.5.2](https://github.com/go-lintpack/lintpack/tree/v0.5.2) -- github.com/go-ole/go-ole: [v1.2.1](https://github.com/go-ole/go-ole/tree/v1.2.1) -- github.com/go-ozzo/ozzo-validation: [v3.5.0+incompatible](https://github.com/go-ozzo/ozzo-validation/tree/v3.5.0) -- github.com/go-toolsmith/astcast: [v1.0.0](https://github.com/go-toolsmith/astcast/tree/v1.0.0) -- github.com/go-toolsmith/astcopy: [v1.0.0](https://github.com/go-toolsmith/astcopy/tree/v1.0.0) -- github.com/go-toolsmith/astequal: [v1.0.0](https://github.com/go-toolsmith/astequal/tree/v1.0.0) -- github.com/go-toolsmith/astfmt: [v1.0.0](https://github.com/go-toolsmith/astfmt/tree/v1.0.0) -- github.com/go-toolsmith/astinfo: [9809ff7](https://github.com/go-toolsmith/astinfo/tree/9809ff7) -- github.com/go-toolsmith/astp: [v1.0.0](https://github.com/go-toolsmith/astp/tree/v1.0.0) -- github.com/go-toolsmith/pkgload: [v1.0.0](https://github.com/go-toolsmith/pkgload/tree/v1.0.0) -- github.com/go-toolsmith/strparse: [v1.0.0](https://github.com/go-toolsmith/strparse/tree/v1.0.0) -- github.com/go-toolsmith/typep: [v1.0.0](https://github.com/go-toolsmith/typep/tree/v1.0.0) -- github.com/gobwas/glob: [v0.2.3](https://github.com/gobwas/glob/tree/v0.2.3) -- github.com/godbus/dbus: [2ff6f7f](https://github.com/godbus/dbus/tree/2ff6f7f) -- github.com/golang/freetype: [e2365df](https://github.com/golang/freetype/tree/e2365df) -- github.com/golangci/check: [cfe4005](https://github.com/golangci/check/tree/cfe4005) -- github.com/golangci/dupl: [3e9179a](https://github.com/golangci/dupl/tree/3e9179a) -- github.com/golangci/errcheck: [ef45e06](https://github.com/golangci/errcheck/tree/ef45e06) -- github.com/golangci/go-misc: [927a3d8](https://github.com/golangci/go-misc/tree/927a3d8) -- github.com/golangci/go-tools: [e32c541](https://github.com/golangci/go-tools/tree/e32c541) -- github.com/golangci/goconst: [041c5f2](https://github.com/golangci/goconst/tree/041c5f2) -- github.com/golangci/gocyclo: [2becd97](https://github.com/golangci/gocyclo/tree/2becd97) -- github.com/golangci/gofmt: [0b8337e](https://github.com/golangci/gofmt/tree/0b8337e) -- github.com/golangci/golangci-lint: [v1.18.0](https://github.com/golangci/golangci-lint/tree/v1.18.0) -- github.com/golangci/gosec: [66fb7fc](https://github.com/golangci/gosec/tree/66fb7fc) -- github.com/golangci/ineffassign: [42439a7](https://github.com/golangci/ineffassign/tree/42439a7) -- github.com/golangci/lint-1: [ee948d0](https://github.com/golangci/lint-1/tree/ee948d0) -- github.com/golangci/maligned: [b1d8939](https://github.com/golangci/maligned/tree/b1d8939) -- github.com/golangci/misspell: [950f5d1](https://github.com/golangci/misspell/tree/950f5d1) -- github.com/golangci/prealloc: [215b22d](https://github.com/golangci/prealloc/tree/215b22d) -- github.com/golangci/revgrep: [d9c87f5](https://github.com/golangci/revgrep/tree/d9c87f5) -- github.com/golangci/unconvert: [28b1c44](https://github.com/golangci/unconvert/tree/28b1c44) -- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) -- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) -- github.com/golangplus/testing: [af21d9c](https://github.com/golangplus/testing/tree/af21d9c) -- github.com/google/cadvisor: [v0.35.0](https://github.com/google/cadvisor/tree/v0.35.0) -- github.com/google/renameio: [v0.1.0](https://github.com/google/renameio/tree/v0.1.0) -- github.com/gopherjs/gopherjs: [0766667](https://github.com/gopherjs/gopherjs/tree/0766667) -- github.com/gostaticanalysis/analysisutil: [v0.0.3](https://github.com/gostaticanalysis/analysisutil/tree/v0.0.3) -- github.com/hashicorp/go-syslog: [v1.0.0](https://github.com/hashicorp/go-syslog/tree/v1.0.0) -- github.com/heketi/heketi: [c2e2a4a](https://github.com/heketi/heketi/tree/c2e2a4a) -- github.com/heketi/tests: [f3775cb](https://github.com/heketi/tests/tree/f3775cb) -- github.com/jellevandenhooff/dkim: [f50fe3d](https://github.com/jellevandenhooff/dkim/tree/f50fe3d) -- github.com/jimstudt/http-authentication: [3eca13d](https://github.com/jimstudt/http-authentication/tree/3eca13d) -- github.com/jtolds/gls: [v4.20.0+incompatible](https://github.com/jtolds/gls/tree/v4.20.0) -- github.com/jung-kurt/gofpdf: [24315ac](https://github.com/jung-kurt/gofpdf/tree/24315ac) -- github.com/karrick/godirwalk: [v1.7.5](https://github.com/karrick/godirwalk/tree/v1.7.5) -- github.com/kubernetes-csi/csi-proxy/client: [9eff164](https://github.com/kubernetes-csi/csi-proxy/client/tree/9eff164) -- github.com/kylelemons/godebug: [d65d576](https://github.com/kylelemons/godebug/tree/d65d576) -- github.com/libopenstorage/openstorage: [v1.0.0](https://github.com/libopenstorage/openstorage/tree/v1.0.0) -- github.com/liggitt/tabwriter: [89fcab3](https://github.com/liggitt/tabwriter/tree/89fcab3) -- github.com/lithammer/dedent: [v1.1.0](https://github.com/lithammer/dedent/tree/v1.1.0) -- github.com/logrusorgru/aurora: [a7b3b31](https://github.com/logrusorgru/aurora/tree/a7b3b31) -- github.com/lpabon/godbc: [v0.1.1](https://github.com/lpabon/godbc/tree/v0.1.1) -- github.com/lucas-clemente/aes12: [cd47fb3](https://github.com/lucas-clemente/aes12/tree/cd47fb3) -- github.com/lucas-clemente/quic-clients: [v0.1.0](https://github.com/lucas-clemente/quic-clients/tree/v0.1.0) -- github.com/lucas-clemente/quic-go-certificates: [d2f8652](https://github.com/lucas-clemente/quic-go-certificates/tree/d2f8652) -- github.com/lucas-clemente/quic-go: [v0.10.2](https://github.com/lucas-clemente/quic-go/tree/v0.10.2) -- github.com/marten-seemann/qtls: [v0.2.3](https://github.com/marten-seemann/qtls/tree/v0.2.3) -- github.com/mattn/go-shellwords: [v1.0.5](https://github.com/mattn/go-shellwords/tree/v1.0.5) -- github.com/mattn/goveralls: [v0.0.2](https://github.com/mattn/goveralls/tree/v0.0.2) -- github.com/mesos/mesos-go: [v0.0.9](https://github.com/mesos/mesos-go/tree/v0.0.9) -- github.com/mholt/certmagic: [6a42ef9](https://github.com/mholt/certmagic/tree/6a42ef9) -- github.com/miekg/dns: [v1.1.4](https://github.com/miekg/dns/tree/v1.1.4) -- github.com/mindprince/gonvml: [9ebdce4](https://github.com/mindprince/gonvml/tree/9ebdce4) -- github.com/mistifyio/go-zfs: [v2.1.1+incompatible](https://github.com/mistifyio/go-zfs/tree/v2.1.1) -- github.com/mitchellh/go-ps: [4fdf99a](https://github.com/mitchellh/go-ps/tree/4fdf99a) -- github.com/mitchellh/go-wordwrap: [v1.0.0](https://github.com/mitchellh/go-wordwrap/tree/v1.0.0) -- github.com/mohae/deepcopy: [491d360](https://github.com/mohae/deepcopy/tree/491d360) -- github.com/morikuni/aec: [v1.0.0](https://github.com/morikuni/aec/tree/v1.0.0) -- github.com/mozilla/tls-observatory: [8791a20](https://github.com/mozilla/tls-observatory/tree/8791a20) -- github.com/mrunalp/fileutils: [7d4729f](https://github.com/mrunalp/fileutils/tree/7d4729f) -- github.com/mvdan/xurls: [v1.1.0](https://github.com/mvdan/xurls/tree/v1.1.0) -- github.com/naoina/go-stringutil: [v0.1.0](https://github.com/naoina/go-stringutil/tree/v0.1.0) -- github.com/naoina/toml: [v0.1.1](https://github.com/naoina/toml/tree/v0.1.1) -- github.com/nbutton23/zxcvbn-go: [eafdab6](https://github.com/nbutton23/zxcvbn-go/tree/eafdab6) -- github.com/opencontainers/runc: [v1.0.0-rc10](https://github.com/opencontainers/runc/tree/v1.0.0-rc10) -- github.com/opencontainers/runtime-spec: [v1.0.0](https://github.com/opencontainers/runtime-spec/tree/v1.0.0) -- github.com/opencontainers/selinux: [5215b18](https://github.com/opencontainers/selinux/tree/5215b18) -- github.com/pquerna/ffjson: [af8b230](https://github.com/pquerna/ffjson/tree/af8b230) -- github.com/quasilyte/go-consistent: [c6f3937](https://github.com/quasilyte/go-consistent/tree/c6f3937) -- github.com/quobyte/api: [v0.1.2](https://github.com/quobyte/api/tree/v0.1.2) -- github.com/robfig/cron: [v1.1.0](https://github.com/robfig/cron/tree/v1.1.0) -- github.com/rogpeppe/fastuuid: [6724a57](https://github.com/rogpeppe/fastuuid/tree/6724a57) -- github.com/rogpeppe/go-internal: [v1.3.0](https://github.com/rogpeppe/go-internal/tree/v1.3.0) -- github.com/rubiojr/go-vhd: [0bfd3b3](https://github.com/rubiojr/go-vhd/tree/0bfd3b3) -- github.com/ryanuber/go-glob: [256dc44](https://github.com/ryanuber/go-glob/tree/256dc44) -- github.com/seccomp/libseccomp-golang: [v0.9.1](https://github.com/seccomp/libseccomp-golang/tree/v0.9.1) -- github.com/sergi/go-diff: [v1.0.0](https://github.com/sergi/go-diff/tree/v1.0.0) -- github.com/shirou/gopsutil: [c95755e](https://github.com/shirou/gopsutil/tree/c95755e) -- github.com/shirou/w32: [bb4de01](https://github.com/shirou/w32/tree/bb4de01) -- github.com/shurcooL/go-goon: [37c2f52](https://github.com/shurcooL/go-goon/tree/37c2f52) -- github.com/smartystreets/assertions: [b2de0cb](https://github.com/smartystreets/assertions/tree/b2de0cb) -- github.com/smartystreets/goconvey: [v1.6.4](https://github.com/smartystreets/goconvey/tree/v1.6.4) -- github.com/sourcegraph/go-diff: [v0.5.1](https://github.com/sourcegraph/go-diff/tree/v0.5.1) -- github.com/storageos/go-api: [343b3ef](https://github.com/storageos/go-api/tree/343b3ef) -- github.com/syndtr/gocapability: [d983527](https://github.com/syndtr/gocapability/tree/d983527) -- github.com/tarm/serial: [98f6abe](https://github.com/tarm/serial/tree/98f6abe) -- github.com/thecodeteam/goscaleio: [v0.1.0](https://github.com/thecodeteam/goscaleio/tree/v0.1.0) -- github.com/tidwall/pretty: [v1.0.0](https://github.com/tidwall/pretty/tree/v1.0.0) -- github.com/timakin/bodyclose: [87058b9](https://github.com/timakin/bodyclose/tree/87058b9) -- github.com/ultraware/funlen: [v0.0.2](https://github.com/ultraware/funlen/tree/v0.0.2) -- github.com/urfave/negroni: [v1.0.0](https://github.com/urfave/negroni/tree/v1.0.0) -- github.com/valyala/bytebufferpool: [v1.0.0](https://github.com/valyala/bytebufferpool/tree/v1.0.0) -- github.com/valyala/fasthttp: [v1.2.0](https://github.com/valyala/fasthttp/tree/v1.2.0) -- github.com/valyala/quicktemplate: [v1.1.1](https://github.com/valyala/quicktemplate/tree/v1.1.1) -- github.com/valyala/tcplisten: [ceec8f9](https://github.com/valyala/tcplisten/tree/ceec8f9) -- github.com/vektah/gqlparser: [v1.1.2](https://github.com/vektah/gqlparser/tree/v1.1.2) -- github.com/vishvananda/netlink: [v1.0.0](https://github.com/vishvananda/netlink/tree/v1.0.0) -- github.com/vishvananda/netns: [be1fbed](https://github.com/vishvananda/netns/tree/be1fbed) -- github.com/vmware/govmomi: [v0.20.3](https://github.com/vmware/govmomi/tree/v0.20.3) -- go.mongodb.org/mongo-driver: v1.1.2 -- go4.org: 417644f -- golang.org/x/build: 2835ba2 -- golang.org/x/mod: c90efee -- golang.org/x/perf: 6e6d33e -- golang.org/x/tools/gopls: v0.3.3 -- gonum.org/v1/plot: e2840ee -- gopkg.in/errgo.v2: v2.1.0 -- gopkg.in/mcuadros/go-syslog.v2: v2.2.1 -- gopkg.in/resty.v1: v1.12.0 -- gotest.tools/gotestsum: v0.3.5 -- grpc.go4.org: 11d0a25 -- k8s.io/cli-runtime: v0.18.0 -- k8s.io/cloud-provider: v0.18.0 -- k8s.io/cluster-bootstrap: v0.18.0 -- k8s.io/cri-api: v0.18.0 -- k8s.io/csi-translation-lib: v0.18.0 -- k8s.io/heapster: v1.2.0-beta.1 -- k8s.io/kube-aggregator: v0.18.0 -- k8s.io/kube-controller-manager: v0.18.0 -- k8s.io/kube-proxy: v0.18.0 -- k8s.io/kube-scheduler: v0.18.0 -- k8s.io/kubectl: v0.18.0 -- k8s.io/kubelet: v0.18.0 -- k8s.io/legacy-cloud-providers: v0.18.0 -- k8s.io/metrics: v0.18.0 -- k8s.io/repo-infra: v0.0.1-alpha.1 -- k8s.io/sample-apiserver: v0.18.0 -- k8s.io/system-validators: v1.0.4 -- mvdan.cc/interfacer: c200402 -- mvdan.cc/lint: adc824a -- mvdan.cc/unparam: fbb5962 -- rsc.io/pdf: v0.1.1 -- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.7 -- sigs.k8s.io/kustomize: v2.0.3+incompatible -- sigs.k8s.io/structured-merge-diff/v3: v3.0.0 -- sourcegraph.com/sqs/pbtypes: d3ebe8f - -### Changed -- github.com/Azure/azure-sdk-for-go: [v21.1.0+incompatible → v35.0.0+incompatible](https://github.com/Azure/azure-sdk-for-go/compare/v21.1.0...v35.0.0) -- github.com/GoogleCloudPlatform/k8s-cloud-provider: [2e19bb3 → 27a4ced](https://github.com/GoogleCloudPlatform/k8s-cloud-provider/compare/2e19bb3...27a4ced) -- github.com/asaskevich/govalidator: [f9ffefc → f61b66f](https://github.com/asaskevich/govalidator/compare/f9ffefc...f61b66f) -- github.com/aws/aws-sdk-go: [v1.23.22 → v1.28.2](https://github.com/aws/aws-sdk-go/compare/v1.23.22...v1.28.2) -- github.com/coreos/etcd: [v3.3.13+incompatible → v3.3.10+incompatible](https://github.com/coreos/etcd/compare/v3.3.13...v3.3.10) -- github.com/coreos/go-oidc: [065b426 → v2.1.0+incompatible](https://github.com/coreos/go-oidc/compare/065b426...v2.1.0) -- github.com/coreos/go-semver: [v0.2.0 → v0.3.0](https://github.com/coreos/go-semver/compare/v0.2.0...v0.3.0) -- github.com/coreos/go-systemd: [39ca1b0 → 95778df](https://github.com/coreos/go-systemd/compare/39ca1b0...95778df) -- github.com/docker/distribution: [83389a1 → v2.7.1+incompatible](https://github.com/docker/distribution/compare/83389a1...v2.7.1) -- github.com/docker/go-units: [v0.3.3 → v0.4.0](https://github.com/docker/go-units/compare/v0.3.3...v0.4.0) -- github.com/elazarl/goproxy: [c4fc265 → 947c36d](https://github.com/elazarl/goproxy/compare/c4fc265...947c36d) -- github.com/envoyproxy/go-control-plane: [v0.9.0 → 5f8ba28](https://github.com/envoyproxy/go-control-plane/compare/v0.9.0...5f8ba28) -- github.com/go-openapi/analysis: [v0.17.2 → v0.19.5](https://github.com/go-openapi/analysis/compare/v0.17.2...v0.19.5) -- github.com/go-openapi/errors: [v0.17.2 → v0.19.2](https://github.com/go-openapi/errors/compare/v0.17.2...v0.19.2) -- github.com/go-openapi/loads: [v0.17.2 → v0.19.4](https://github.com/go-openapi/loads/compare/v0.17.2...v0.19.4) -- github.com/go-openapi/runtime: [v0.17.2 → v0.19.4](https://github.com/go-openapi/runtime/compare/v0.17.2...v0.19.4) -- github.com/go-openapi/strfmt: [v0.17.0 → v0.19.3](https://github.com/go-openapi/strfmt/compare/v0.17.0...v0.19.3) -- github.com/go-openapi/validate: [v0.18.0 → v0.19.5](https://github.com/go-openapi/validate/compare/v0.18.0...v0.19.5) -- github.com/gogo/protobuf: [65acae2 → v1.3.1](https://github.com/gogo/protobuf/compare/65acae2...v1.3.1) -- github.com/golang/protobuf: [v1.3.2 → v1.3.4](https://github.com/golang/protobuf/compare/v1.3.2...v1.3.4) -- github.com/google/gofuzz: [v1.0.0 → v1.1.0](https://github.com/google/gofuzz/compare/v1.0.0...v1.1.0) -- github.com/gophercloud/gophercloud: [c818fa6 → v0.1.0](https://github.com/gophercloud/gophercloud/compare/c818fa6...v0.1.0) -- github.com/gorilla/mux: [v1.6.2 → v1.7.0](https://github.com/gorilla/mux/compare/v1.6.2...v1.7.0) -- github.com/gorilla/websocket: [4201258 → v1.4.0](https://github.com/gorilla/websocket/compare/4201258...v1.4.0) -- github.com/grpc-ecosystem/go-grpc-middleware: [v1.0.0 → f849b54](https://github.com/grpc-ecosystem/go-grpc-middleware/compare/v1.0.0...f849b54) -- github.com/grpc-ecosystem/grpc-gateway: [v1.4.1 → v1.9.5](https://github.com/grpc-ecosystem/grpc-gateway/compare/v1.4.1...v1.9.5) -- github.com/mailru/easyjson: [b2ccc51 → v0.7.0](https://github.com/mailru/easyjson/compare/b2ccc51...v0.7.0) -- github.com/mattn/go-isatty: [v0.0.4 → v0.0.9](https://github.com/mattn/go-isatty/compare/v0.0.4...v0.0.9) -- github.com/munnerz/goautoneg: [a547fc6 → a7dc8b6](https://github.com/munnerz/goautoneg/compare/a547fc6...a7dc8b6) -- github.com/onsi/ginkgo: [v1.10.3 → v1.11.0](https://github.com/onsi/ginkgo/compare/v1.10.3...v1.11.0) -- github.com/prometheus/client_model: [14fe0d1 → v0.2.0](https://github.com/prometheus/client_model/compare/14fe0d1...v0.2.0) -- github.com/satori/go.uuid: [0aa62d5 → v1.2.0](https://github.com/satori/go.uuid/compare/0aa62d5...v1.2.0) -- github.com/spf13/jwalterweatherman: [v1.0.0 → v1.1.0](https://github.com/spf13/jwalterweatherman/compare/v1.0.0...v1.1.0) -- github.com/urfave/cli: [v1.18.0 → v1.20.0](https://github.com/urfave/cli/compare/v1.18.0...v1.20.0) -- github.com/xiang90/probing: [07dd2e8 → 43a291a](https://github.com/xiang90/probing/compare/07dd2e8...43a291a) -- go.etcd.io/bbolt: v1.3.1-etcd.7 → v1.3.3 -- go.etcd.io/etcd: 83304cf → 3cf2f69 -- go.uber.org/zap: v1.9.1 → v1.10.0 -- golang.org/x/crypto: 5c40567 → bac4c82 -- golang.org/x/tools: 5eefd05 → 6862ede -- golang.org/x/xerrors: a985d34 → 1b5146a -- gonum.org/v1/gonum: 3d26580 → v0.6.2 -- google.golang.org/grpc: v1.25.1 → v1.27.1 -- gopkg.in/natefinch/lumberjack.v2: 20b71e5 → v2.0.0 -- gopkg.in/square/go-jose.v2: 89060de → v2.2.2 -- gopkg.in/yaml.v2: v2.2.7 → v2.2.8 -- honnef.co/go/tools: ea95bdf → v0.0.1-2020.1.3 -- k8s.io/api: bd6ac52 → v0.18.0 -- k8s.io/apiextensions-apiserver: 3de7581 → v0.18.0 -- k8s.io/apimachinery: v0.17.1 → v0.18.0 -- k8s.io/apiserver: 1e17798 → v0.18.0 -- k8s.io/client-go: 6502b5e → v0.18.0 -- k8s.io/code-generator: 732c9ca → v0.18.0 -- k8s.io/component-base: ed2f086 → v0.18.0 -- k8s.io/gengo: 26a6646 → 36b2048 -- k8s.io/kube-openapi: 30be4d1 → bf4fb3b -- k8s.io/kubernetes: v1.14.7 → v1.18.0 -- k8s.io/utils: 8619460 → a9aa75a -- mvdan.cc/xurls/v2: v2.0.0 → v2.1.0 -- sigs.k8s.io/yaml: v1.1.0 → v1.2.0 - -### Removed -- github.com/coreos/bbolt: [v1.3.1-coreos.6](https://github.com/coreos/bbolt/tree/v1.3.1-coreos.6) -- github.com/natefinch/lumberjack: [v2.0.0+incompatible](https://github.com/natefinch/lumberjack/tree/v2.0.0) -- gopkg.in/yaml.v1: 9f9df34 -- sigs.k8s.io/structured-merge-diff: 15d366b diff --git a/CHANGELOG/CHANGELOG-1.1.md b/CHANGELOG/CHANGELOG-1.1.md deleted file mode 100644 index c7ffa593..00000000 --- a/CHANGELOG/CHANGELOG-1.1.md +++ /dev/null @@ -1,47 +0,0 @@ -# v1.1.0 - Changelog since v1.0.0 - -## Changes by Kind - -## Feature - -- Improved Windows Support - - Update driver to use CSI proxy beta for Windows ([#607](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/607), [@jingxu97](https://github.com/jingxu97)) - - Add volume expansion support for Windows in GCE PD CSI driver ([#637](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/637), [@jingxu97](https://github.com/jingxu97)) - - Add defensive check for Windows. GCE PD CSI driver only support ntfs for Windows. If other fstype is passed, return error. ([#641](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/641), [@jingxu97](https://github.com/jingxu97)) - - Modify NodeUnstageVolume call for Windows to use csi_proxy dismount call. With CSI proxy v0.2.2+, this will also result in flush of data cache before mount point removal. ([#633](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/633), [@jingxu97](https://github.com/jingxu97)) - - Add VolumeStats for Windows ([#627](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/627), [@jingxu97](https://github.com/jingxu97)) - -## Bug or Regression - -- Add PSP for the controller Deployment ([#623](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/623), [@george-angel](https://github.com/george-angel)) -- Update GCE PD CSI Driver Docker base image to `k8s.gcr.io/build-image/debian-base-amd64:v2.1.3` (previously `gcr.io/google-containers/debian-base-amd64:v2.0.0`) to address CVEs. ([#596](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/596), [@saad-ali](https://github.com/saad-ali)) - - Also cherry picked to 1.0.1. - -## Tests - -- PD CSI e2e test infra should take GKE node version as an optional input argument. ([#603](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/603), [@saikat-royc](https://github.com/saikat-royc)) -- Collect managed pd csi driver logs from node ([#619](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/619), [@saikat-royc](https://github.com/saikat-royc)) -- Enable dump GKE node logs ([#635](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/635), [@saikat-royc](https://github.com/saikat-royc)) -- Enable volume expansion test for GKE managed driver ([#584](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/584), [@saikat-royc](https://github.com/saikat-royc)) -- Provide a knob to run intree and csi plugin tests ([#629](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/629), [@saikat-royc](https://github.com/saikat-royc)) -- Fix CI script focus string ([#630](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/630), [@saikat-royc](https://github.com/saikat-royc)) -- Build only linux container image for tests on Linux ([#636](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/636), [@jingxu97](https://github.com/jingxu97)) - -## Dependencies - -### Added -- google.golang.org/protobuf: v1.25.0 - -### Changed -- github.com/golang/protobuf: [v1.3.4 → v1.4.1](https://github.com/golang/protobuf/compare/v1.3.4...v1.4.1) -- github.com/google/go-cmp: [v0.3.1 → v0.5.0](https://github.com/google/go-cmp/compare/v0.3.1...v0.5.0) -- github.com/kubernetes-csi/csi-proxy/client: [9eff164 → v0.2.1](https://github.com/kubernetes-csi/csi-proxy/client/compare/9eff164...v0.2.1) -- golang.org/x/mod: c90efee → 4bf6d31 -- golang.org/x/tools: 6862ede → 5eefd05 -- golang.org/x/xerrors: 1b5146a → 9bdfabe -- google.golang.org/genproto: 6bbd007 → cb27e3a -- honnef.co/go/tools: v0.0.1-2020.1.3 → v0.0.1-2019.2.2 -- mvdan.cc/xurls/v2: v2.1.0 → v2.0.0 - -### Removed -- golang.org/x/tools/gopls: v0.3.3 diff --git a/CHANGELOG/CHANGELOG-1.2.md b/CHANGELOG/CHANGELOG-1.2.md deleted file mode 100644 index 75a70458..00000000 --- a/CHANGELOG/CHANGELOG-1.2.md +++ /dev/null @@ -1,139 +0,0 @@ -# v1.2.6 - Changelog since v1.2.5 - -## Changes by Kind - -### Uncategorized - -- Cherry-pick #930: Update golang version to 1.17.8 for building drivers. ([#938](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/938), [@pwschuurman](https://github.com/pwschuurman)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.2.5 - Changelog since v1.2.4 - -- Update base image to buster-1.10.0 - -# v1.2.4 - Changelog since v1.2.3 - -## Changes by Kind - -### Bug or Regression - -- Update go builder to 1.17 ([#850](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/850), [@mattcary](https://github.com/mattcary)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.2.3 - Changelog since v1.2.1 - -## Changes by Kind - -### Bug or Regression - -- Update debian images to buster-v1.9.0 ([#839](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/839), [@mattcary](https://github.com/mattcary)) - -### Uncategorized - -- It is now possible to mount a volume with XFS filesystem and its restored snapshot. ([#838](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/838), [@leiyiz](https://github.com/leiyiz)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.2.2 - Changelog since v1.2.1 - -## Changes by Kind - -### Feature - -- Update base image to buster-1.6.0 ([#760](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/760), [@cpanato](https://github.com/cpanato)) - -- Update base image to buster-1.5.0. ([#755](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/755), [@mattcary](https://github.com/mattcary)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - - -# v1.2.1 - Changelog since v1.2.0 - -## Tests - -- Update kustomize to 3.9.4 ([703](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/703), [@saikat-royc](https://github.com/saikat-royc)) -- Fix cluster list parsing for latest gcloud version ([720](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/720), [@verult](https://github.com/verult)) - -## Other - -- Remove Probe logging ([682](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/682), [@Jiawei0227](https://github.com/Jiawei0227)) -- Round up pdcsi driver size in CreateVolume ([684](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/684), [@Jiawei0227](https://github.com/Jiawei0227)) -- Add gce disk labels support via create volume parameters ([718](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/718), [@mattcary](https://github.com/mattcary)) -- Emit GKE PDCSI component version metric ([719](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/719), [@saikat-royc](https://github.com/saikat-royc)) -- Add cloudbuild configuration to build the image gcp-compute-persistent-disk-csi-driver ([734](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/734), [@cpanato](https://github.com/cpanato)) -- Bump go to the latest 1.13 available in Dockerfile ([734](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/734), [@cpanato](https://github.com/cpanato)) - -# v1.2.0 - Changelog since v1.1.0 - -## Features - -- Improved Windows Support - - Add Disk online/offline logic in nodeStageVolume/nodeUnstageVolume calls for Windows. This requires the CSI Proxy `disk.v1beta2` group, which is only available in CSI proxy v0.2.2+. ([#661](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/661), [@jingxu97](https://github.com/jingxu97)) - -## Bugs or Regressions - -- Fix "volume is mounted" check during NodePublishVolume for Windows ([#666](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/666), [@jingxu97](https://github.com/jingxu97)) -- Add empty string check on returned volumeIds to avoid nil pointer panic ([#673](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/673), [@jingxu97](https://github.com/jingxu97)) - -## Tests - -- Update kustomizer to 3.8.6. ([#661](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/661), [@jingxu97](https://github.com/jingxu97)) -- Updates named pipe path in node.yaml for base deployment overlay to use CSI proxy disk v1beta2 (previously v1beta1) for Windows. ([#669](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/669), [@jingxu97](https://github.com/jingxu97)) -- Fix node version check for node skew tests. ([#645](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/645), [@saikat-royc](https://github.com/saikat-royc)) -- Add run-k8s-integration-ci.sh that the prow driver now expects to be in each release. ([#655](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/655), [@mattcary](https://github.com/mattcary)) -- Use node version on GKE when detecting XFS compatibility. ([#656](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/656), [@mattcary](https://github.com/mattcary)) -- Skip Pod fsgroupchange policy tests for < 1.20 k8s. ([#667](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/667), [@saikat-royc](https://github.com/saikat-royc)) -- Shorten the GKE cluster name. ([#671](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/671), [@saikat-royc](https://github.com/saikat-royc)) - -## Documentation - -No notable changes. - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -- github.com/kubernetes-csi/csi-proxy/client: [v0.2.1 → v0.2.2](https://github.com/kubernetes-csi/csi-proxy/client/compare/v0.2.1...v0.2.2) - -### Removed -_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.3.md b/CHANGELOG/CHANGELOG-1.3.md deleted file mode 100644 index 5ae26b7a..00000000 --- a/CHANGELOG/CHANGELOG-1.3.md +++ /dev/null @@ -1,150 +0,0 @@ -# v1.3.8 - Changelog since v1.3.7 - -- Update base image to bullseye-v1.4.2 to fix CVE-2021-46828, CVE-2022-2509. - -# v1.3.7 - Changelog since v1.3.6 - -- Update to go1.18.4 and base image to bullseye-v1.4.1 to fix CVE-2022-1271, CVE-2022-1664, CVE-2022-24675, CVE-2022-34903, CVE-2018-25032, CVE-2022-28327, CVE-2021-43618 ([#1033](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1033), [@mattcary](https://github.com/mattcary)) -- Default to MAXPROCS=1 to improve memory usage on nodes with many CPUs. ([#1022](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1022), [@mattcary](https://github.com/mattcary)) -- Remove passwd- file to make CIS benchmark happy. ([#1021](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1021), [@mattcary](https://github.com/mattcary)) - -# v1.3.6 - Changelog since v1.3.5 - -## Changes by Kind - -### Uncategorized - -- Cherry-pick #930: Update golang version to 1.17.8 for building drivers. ([#937](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/937), [@pwschuurman](https://github.com/pwschuurman)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.3.5 - Changelog since v1.3.4 - -- Bump base image to buster-v1.10.0 - - -# v1.3.4 - Changelog since v1.3.3 - -## Changes by Kind - -### Bug or Regression - -- Update go builder to 1.17 ([#850](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/850), [@mattcary](https://github.com/mattcary)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.3.3 - Changelog since v1.3.1 - -## Changes by Kind - -### Bug or Regression - -- Update debian image to buster-1.9.0. ([#841](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/841), [@mattcary](https://github.com/mattcary)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.3.1 - Changelog since v1.3.0 - -### Issues - -- Fixes issue where `ControllerPublishVolume` is called repeatly if gke nodes are in different cloud zones than the gke controller ([#817](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/817), [@leiyiz](https://github.com/leiyiz)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.3.0 - Changelog since v1.2.2 - -### Feature - -- A new `k8s-tag-cluster-id` command line option has been added. If specified, the resulting PD disk will be labeled with "kubernetes_io_cluster_": "owned". ([#693](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/693), [@tsmetana](https://github.com/tsmetana)) -- Add cloudbuild config to build gcp-compute-persistent-disk-csi-driver image ([#724](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/724), [@cpanato](https://github.com/cpanato)) -- Added Support for Windows Server 2004 and 20H2. ([#691](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/691), [@jeremyje](https://github.com/jeremyje)) -- Bumped csi-proxy client library to v1.0.0 ([#738](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/738), [@mauriciopoppe](https://github.com/mauriciopoppe)) -- It is now possible to access snapshots and volumes across different projects. ([#782](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/782), [@christian-roggia](https://github.com/christian-roggia)) -- Updating the following image versions in stable deployment specs: - - csi-provisioner: v2.1.0 - - csi-attacher: v3.1.0 - - csi-resizer: v1.1.0 - - csi-snapshotter: v3.0.3 - - csi-node-driver-registrar: v2.1.0 - - Adding a liveness probe to restart a sidecar if it fails leader election health check. ([#699](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/699), [@verult](https://github.com/verult)) -- Users will be able to set the storage locations for snapshots by specifying them in the snapshot class. ([#793](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/793), [@TeweiLuo](https://github.com/TeweiLuo)) -- Disk labels support via CreateVolume (and hence StorageClass) parameters ([#718](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/718), [@mattcary](https://github.com/mattcary)) - -### Documentation - -- Documentation for overlays ([#708](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/708), [@saikat-royc](https://github.com/saikat-royc)) -- Update README for overlays ([#715](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/715), [@saikat-royc](https://github.com/saikat-royc)) - -### Failing Test - -- V1 CSIDriver resources are deployed for 1.18+ clusters. ([#783](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/783), [@mattcary](https://github.com/mattcary)) - -### Bug or Regression - -- Do not run controller service in node. ([#702](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/702), [@mattcary](https://github.com/mattcary)) -- Fix a bug that CreateVolume should round up the request_bytes. ([#684](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/684), [@Jiawei0227](https://github.com/Jiawei0227)) -- It is now possible to mount a volume with XFS filesystem and its restored snapshot. ([#788](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/788), [@jsafrane](https://github.com/jsafrane)) - -### Other (Cleanup or Flake) - -- Emit GKE component version metric ([#719](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/719), [@saikat-royc](https://github.com/saikat-royc)) - -### Uncategorized - -- Remove probe logging to reduce noise ([#682](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/682), [@Jiawei0227](https://github.com/Jiawei0227)) - -## Dependencies - -### Added -- k8s.io/klog/v2: v2.4.0 -- k8s.io/mount-utils: v0.20.6 - -### Changed -- github.com/Microsoft/go-winio: [v0.4.14 → v0.4.16](https://github.com/Microsoft/go-winio/compare/v0.4.14...v0.4.16) -- github.com/go-logr/logr: [v0.1.0 → v0.2.0](https://github.com/go-logr/logr/compare/v0.1.0...v0.2.0) -- github.com/kr/pretty: [v0.1.0 → v0.2.0](https://github.com/kr/pretty/compare/v0.1.0...v0.2.0) -- github.com/kubernetes-csi/csi-proxy/client: [v0.2.2 → v1.0.0](https://github.com/kubernetes-csi/csi-proxy/client/compare/v0.2.2...v1.0.0) -- github.com/pkg/errors: [v0.8.1 → v0.9.1](https://github.com/pkg/errors/compare/v0.8.1...v0.9.1) -- github.com/stretchr/testify: [v1.4.0 → v1.6.1](https://github.com/stretchr/testify/compare/v1.4.0...v1.6.1) -- gopkg.in/check.v1: 788fd78 → 41f04d3 -- gopkg.in/yaml.v3: 674ba3e → 9f266ea -- k8s.io/utils: a9aa75a → 67b214c - -### Removed -_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.4.md b/CHANGELOG/CHANGELOG-1.4.md deleted file mode 100644 index ab67839c..00000000 --- a/CHANGELOG/CHANGELOG-1.4.md +++ /dev/null @@ -1,439 +0,0 @@ -# v1.4.1 - Changelog since v1.4.0 - -## Changes by Kind - -### Bug or Regression - -- Fix #942 that can cause attacher to think a disk has been attached even if the attach failed. (#945, @mattcary) - -### Uncategorized - -- Add support for NVMe persistent disks (#946, @pwschuurman) -- Cherry-pick #930: Update golang version to 1.17.8 for building drivers. (#936, @pwschuurman) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - -# v1.4.0 - Changelog since v1.3.4 - -## Changes by Kind - -### Feature - -- Updates Kubernetes dependencies to v1.22.0 ([#814](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/814), [@chrishenzie](https://github.com/chrishenzie)) -- Add attach/detach back off ([#847](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/847), [@lizhuqi](https://github.com/lizhuqi)) -- Enables volume cloning. ([#854](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/854), [@amacaskill](https://github.com/amacaskill)) -- Use the most recent 1.3.4 image for prow rc master ([#864](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/864), [@saikat-royc](https://github.com/saikat-royc)) -- Change to distroless base image ([#870](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/870), [@amacaskill](https://github.com/amacaskill)) -- Turn on controller-publish-readonly flag and add validation in pd-csi driver for when readonly is on ([#869](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/869), [@leiyiz](https://github.com/leiyiz)) - -### Documentation - -- Doc and image update for 1.3.4 release ([#855](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/855), [@saikat-royc](https://github.com/saikat-royc)) -- Update release 1.2.4 CHANGELOG ([#862](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/862), [@saikat-royc](https://github.com/saikat-royc)) - -### Bug or Regression - -### Other (Cleanup or Flake) - -- Update debian base image to 1.9.0 ([#826](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/826), [@saikat-royc](https://github.com/saikat-royc)) -- Updates the CSI sanity test suite to v4.2.0 ([#816](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/816), [@chrishenzie](https://github.com/chrishenzie)) - -### Uncategorized - -- Update snapshot sidecar roles ([#857](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/857), [@saikat-royc](https://github.com/saikat-royc)) - -## Dependencies - -### Added -- cloud.google.com/go/firestore: v1.1.0 -- github.com/OneOfOne/xxhash: [v1.2.2](https://github.com/OneOfOne/xxhash/tree/v1.2.2) -- github.com/antihax/optional: [v1.0.0](https://github.com/antihax/optional/tree/v1.0.0) -- github.com/armon/go-metrics: [f0300d1](https://github.com/armon/go-metrics/tree/f0300d1) -- github.com/armon/go-radix: [7fddfc3](https://github.com/armon/go-radix/tree/7fddfc3) -- github.com/benbjohnson/clock: [v1.0.3](https://github.com/benbjohnson/clock/tree/v1.0.3) -- github.com/bits-and-blooms/bitset: [v1.2.0](https://github.com/bits-and-blooms/bitset/tree/v1.2.0) -- github.com/bketelsen/crypt: [5cbc8cc](https://github.com/bketelsen/crypt/tree/5cbc8cc) -- github.com/certifi/gocertifi: [2c3bb06](https://github.com/certifi/gocertifi/tree/2c3bb06) -- github.com/cespare/xxhash/v2: [v2.1.1](https://github.com/cespare/xxhash/v2/tree/v2.1.1) -- github.com/cespare/xxhash: [v1.1.0](https://github.com/cespare/xxhash/tree/v1.1.0) -- github.com/checkpoint-restore/go-criu/v5: [v5.0.0](https://github.com/checkpoint-restore/go-criu/v5/tree/v5.0.0) -- github.com/cockroachdb/errors: [v1.2.4](https://github.com/cockroachdb/errors/tree/v1.2.4) -- github.com/cockroachdb/logtags: [eb05cc2](https://github.com/cockroachdb/logtags/tree/eb05cc2) -- github.com/containerd/cgroups: [0dbf7f0](https://github.com/containerd/cgroups/tree/0dbf7f0) -- github.com/containerd/continuity: [aaeac12](https://github.com/containerd/continuity/tree/aaeac12) -- github.com/containerd/fifo: [a9fb20d](https://github.com/containerd/fifo/tree/a9fb20d) -- github.com/containerd/go-runc: [5a6d9f3](https://github.com/containerd/go-runc/tree/5a6d9f3) -- github.com/containerd/ttrpc: [v1.0.2](https://github.com/containerd/ttrpc/tree/v1.0.2) -- github.com/coredns/caddy: [v1.1.0](https://github.com/coredns/caddy/tree/v1.1.0) -- github.com/coreos/bbolt: [v1.3.2](https://github.com/coreos/bbolt/tree/v1.3.2) -- github.com/coreos/go-systemd/v22: [v22.3.2](https://github.com/coreos/go-systemd/v22/tree/v22.3.2) -- github.com/cpuguy83/go-md2man/v2: [v2.0.0](https://github.com/cpuguy83/go-md2man/v2/tree/v2.0.0) -- github.com/dgryski/go-sip13: [e10d5fe](https://github.com/dgryski/go-sip13/tree/e10d5fe) -- github.com/felixge/httpsnoop: [v1.0.1](https://github.com/felixge/httpsnoop/tree/v1.0.1) -- github.com/form3tech-oss/jwt-go: [v3.2.3+incompatible](https://github.com/form3tech-oss/jwt-go/tree/v3.2.3) -- github.com/frankban/quicktest: [v1.11.3](https://github.com/frankban/quicktest/tree/v1.11.3) -- github.com/fvbommel/sortorder: [v1.0.1](https://github.com/fvbommel/sortorder/tree/v1.0.1) -- github.com/getsentry/raven-go: [v0.2.0](https://github.com/getsentry/raven-go/tree/v0.2.0) -- github.com/go-errors/errors: [v1.0.1](https://github.com/go-errors/errors/tree/v1.0.1) -- github.com/go-kit/log: [v0.1.0](https://github.com/go-kit/log/tree/v0.1.0) -- github.com/godbus/dbus/v5: [v5.0.4](https://github.com/godbus/dbus/v5/tree/v5.0.4) -- github.com/gofrs/uuid: [v4.0.0+incompatible](https://github.com/gofrs/uuid/tree/v4.0.0) -- github.com/google/shlex: [e7afc7f](https://github.com/google/shlex/tree/e7afc7f) -- github.com/hashicorp/consul/api: [v1.1.0](https://github.com/hashicorp/consul/api/tree/v1.1.0) -- github.com/hashicorp/consul/sdk: [v0.1.1](https://github.com/hashicorp/consul/sdk/tree/v0.1.1) -- github.com/hashicorp/go-cleanhttp: [v0.5.1](https://github.com/hashicorp/go-cleanhttp/tree/v0.5.1) -- github.com/hashicorp/go-immutable-radix: [v1.0.0](https://github.com/hashicorp/go-immutable-radix/tree/v1.0.0) -- github.com/hashicorp/go-msgpack: [v0.5.3](https://github.com/hashicorp/go-msgpack/tree/v0.5.3) -- github.com/hashicorp/go-rootcerts: [v1.0.0](https://github.com/hashicorp/go-rootcerts/tree/v1.0.0) -- github.com/hashicorp/go-sockaddr: [v1.0.0](https://github.com/hashicorp/go-sockaddr/tree/v1.0.0) -- github.com/hashicorp/go-uuid: [v1.0.1](https://github.com/hashicorp/go-uuid/tree/v1.0.1) -- github.com/hashicorp/go.net: [v0.0.1](https://github.com/hashicorp/go.net/tree/v0.0.1) -- github.com/hashicorp/logutils: [v1.0.0](https://github.com/hashicorp/logutils/tree/v1.0.0) -- github.com/hashicorp/mdns: [v1.0.0](https://github.com/hashicorp/mdns/tree/v1.0.0) -- github.com/hashicorp/memberlist: [v0.1.3](https://github.com/hashicorp/memberlist/tree/v0.1.3) -- github.com/hashicorp/serf: [v0.8.2](https://github.com/hashicorp/serf/tree/v0.8.2) -- github.com/ishidawataru/sctp: [7c296d4](https://github.com/ishidawataru/sctp/tree/7c296d4) -- github.com/jmespath/go-jmespath/internal/testify: [v1.5.1](https://github.com/jmespath/go-jmespath/internal/testify/tree/v1.5.1) -- github.com/josharian/intern: [v1.0.0](https://github.com/josharian/intern/tree/v1.0.0) -- github.com/jpillora/backoff: [v1.0.0](https://github.com/jpillora/backoff/tree/v1.0.0) -- github.com/kubernetes-csi/csi-test/v4: [v4.2.0](https://github.com/kubernetes-csi/csi-test/v4/tree/v4.2.0) -- github.com/mitchellh/cli: [v1.0.0](https://github.com/mitchellh/cli/tree/v1.0.0) -- github.com/mitchellh/go-testing-interface: [v1.0.0](https://github.com/mitchellh/go-testing-interface/tree/v1.0.0) -- github.com/mitchellh/gox: [v0.4.0](https://github.com/mitchellh/gox/tree/v0.4.0) -- github.com/mitchellh/iochan: [v1.0.0](https://github.com/mitchellh/iochan/tree/v1.0.0) -- github.com/moby/ipvs: [v1.0.1](https://github.com/moby/ipvs/tree/v1.0.1) -- github.com/moby/spdystream: [v0.2.0](https://github.com/moby/spdystream/tree/v0.2.0) -- github.com/moby/sys/mountinfo: [v0.4.1](https://github.com/moby/sys/mountinfo/tree/v0.4.1) -- github.com/moby/term: [9d4ed18](https://github.com/moby/term/tree/9d4ed18) -- github.com/monochromegane/go-gitignore: [205db1a](https://github.com/monochromegane/go-gitignore/tree/205db1a) -- github.com/niemeyer/pretty: [a10e7ca](https://github.com/niemeyer/pretty/tree/a10e7ca) -- github.com/nxadm/tail: [v1.4.5](https://github.com/nxadm/tail/tree/v1.4.5) -- github.com/oklog/ulid: [v1.3.1](https://github.com/oklog/ulid/tree/v1.3.1) -- github.com/opentracing/opentracing-go: [v1.1.0](https://github.com/opentracing/opentracing-go/tree/v1.1.0) -- github.com/pascaldekloe/goe: [57f6aae](https://github.com/pascaldekloe/goe/tree/57f6aae) -- github.com/posener/complete: [v1.1.1](https://github.com/posener/complete/tree/v1.1.1) -- github.com/prometheus/tsdb: [v0.7.1](https://github.com/prometheus/tsdb/tree/v0.7.1) -- github.com/robertkrimen/otto: [ef014fd](https://github.com/robertkrimen/otto/tree/ef014fd) -- github.com/robfig/cron/v3: [v3.0.1](https://github.com/robfig/cron/v3/tree/v3.0.1) -- github.com/russross/blackfriday/v2: [v2.0.1](https://github.com/russross/blackfriday/v2/tree/v2.0.1) -- github.com/ryanuber/columnize: [9b3edd6](https://github.com/ryanuber/columnize/tree/9b3edd6) -- github.com/sean-/seed: [e2103e2](https://github.com/sean-/seed/tree/e2103e2) -- github.com/shurcooL/sanitized_anchor_name: [v1.0.0](https://github.com/shurcooL/sanitized_anchor_name/tree/v1.0.0) -- github.com/spaolacci/murmur3: [f09979e](https://github.com/spaolacci/murmur3/tree/f09979e) -- github.com/stoewer/go-strcase: [v1.2.0](https://github.com/stoewer/go-strcase/tree/v1.2.0) -- github.com/subosito/gotenv: [v1.2.0](https://github.com/subosito/gotenv/tree/v1.2.0) -- github.com/willf/bitset: [v1.1.11](https://github.com/willf/bitset/tree/v1.1.11) -- github.com/xlab/treeprint: [a009c39](https://github.com/xlab/treeprint/tree/a009c39) -- go.etcd.io/etcd/api/v3: v3.5.0 -- go.etcd.io/etcd/client/pkg/v3: v3.5.0 -- go.etcd.io/etcd/client/v2: v2.305.0 -- go.etcd.io/etcd/client/v3: v3.5.0 -- go.etcd.io/etcd/pkg/v3: v3.5.0 -- go.etcd.io/etcd/raft/v3: v3.5.0 -- go.etcd.io/etcd/server/v3: v3.5.0 -- go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc: v0.20.0 -- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp: v0.20.0 -- go.opentelemetry.io/contrib: v0.20.0 -- go.opentelemetry.io/otel/exporters/otlp: v0.20.0 -- go.opentelemetry.io/otel/metric: v0.20.0 -- go.opentelemetry.io/otel/oteltest: v0.20.0 -- go.opentelemetry.io/otel/sdk/export/metric: v0.20.0 -- go.opentelemetry.io/otel/sdk/metric: v0.20.0 -- go.opentelemetry.io/otel/sdk: v0.20.0 -- go.opentelemetry.io/otel/trace: v0.20.0 -- go.opentelemetry.io/otel: v0.20.0 -- go.opentelemetry.io/proto/otlp: v0.7.0 -- go.starlark.net: 8dd3e2e -- go.uber.org/goleak: v1.1.10 -- golang.org/x/term: 6a3ed07 -- gopkg.in/ini.v1: v1.51.0 -- gopkg.in/sourcemap.v1: v1.0.5 -- gotest.tools/v3: v3.0.3 -- k8s.io/component-helpers: v0.22.0 -- k8s.io/controller-manager: v0.22.0 -- k8s.io/pod-security-admission: v0.22.0 -- sigs.k8s.io/kustomize/api: v0.8.11 -- sigs.k8s.io/kustomize/cmd/config: v0.9.13 -- sigs.k8s.io/kustomize/kustomize/v4: v4.2.0 -- sigs.k8s.io/kustomize/kyaml: v0.11.0 -- sigs.k8s.io/structured-merge-diff/v4: v4.1.2 - -### Changed -- dmitri.shuralyov.com/gpu/mtl: 666a987 → 28db891 -- github.com/Azure/azure-sdk-for-go: [v35.0.0+incompatible → v55.0.0+incompatible](https://github.com/Azure/azure-sdk-for-go/compare/v35.0.0...v55.0.0) -- github.com/Azure/go-ansiterm: [d6e3b33 → d185dfc](https://github.com/Azure/go-ansiterm/compare/d6e3b33...d185dfc) -- github.com/Azure/go-autorest/autorest/adal: [v0.5.0 → v0.9.13](https://github.com/Azure/go-autorest/autorest/adal/compare/v0.5.0...v0.9.13) -- github.com/Azure/go-autorest/autorest/date: [v0.1.0 → v0.3.0](https://github.com/Azure/go-autorest/autorest/date/compare/v0.1.0...v0.3.0) -- github.com/Azure/go-autorest/autorest/mocks: [v0.2.0 → v0.4.1](https://github.com/Azure/go-autorest/autorest/mocks/compare/v0.2.0...v0.4.1) -- github.com/Azure/go-autorest/autorest/to: [v0.2.0 → v0.4.0](https://github.com/Azure/go-autorest/autorest/to/compare/v0.2.0...v0.4.0) -- github.com/Azure/go-autorest/autorest: [v0.9.0 → v0.11.18](https://github.com/Azure/go-autorest/autorest/compare/v0.9.0...v0.11.18) -- github.com/Azure/go-autorest/logger: [v0.1.0 → v0.2.1](https://github.com/Azure/go-autorest/logger/compare/v0.1.0...v0.2.1) -- github.com/Azure/go-autorest/tracing: [v0.5.0 → v0.6.0](https://github.com/Azure/go-autorest/tracing/compare/v0.5.0...v0.6.0) -- github.com/Azure/go-autorest: [v11.1.2+incompatible → v14.2.0+incompatible](https://github.com/Azure/go-autorest/compare/v11.1.2...v14.2.0) -- github.com/GoogleCloudPlatform/k8s-cloud-provider: [27a4ced → 7901bc8](https://github.com/GoogleCloudPlatform/k8s-cloud-provider/compare/27a4ced...7901bc8) -- github.com/Microsoft/hcsshim: [672e52e → 5eafd15](https://github.com/Microsoft/hcsshim/compare/672e52e...5eafd15) -- github.com/NYTimes/gziphandler: [56545f4 → v1.1.1](https://github.com/NYTimes/gziphandler/compare/56545f4...v1.1.1) -- github.com/alecthomas/template: [a0175ee → fb15b89](https://github.com/alecthomas/template/compare/a0175ee...fb15b89) -- github.com/alecthomas/units: [2efee85 → f65c72e](https://github.com/alecthomas/units/compare/2efee85...f65c72e) -- github.com/auth0/go-jwt-middleware: [5493cab → v1.0.1](https://github.com/auth0/go-jwt-middleware/compare/5493cab...v1.0.1) -- github.com/aws/aws-sdk-go: [v1.28.2 → v1.38.49](https://github.com/aws/aws-sdk-go/compare/v1.28.2...v1.38.49) -- github.com/cilium/ebpf: [95b36a5 → v0.6.2](https://github.com/cilium/ebpf/compare/95b36a5...v0.6.2) -- github.com/cncf/udpa/go: [269d4d4 → 5459f2c](https://github.com/cncf/udpa/go/compare/269d4d4...5459f2c) -- github.com/cockroachdb/datadriven: [80d97fb → bf6692d](https://github.com/cockroachdb/datadriven/compare/80d97fb...bf6692d) -- github.com/container-storage-interface/spec: [v1.2.0 → v1.5.0](https://github.com/container-storage-interface/spec/compare/v1.2.0...v1.5.0) -- github.com/containerd/console: [84eeaae → v1.0.2](https://github.com/containerd/console/compare/84eeaae...v1.0.2) -- github.com/containerd/containerd: [v1.0.2 → v1.4.4](https://github.com/containerd/containerd/compare/v1.0.2...v1.4.4) -- github.com/containerd/typeurl: [2a93cfd → v1.0.1](https://github.com/containerd/typeurl/compare/2a93cfd...v1.0.1) -- github.com/containernetworking/cni: [v0.7.1 → v0.8.1](https://github.com/containernetworking/cni/compare/v0.7.1...v0.8.1) -- github.com/coredns/corefile-migration: [v1.0.6 → v1.0.12](https://github.com/coredns/corefile-migration/compare/v1.0.6...v1.0.12) -- github.com/coreos/etcd: [v3.3.10+incompatible → v3.3.13+incompatible](https://github.com/coreos/etcd/compare/v3.3.10...v3.3.13) -- github.com/coreos/pkg: [97fdf19 → 399ea9e](https://github.com/coreos/pkg/compare/97fdf19...399ea9e) -- github.com/creack/pty: [v1.1.7 → v1.1.11](https://github.com/creack/pty/compare/v1.1.7...v1.1.11) -- github.com/docker/docker: [71cd53e → v20.10.2+incompatible](https://github.com/docker/docker/compare/71cd53e...v20.10.2) -- github.com/envoyproxy/go-control-plane: [v0.9.4 → 668b12f](https://github.com/envoyproxy/go-control-plane/compare/v0.9.4...668b12f) -- github.com/evanphx/json-patch: [v4.5.0+incompatible → v4.11.0+incompatible](https://github.com/evanphx/json-patch/compare/v4.5.0...v4.11.0) -- github.com/fsnotify/fsnotify: [v1.4.7 → v1.4.9](https://github.com/fsnotify/fsnotify/compare/v1.4.7...v1.4.9) -- github.com/go-kit/kit: [v0.8.0 → v0.9.0](https://github.com/go-kit/kit/compare/v0.8.0...v0.9.0) -- github.com/go-logfmt/logfmt: [v0.3.0 → v0.5.0](https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.5.0) -- github.com/go-logr/logr: [v0.2.0 → v0.4.0](https://github.com/go-logr/logr/compare/v0.2.0...v0.4.0) -- github.com/go-openapi/jsonpointer: [v0.19.3 → v0.19.5](https://github.com/go-openapi/jsonpointer/compare/v0.19.3...v0.19.5) -- github.com/go-openapi/jsonreference: [v0.19.3 → v0.19.5](https://github.com/go-openapi/jsonreference/compare/v0.19.3...v0.19.5) -- github.com/go-openapi/swag: [v0.19.5 → v0.19.14](https://github.com/go-openapi/swag/compare/v0.19.5...v0.19.14) -- github.com/gogo/protobuf: [v1.3.1 → v1.3.2](https://github.com/gogo/protobuf/compare/v1.3.1...v1.3.2) -- github.com/golang/groupcache: [8c9f03a → 41bb18b](https://github.com/golang/groupcache/compare/8c9f03a...41bb18b) -- github.com/golang/protobuf: [v1.4.2 → v1.5.2](https://github.com/golang/protobuf/compare/v1.4.2...v1.5.2) -- github.com/google/btree: [v1.0.0 → v1.0.1](https://github.com/google/btree/compare/v1.0.0...v1.0.1) -- github.com/google/cadvisor: [v0.35.0 → v0.39.2](https://github.com/google/cadvisor/compare/v0.35.0...v0.39.2) -- github.com/google/go-cmp: [v0.5.2 → v0.5.5](https://github.com/google/go-cmp/compare/v0.5.2...v0.5.5) -- github.com/google/uuid: [v1.1.1 → v1.1.2](https://github.com/google/uuid/compare/v1.1.1...v1.1.2) -- github.com/googleapis/gnostic: [v0.3.1 → v0.5.5](https://github.com/googleapis/gnostic/compare/v0.3.1...v0.5.5) -- github.com/gopherjs/gopherjs: [0766667 → fce0ec3](https://github.com/gopherjs/gopherjs/compare/0766667...fce0ec3) -- github.com/gorilla/mux: [v1.7.0 → v1.8.0](https://github.com/gorilla/mux/compare/v1.7.0...v1.8.0) -- github.com/gorilla/websocket: [v1.4.0 → v1.4.2](https://github.com/gorilla/websocket/compare/v1.4.0...v1.4.2) -- github.com/grpc-ecosystem/go-grpc-middleware: [f849b54 → v1.3.0](https://github.com/grpc-ecosystem/go-grpc-middleware/compare/f849b54...v1.3.0) -- github.com/grpc-ecosystem/grpc-gateway: [v1.9.5 → v1.16.0](https://github.com/grpc-ecosystem/grpc-gateway/compare/v1.9.5...v1.16.0) -- github.com/heketi/heketi: [c2e2a4a → v10.3.0+incompatible](https://github.com/heketi/heketi/compare/c2e2a4a...v10.3.0) -- github.com/jmespath/go-jmespath: [c2b33e8 → v0.4.0](https://github.com/jmespath/go-jmespath/compare/c2b33e8...v0.4.0) -- github.com/jonboulle/clockwork: [v0.1.0 → v0.2.2](https://github.com/jonboulle/clockwork/compare/v0.1.0...v0.2.2) -- github.com/json-iterator/go: [v1.1.8 → v1.1.11](https://github.com/json-iterator/go/compare/v1.1.8...v1.1.11) -- github.com/julienschmidt/httprouter: [v1.2.0 → v1.3.0](https://github.com/julienschmidt/httprouter/compare/v1.2.0...v1.3.0) -- github.com/karrick/godirwalk: [v1.7.5 → v1.16.1](https://github.com/karrick/godirwalk/compare/v1.7.5...v1.16.1) -- github.com/kisielk/errcheck: [v1.2.0 → v1.5.0](https://github.com/kisielk/errcheck/compare/v1.2.0...v1.5.0) -- github.com/konsorten/go-windows-terminal-sequences: [v1.0.2 → v1.0.3](https://github.com/konsorten/go-windows-terminal-sequences/compare/v1.0.2...v1.0.3) -- github.com/kr/pretty: [v0.2.0 → v0.2.1](https://github.com/kr/pretty/compare/v0.2.0...v0.2.1) -- github.com/kr/text: [v0.1.0 → v0.2.0](https://github.com/kr/text/compare/v0.1.0...v0.2.0) -- github.com/mailru/easyjson: [v0.7.0 → v0.7.6](https://github.com/mailru/easyjson/compare/v0.7.0...v0.7.6) -- github.com/mattn/go-isatty: [v0.0.9 → v0.0.4](https://github.com/mattn/go-isatty/compare/v0.0.9...v0.0.4) -- github.com/mattn/go-runewidth: [v0.0.2 → v0.0.7](https://github.com/mattn/go-runewidth/compare/v0.0.2...v0.0.7) -- github.com/matttproud/golang_protobuf_extensions: [v1.0.1 → c182aff](https://github.com/matttproud/golang_protobuf_extensions/compare/v1.0.1...c182aff) -- github.com/miekg/dns: [v1.1.4 → v1.0.14](https://github.com/miekg/dns/compare/v1.1.4...v1.0.14) -- github.com/mistifyio/go-zfs: [v2.1.1+incompatible → f784269](https://github.com/mistifyio/go-zfs/compare/v2.1.1...f784269) -- github.com/mrunalp/fileutils: [7d4729f → v0.5.0](https://github.com/mrunalp/fileutils/compare/7d4729f...v0.5.0) -- github.com/mwitkow/go-conntrack: [cc309e4 → 2f06839](https://github.com/mwitkow/go-conntrack/compare/cc309e4...2f06839) -- github.com/olekukonko/tablewriter: [a0225b3 → v0.0.4](https://github.com/olekukonko/tablewriter/compare/a0225b3...v0.0.4) -- github.com/onsi/ginkgo: [v1.11.0 → v1.14.2](https://github.com/onsi/ginkgo/compare/v1.11.0...v1.14.2) -- github.com/onsi/gomega: [v1.7.1 → v1.10.4](https://github.com/onsi/gomega/compare/v1.7.1...v1.10.4) -- github.com/opencontainers/go-digest: [v1.0.0-rc1 → v1.0.0](https://github.com/opencontainers/go-digest/compare/v1.0.0-rc1...v1.0.0) -- github.com/opencontainers/runc: [v1.0.0-rc10 → v1.0.1](https://github.com/opencontainers/runc/compare/v1.0.0-rc10...v1.0.1) -- github.com/opencontainers/runtime-spec: [v1.0.0 → 1c3f411](https://github.com/opencontainers/runtime-spec/compare/v1.0.0...1c3f411) -- github.com/opencontainers/selinux: [5215b18 → v1.8.2](https://github.com/opencontainers/selinux/compare/5215b18...v1.8.2) -- github.com/prometheus/client_golang: [v1.0.0 → v1.11.0](https://github.com/prometheus/client_golang/compare/v1.0.0...v1.11.0) -- github.com/prometheus/common: [v0.4.1 → v0.26.0](https://github.com/prometheus/common/compare/v0.4.1...v0.26.0) -- github.com/prometheus/procfs: [v0.0.8 → v0.6.0](https://github.com/prometheus/procfs/compare/v0.0.8...v0.6.0) -- github.com/quobyte/api: [v0.1.2 → v0.1.8](https://github.com/quobyte/api/compare/v0.1.2...v0.1.8) -- github.com/rogpeppe/fastuuid: [6724a57 → v1.2.0](https://github.com/rogpeppe/fastuuid/compare/6724a57...v1.2.0) -- github.com/rubiojr/go-vhd: [0bfd3b3 → 02e2102](https://github.com/rubiojr/go-vhd/compare/0bfd3b3...02e2102) -- github.com/satori/go.uuid: [v1.2.0 → 0aa62d5](https://github.com/satori/go.uuid/compare/v1.2.0...0aa62d5) -- github.com/sergi/go-diff: [v1.0.0 → v1.1.0](https://github.com/sergi/go-diff/compare/v1.0.0...v1.1.0) -- github.com/sirupsen/logrus: [v1.4.2 → v1.8.1](https://github.com/sirupsen/logrus/compare/v1.4.2...v1.8.1) -- github.com/smartystreets/assertions: [b2de0cb → v1.1.0](https://github.com/smartystreets/assertions/compare/b2de0cb...v1.1.0) -- github.com/soheilhy/cmux: [v0.1.4 → v0.1.5](https://github.com/soheilhy/cmux/compare/v0.1.4...v0.1.5) -- github.com/spf13/cobra: [v0.0.5 → v1.1.3](https://github.com/spf13/cobra/compare/v0.0.5...v1.1.3) -- github.com/spf13/jwalterweatherman: [v1.1.0 → v1.0.0](https://github.com/spf13/jwalterweatherman/compare/v1.1.0...v1.0.0) -- github.com/spf13/viper: [v1.3.2 → v1.7.0](https://github.com/spf13/viper/compare/v1.3.2...v1.7.0) -- github.com/storageos/go-api: [343b3ef → v2.2.0+incompatible](https://github.com/storageos/go-api/compare/343b3ef...v2.2.0) -- github.com/stretchr/testify: [v1.6.1 → v1.7.0](https://github.com/stretchr/testify/compare/v1.6.1...v1.7.0) -- github.com/syndtr/gocapability: [d983527 → 42c35b4](https://github.com/syndtr/gocapability/compare/d983527...42c35b4) -- github.com/tmc/grpc-websocket-proxy: [89b8d40 → e5319fd](https://github.com/tmc/grpc-websocket-proxy/compare/89b8d40...e5319fd) -- github.com/ugorji/go: [v1.1.1 → v1.1.4](https://github.com/ugorji/go/compare/v1.1.1...v1.1.4) -- github.com/urfave/cli: [v1.20.0 → v1.22.2](https://github.com/urfave/cli/compare/v1.20.0...v1.22.2) -- github.com/vishvananda/netlink: [v1.0.0 → v1.1.0](https://github.com/vishvananda/netlink/compare/v1.0.0...v1.1.0) -- github.com/vishvananda/netns: [be1fbed → db3c7e5](https://github.com/vishvananda/netns/compare/be1fbed...db3c7e5) -- github.com/yuin/goldmark: [v1.2.1 → v1.3.5](https://github.com/yuin/goldmark/compare/v1.2.1...v1.3.5) -- go.etcd.io/bbolt: v1.3.3 → v1.3.6 -- go.etcd.io/etcd: 3cf2f69 → 83304cf -- go.uber.org/atomic: v1.3.2 → v1.7.0 -- go.uber.org/multierr: v1.1.0 → v1.6.0 -- go.uber.org/zap: v1.10.0 → v1.17.0 -- golang.org/x/crypto: 75b2880 → 5ea612d -- golang.org/x/exp: 6cc2880 → 85be41e -- golang.org/x/lint: 738671d → 6edffad -- golang.org/x/mobile: d2bd2a2 → e6ae53a -- golang.org/x/mod: v0.3.0 → v0.4.2 -- golang.org/x/net: f585440 → 37e1c6a -- golang.org/x/sync: 6e8e738 → 036812b -- golang.org/x/sys: fdedc70 → 59db8d7 -- golang.org/x/text: v0.3.3 → v0.3.6 -- golang.org/x/time: 555d28b → 1f47c86 -- golang.org/x/tools: 39188db → v0.1.2 -- google.golang.org/genproto: 0bd0a95 → f16073e -- google.golang.org/grpc: v1.31.1 → v1.38.0 -- google.golang.org/protobuf: v1.25.0 → v1.26.0 -- gopkg.in/check.v1: 41f04d3 → 8fa4692 -- gopkg.in/yaml.v2: v2.2.8 → v2.4.0 -- gopkg.in/yaml.v3: 9f266ea → 496545a -- k8s.io/api: v0.18.0 → v0.22.0 -- k8s.io/apiextensions-apiserver: v0.18.0 → v0.22.0 -- k8s.io/apimachinery: v0.18.0 → v0.22.0 -- k8s.io/apiserver: v0.18.0 → v0.22.0 -- k8s.io/cli-runtime: v0.18.0 → v0.22.0 -- k8s.io/client-go: v0.18.0 → v0.22.0 -- k8s.io/cloud-provider: v0.18.0 → v0.22.0 -- k8s.io/cluster-bootstrap: v0.18.0 → v0.22.0 -- k8s.io/code-generator: v0.18.0 → v0.22.0 -- k8s.io/component-base: v0.18.0 → v0.22.0 -- k8s.io/cri-api: v0.18.0 → v0.22.0 -- k8s.io/csi-translation-lib: v0.18.0 → v0.22.0 -- k8s.io/gengo: 36b2048 → b6c5ce2 -- k8s.io/klog/v2: v2.4.0 → v2.9.0 -- k8s.io/kube-aggregator: v0.18.0 → v0.22.0 -- k8s.io/kube-controller-manager: v0.18.0 → v0.22.0 -- k8s.io/kube-openapi: bf4fb3b → 9528897 -- k8s.io/kube-proxy: v0.18.0 → v0.22.0 -- k8s.io/kube-scheduler: v0.18.0 → v0.22.0 -- k8s.io/kubectl: v0.18.0 → v0.22.0 -- k8s.io/kubelet: v0.18.0 → v0.22.0 -- k8s.io/kubernetes: v1.18.0 → v1.22.0 -- k8s.io/legacy-cloud-providers: v0.18.0 → v0.22.0 -- k8s.io/metrics: v0.18.0 → v0.22.0 -- k8s.io/mount-utils: v0.20.6 → v0.22.0 -- k8s.io/sample-apiserver: v0.18.0 → v0.22.0 -- k8s.io/system-validators: v1.0.4 → v1.5.0 -- k8s.io/utils: 67b214c → 4b05e18 -- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.7 → v0.0.22 - -### Removed -- github.com/OpenPeeDeeP/depguard: [v1.0.1](https://github.com/OpenPeeDeeP/depguard/tree/v1.0.1) -- github.com/Rican7/retry: [v0.1.0](https://github.com/Rican7/retry/tree/v0.1.0) -- github.com/StackExchange/wmi: [5d04971](https://github.com/StackExchange/wmi/tree/5d04971) -- github.com/agnivade/levenshtein: [v1.0.1](https://github.com/agnivade/levenshtein/tree/v1.0.1) -- github.com/andreyvit/diff: [c7f18ee](https://github.com/andreyvit/diff/tree/c7f18ee) -- github.com/anmitsu/go-shlex: [648efa6](https://github.com/anmitsu/go-shlex/tree/648efa6) -- github.com/bazelbuild/bazel-gazelle: [70208cb](https://github.com/bazelbuild/bazel-gazelle/tree/70208cb) -- github.com/bazelbuild/rules_go: [6dae44d](https://github.com/bazelbuild/rules_go/tree/6dae44d) -- github.com/bifurcation/mint: [93c51c6](https://github.com/bifurcation/mint/tree/93c51c6) -- github.com/bradfitz/go-smtpd: [deb6d62](https://github.com/bradfitz/go-smtpd/tree/deb6d62) -- github.com/caddyserver/caddy: [v1.0.3](https://github.com/caddyserver/caddy/tree/v1.0.3) -- github.com/cenkalti/backoff: [v2.1.1+incompatible](https://github.com/cenkalti/backoff/tree/v2.1.1) -- github.com/cespare/prettybench: [03b8cfe](https://github.com/cespare/prettybench/tree/03b8cfe) -- github.com/checkpoint-restore/go-criu: [17b0214](https://github.com/checkpoint-restore/go-criu/tree/17b0214) -- github.com/cheekybits/genny: [9127e81](https://github.com/cheekybits/genny/tree/9127e81) -- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) -- github.com/docker/libnetwork: [c8a5fca](https://github.com/docker/libnetwork/tree/c8a5fca) -- github.com/docker/spdystream: [449fdfc](https://github.com/docker/spdystream/tree/449fdfc) -- github.com/gliderlabs/ssh: [v0.1.1](https://github.com/gliderlabs/ssh/tree/v0.1.1) -- github.com/globalsign/mgo: [eeefdec](https://github.com/globalsign/mgo/tree/eeefdec) -- github.com/go-acme/lego: [v2.5.0+incompatible](https://github.com/go-acme/lego/tree/v2.5.0) -- github.com/go-bindata/go-bindata: [v3.1.1+incompatible](https://github.com/go-bindata/go-bindata/tree/v3.1.1) -- github.com/go-critic/go-critic: [1df3008](https://github.com/go-critic/go-critic/tree/1df3008) -- github.com/go-lintpack/lintpack: [v0.5.2](https://github.com/go-lintpack/lintpack/tree/v0.5.2) -- github.com/go-ole/go-ole: [v1.2.1](https://github.com/go-ole/go-ole/tree/v1.2.1) -- github.com/go-openapi/analysis: [v0.19.5](https://github.com/go-openapi/analysis/tree/v0.19.5) -- github.com/go-openapi/errors: [v0.19.2](https://github.com/go-openapi/errors/tree/v0.19.2) -- github.com/go-openapi/loads: [v0.19.4](https://github.com/go-openapi/loads/tree/v0.19.4) -- github.com/go-openapi/runtime: [v0.19.4](https://github.com/go-openapi/runtime/tree/v0.19.4) -- github.com/go-openapi/strfmt: [v0.19.3](https://github.com/go-openapi/strfmt/tree/v0.19.3) -- github.com/go-openapi/validate: [v0.19.5](https://github.com/go-openapi/validate/tree/v0.19.5) -- github.com/go-toolsmith/astcast: [v1.0.0](https://github.com/go-toolsmith/astcast/tree/v1.0.0) -- github.com/go-toolsmith/astcopy: [v1.0.0](https://github.com/go-toolsmith/astcopy/tree/v1.0.0) -- github.com/go-toolsmith/astequal: [v1.0.0](https://github.com/go-toolsmith/astequal/tree/v1.0.0) -- github.com/go-toolsmith/astfmt: [v1.0.0](https://github.com/go-toolsmith/astfmt/tree/v1.0.0) -- github.com/go-toolsmith/astinfo: [9809ff7](https://github.com/go-toolsmith/astinfo/tree/9809ff7) -- github.com/go-toolsmith/astp: [v1.0.0](https://github.com/go-toolsmith/astp/tree/v1.0.0) -- github.com/go-toolsmith/pkgload: [v1.0.0](https://github.com/go-toolsmith/pkgload/tree/v1.0.0) -- github.com/go-toolsmith/strparse: [v1.0.0](https://github.com/go-toolsmith/strparse/tree/v1.0.0) -- github.com/go-toolsmith/typep: [v1.0.0](https://github.com/go-toolsmith/typep/tree/v1.0.0) -- github.com/gobwas/glob: [v0.2.3](https://github.com/gobwas/glob/tree/v0.2.3) -- github.com/godbus/dbus: [2ff6f7f](https://github.com/godbus/dbus/tree/2ff6f7f) -- github.com/golangci/check: [cfe4005](https://github.com/golangci/check/tree/cfe4005) -- github.com/golangci/dupl: [3e9179a](https://github.com/golangci/dupl/tree/3e9179a) -- github.com/golangci/errcheck: [ef45e06](https://github.com/golangci/errcheck/tree/ef45e06) -- github.com/golangci/go-misc: [927a3d8](https://github.com/golangci/go-misc/tree/927a3d8) -- github.com/golangci/go-tools: [e32c541](https://github.com/golangci/go-tools/tree/e32c541) -- github.com/golangci/goconst: [041c5f2](https://github.com/golangci/goconst/tree/041c5f2) -- github.com/golangci/gocyclo: [2becd97](https://github.com/golangci/gocyclo/tree/2becd97) -- github.com/golangci/gofmt: [0b8337e](https://github.com/golangci/gofmt/tree/0b8337e) -- github.com/golangci/golangci-lint: [v1.18.0](https://github.com/golangci/golangci-lint/tree/v1.18.0) -- github.com/golangci/gosec: [66fb7fc](https://github.com/golangci/gosec/tree/66fb7fc) -- github.com/golangci/ineffassign: [42439a7](https://github.com/golangci/ineffassign/tree/42439a7) -- github.com/golangci/lint-1: [ee948d0](https://github.com/golangci/lint-1/tree/ee948d0) -- github.com/golangci/maligned: [b1d8939](https://github.com/golangci/maligned/tree/b1d8939) -- github.com/golangci/misspell: [950f5d1](https://github.com/golangci/misspell/tree/950f5d1) -- github.com/golangci/prealloc: [215b22d](https://github.com/golangci/prealloc/tree/215b22d) -- github.com/golangci/revgrep: [d9c87f5](https://github.com/golangci/revgrep/tree/d9c87f5) -- github.com/golangci/unconvert: [28b1c44](https://github.com/golangci/unconvert/tree/28b1c44) -- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) -- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) -- github.com/gostaticanalysis/analysisutil: [v0.0.3](https://github.com/gostaticanalysis/analysisutil/tree/v0.0.3) -- github.com/jellevandenhooff/dkim: [f50fe3d](https://github.com/jellevandenhooff/dkim/tree/f50fe3d) -- github.com/jimstudt/http-authentication: [3eca13d](https://github.com/jimstudt/http-authentication/tree/3eca13d) -- github.com/kubernetes-csi/csi-test/v3: [v3.0.0](https://github.com/kubernetes-csi/csi-test/v3/tree/v3.0.0) -- github.com/kylelemons/godebug: [d65d576](https://github.com/kylelemons/godebug/tree/d65d576) -- github.com/logrusorgru/aurora: [a7b3b31](https://github.com/logrusorgru/aurora/tree/a7b3b31) -- github.com/lucas-clemente/aes12: [cd47fb3](https://github.com/lucas-clemente/aes12/tree/cd47fb3) -- github.com/lucas-clemente/quic-clients: [v0.1.0](https://github.com/lucas-clemente/quic-clients/tree/v0.1.0) -- github.com/lucas-clemente/quic-go-certificates: [d2f8652](https://github.com/lucas-clemente/quic-go-certificates/tree/d2f8652) -- github.com/lucas-clemente/quic-go: [v0.10.2](https://github.com/lucas-clemente/quic-go/tree/v0.10.2) -- github.com/marten-seemann/qtls: [v0.2.3](https://github.com/marten-seemann/qtls/tree/v0.2.3) -- github.com/mattn/go-shellwords: [v1.0.5](https://github.com/mattn/go-shellwords/tree/v1.0.5) -- github.com/mattn/goveralls: [v0.0.2](https://github.com/mattn/goveralls/tree/v0.0.2) -- github.com/mesos/mesos-go: [v0.0.9](https://github.com/mesos/mesos-go/tree/v0.0.9) -- github.com/mholt/certmagic: [6a42ef9](https://github.com/mholt/certmagic/tree/6a42ef9) -- github.com/mitchellh/go-ps: [4fdf99a](https://github.com/mitchellh/go-ps/tree/4fdf99a) -- github.com/mozilla/tls-observatory: [8791a20](https://github.com/mozilla/tls-observatory/tree/8791a20) -- github.com/naoina/go-stringutil: [v0.1.0](https://github.com/naoina/go-stringutil/tree/v0.1.0) -- github.com/naoina/toml: [v0.1.1](https://github.com/naoina/toml/tree/v0.1.1) -- github.com/nbutton23/zxcvbn-go: [eafdab6](https://github.com/nbutton23/zxcvbn-go/tree/eafdab6) -- github.com/pborman/uuid: [v1.2.0](https://github.com/pborman/uuid/tree/v1.2.0) -- github.com/pquerna/ffjson: [af8b230](https://github.com/pquerna/ffjson/tree/af8b230) -- github.com/quasilyte/go-consistent: [c6f3937](https://github.com/quasilyte/go-consistent/tree/c6f3937) -- github.com/robfig/cron: [v1.1.0](https://github.com/robfig/cron/tree/v1.1.0) -- github.com/ryanuber/go-glob: [256dc44](https://github.com/ryanuber/go-glob/tree/256dc44) -- github.com/shirou/gopsutil: [c95755e](https://github.com/shirou/gopsutil/tree/c95755e) -- github.com/shirou/w32: [bb4de01](https://github.com/shirou/w32/tree/bb4de01) -- github.com/shurcooL/go-goon: [37c2f52](https://github.com/shurcooL/go-goon/tree/37c2f52) -- github.com/sourcegraph/go-diff: [v0.5.1](https://github.com/sourcegraph/go-diff/tree/v0.5.1) -- github.com/tarm/serial: [98f6abe](https://github.com/tarm/serial/tree/98f6abe) -- github.com/thecodeteam/goscaleio: [v0.1.0](https://github.com/thecodeteam/goscaleio/tree/v0.1.0) -- github.com/tidwall/pretty: [v1.0.0](https://github.com/tidwall/pretty/tree/v1.0.0) -- github.com/timakin/bodyclose: [87058b9](https://github.com/timakin/bodyclose/tree/87058b9) -- github.com/ultraware/funlen: [v0.0.2](https://github.com/ultraware/funlen/tree/v0.0.2) -- github.com/valyala/bytebufferpool: [v1.0.0](https://github.com/valyala/bytebufferpool/tree/v1.0.0) -- github.com/valyala/fasthttp: [v1.2.0](https://github.com/valyala/fasthttp/tree/v1.2.0) -- github.com/valyala/quicktemplate: [v1.1.1](https://github.com/valyala/quicktemplate/tree/v1.1.1) -- github.com/valyala/tcplisten: [ceec8f9](https://github.com/valyala/tcplisten/tree/ceec8f9) -- github.com/vektah/gqlparser: [v1.1.2](https://github.com/vektah/gqlparser/tree/v1.1.2) -- go.mongodb.org/mongo-driver: v1.1.2 -- go4.org: 417644f -- golang.org/x/build: 2835ba2 -- golang.org/x/perf: 6e6d33e -- gopkg.in/mcuadros/go-syslog.v2: v2.2.1 -- gotest.tools/gotestsum: v0.3.5 -- grpc.go4.org: 11d0a25 -- k8s.io/heapster: v1.2.0-beta.1 -- k8s.io/repo-infra: v0.0.1-alpha.1 -- mvdan.cc/interfacer: c200402 -- mvdan.cc/lint: adc824a -- mvdan.cc/unparam: fbb5962 -- sigs.k8s.io/kustomize: v2.0.3+incompatible -- sigs.k8s.io/structured-merge-diff/v3: v3.0.0 -- sourcegraph.com/sqs/pbtypes: d3ebe8f diff --git a/CHANGELOG/CHANGELOG-1.5.md b/CHANGELOG/CHANGELOG-1.5.md deleted file mode 100644 index f273ee4f..00000000 --- a/CHANGELOG/CHANGELOG-1.5.md +++ /dev/null @@ -1,26 +0,0 @@ -# v1.5.0 - Changelog since v1.4.1 - -## Changes by Kind - -### Feature - -- Add parameters to VolumeSnapshotClass for disk image config. ([#926](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/926), [@luohao](https://github.com/luohao)) - -### Bug or Regression - -- Fix ControllerUnpublish backoff ([#953](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/953), [@saikat-royc](https://github.com/saikat-royc)) - -### Documentation - -- Adds documentation for how to use the PD CSI Driver overlays for testing and deploying the driver. ([#932](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/932), [@amacaskill](https://github.com/amacaskill)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -- github.com/GoogleCloudPlatform/guest-configs: [a0dacef](https://github.com/GoogleCloudPlatform/guest-configs/tree/a0dacef) diff --git a/CHANGELOG/CHANGELOG-1.6.md b/CHANGELOG/CHANGELOG-1.6.md deleted file mode 100644 index a4fbee06..00000000 --- a/CHANGELOG/CHANGELOG-1.6.md +++ /dev/null @@ -1,29 +0,0 @@ -**Attention:** 1.6.0 is not a recommended version to use because of known issues where pods can get stuck (due to controller publish/unpublish failures) during cluster upgrades or during node reboot (as seen in [#987](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/987)). Users should upgrade directly to the 1.7 branch. - - -# v1.6.0 - Changelog since v1.5.1 - -## Changes by Kind - -### Feature - -- Allow to specify how frequently to poll for AttachDisk operation status, or any other global\regional\zonal operation. ([#956](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/956), [@sagor999](https://github.com/sagor999)) - -### Bug or Regression - -- Default to MAXPROCS=1 to improve memory usage on nodes with many CPUs. ([#969](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/969), [@mattcary](https://github.com/mattcary)) - -### Uncategorized - -- Lets users clone a regional disk from a zonal disk if one of the replica zones of the clone matches the zone of the source disk. ([#890](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/890), [@amacaskill](https://github.com/amacaskill)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -- github.com/prometheus/client_golang: [v1.11.0 → v1.11.1](https://github.com/prometheus/client_golang/compare/v1.11.0...v1.11.1) - -### Removed -_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.7.md b/CHANGELOG/CHANGELOG-1.7.md deleted file mode 100644 index 8792fa3c..00000000 --- a/CHANGELOG/CHANGELOG-1.7.md +++ /dev/null @@ -1,57 +0,0 @@ -# v1.7.3 - Changelog since v.1.7.2 - -- Update go builder to 1.18.4. Fixes several CVEs. (#1031, @mattcary) - -- Cherry pick #1028, Improve backoff to be per-node and disk to avoid missing disks from blocking all operations (#1036, @mattcary) - -# v1.7.2 - Changelog since v1.7.1 - -## Changes by Kind - -### Uncategorized - -- Enforce implicit pagination limit of 500 of the ListVolumesResponse#Entry field when ListVolumesRequest#max_entries is not set (#1011, @pwschuurman) - -## Dependencies - -_Nothing has changed._ - -# v1.7.1 - Changelog since v1.7.0 - -- Creates v1.7.1 upstream tag with changes from 1.7.0 release. - - Cloud builder was broken when 1.7.0 was cut, so v1.7.0 upstream tag was not - created. - -# v1.7.0 - Changelog since v1.5.1 - ->**Attention:** 1.6.0 is not a recommended version to use because of known issues where pods can get stuck (due to controller publish/unpublish failures) during cluster upgrades or during node reboot (as seen in [#987](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/987)). Users should upgrade directly to the 1.7 branch. - -## Changes by Kind - -### Feature - -- Allow to specify how frequently to poll for AttachDisk operation status, or any other global\regional\zonal operation. ([#956](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/956), [@sagor999](https://github.com/sagor999)) - -### Bug or Regression - -- Default to MAXPROCS=1 to improve memory usage on nodes with many CPUs. ([#969](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/969), [@mattcary](https://github.com/mattcary)) -- Simplify node backoff logic for controller publish/unpublish op ([#988](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/988), [@saikat-royc](https://github.com/saikat-royc)) - -### Other (Cleanup or Flake) - -- Remove PodSecurityPolicy from deployment for 1.25+ clusters. ([#989](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/989), [@mattcary](https://github.com/mattcary)) - -### Uncategorized - -- Lets users clone a regional disk from a zonal disk if one of the replica zones of the clone matches the zone of the source disk. ([#890](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/890), [@amacaskill](https://github.com/amacaskill)) - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -- github.com/prometheus/client_golang: [v1.11.0 → v1.11.1](https://github.com/prometheus/client_golang/compare/v1.11.0...v1.11.1) - -### Removed -_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.8.md b/CHANGELOG/CHANGELOG-1.8.md deleted file mode 100644 index fe566f00..00000000 --- a/CHANGELOG/CHANGELOG-1.8.md +++ /dev/null @@ -1,18 +0,0 @@ -# v1.8.0 - Changelog since v.1.7.3 - -## Changes by Kind - -### Feature - -- Add support for setting snapshot labels ([#1017](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1017), [@sagor999](https://github.com/sagor999)) -- Go builder updated from 1.18.4 to 1.19.1. ([#1048](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1048), [@kon-angelo](https://github.com/kon-angelo)) - -### Bug or Regression - -- Disable devices in node unstage prior to detaching. ([#1051](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1051), [@mattcary](https://github.com/mattcary)) -- Enforce implicit pagination limit of 500 of the ListVolumesResponse#Entry field when ListVolumesRequest#max_entries is not set ([#999](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/999), [@pwschuurman](https://github.com/pwschuurman)) -- Fixed issue where Regional disks are repeatedly queued for re-attaching and consuming api quota ([#1050](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1050), [@leiyiz](https://github.com/leiyiz)) - -### Uncategorized - -- Migrate from github.com/golang/protobuf to google.golang.org/protobuf ([#1027](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1027), [@leiyiz](https://github.com/leiyiz)) diff --git a/CHANGELOG/CHANGELOG.md b/CHANGELOG/CHANGELOG.md new file mode 100644 index 00000000..0dce137a --- /dev/null +++ b/CHANGELOG/CHANGELOG.md @@ -0,0 +1 @@ +See the [1.9 changelog in the master branch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/CHANGELOG/CHANGELOG-1.9.md). \ No newline at end of file diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md deleted file mode 100644 index 1d8fae91..00000000 --- a/CHANGELOG/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# CHANGELOGs - -- [CHANGELOG v1.8.0](./CHANGELOG-1.8.md) -- [CHANGELOG v1.7.0](./CHANGELOG-1.7.md) -- [CHANGELOG v1.6.0](./CHANGELOG-1.6.md) -- [CHANGELOG v1.5.0](./CHANGELOG-1.5.md) -- [CHANGELOG v1.4.0](./CHANGELOG-1.4.md) -- [CHANGELOG v1.3.0](./CHANGELOG-1.3.md) -- [CHANGELOG v1.2.0](./CHANGELOG-1.2.md) -- [CHANGELOG v1.1.0](./CHANGELOG-1.1.md) -- [CHANGELOG v1.0.0](./CHANGELOG-1.0.md) -- [CHANGELOG v0.7.0](./CHANGELOG-0.7.md) -- [CHANGELOG v0.6.0](./CHANGELOG-0.6.md) diff --git a/Dockerfile.Windows b/Dockerfile.Windows index 64c74f2d..091b61cd 100644 --- a/Dockerfile.Windows +++ b/Dockerfile.Windows @@ -13,7 +13,7 @@ # limitations under the License. ARG BASE_IMAGE -FROM --platform=$BUILDPLATFORM golang:1.17.8 AS builder +FROM --platform=$BUILDPLATFORM golang:1.19.6 AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/Dockerfile.debug b/Dockerfile.debug index de3431a5..249f62d9 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.19 as builder +FROM golang:1.19.6 as builder WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver ADD . . @@ -33,6 +33,7 @@ FROM k8s.gcr.io/build-image/debian-base-amd64:buster-v1.5.0 # Copy source code too to correlate the binary and the breakpoints WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver ADD . . +COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver COPY --from=builder /go/bin/dlv /go/bin/dlv diff --git a/Makefile b/Makefile index 96ac9c90..bf45a313 100644 --- a/Makefile +++ b/Makefile @@ -42,11 +42,10 @@ DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe DOCKER=DOCKER_BUILDKIT=1 docker BASE_IMAGE_LTSC2019=mcr.microsoft.com/windows/servercore:ltsc2019 -BASE_IMAGE_20H2=mcr.microsoft.com/windows/servercore:20H2 # Both arrays MUST be index aligned. -WINDOWS_IMAGE_TAGS=ltsc2019 20H2 -WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_20H2) +WINDOWS_IMAGE_TAGS=ltsc2019 +WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) GCFLAGS="" ifdef GCE_PD_CSI_DEBUG @@ -79,15 +78,8 @@ build-and-push-windows-container-ltsc2019: require-GCE_PD_CSI_STAGING_IMAGE --build-arg STAGINGVERSION=$(STAGINGVERSION) . $(DOCKER) push $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 -build-and-push-windows-container-20H2: require-GCE_PD_CSI_STAGING_IMAGE - $(DOCKER) build --file=Dockerfile.Windows --platform=windows \ - -t $(STAGINGIMAGE):$(STAGINGVERSION)_20H2 \ - --build-arg BASE_IMAGE=$(BASE_IMAGE_20H2) \ - --build-arg STAGINGVERSION=$(STAGINGVERSION) . - $(DOCKER) push $(STAGINGIMAGE):$(STAGINGVERSION)_20H2 - -build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019 build-and-push-windows-container-20H2 - $(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_20H2 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 +build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-windows-container-ltsc2019 + $(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh $(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION) diff --git a/OWNERS b/OWNERS index 0aa7ba68..043f6732 100644 --- a/OWNERS +++ b/OWNERS @@ -1,14 +1,16 @@ reviewers: - - davidz627 - - jingxu97 + - amacaskill + - leiyiz - mattcary - msau42 - - saad-ali - saikat-royc approvers: - - davidz627 - - jingxu97 + - amacaskill + - leiyiz - mattcary - msau42 - - saad-ali - saikat-royc +emeritus_reviewers: + - davidz627 + - jingxu97 + - saad-ali diff --git a/README.md b/README.md index ecd3f26b..0a5ea028 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,14 @@ This driver allows a Constellation cluster to use [GCP Persistent Disks](https:/ ### CreateVolume Parameters -| Parameter | Values | Default | Description | -|------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------| -| type | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks or Solid State Drive Persistent Disks | -| replication-type | `none` OR `regional-pd` | `none` | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks | -| disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. | -| labels | `key1=value1,key2=value2` | | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). | +| Parameter | Values | Default | Description | +|-----------------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------| +| type | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks or Solid State Drive Persistent Disks | +| replication-type | `none` OR `regional-pd` | `none` | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks | +| disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. | +| labels | `key1=value1,key2=value2` | | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). | +| provisioned-iops-on-create | string (int64 format). Values typically between 10,000 and 120,000 | | Indicates how many IOPS to provision for the disk. See the [Extreme persistent disk documentation](https://cloud.google.com/compute/docs/disks/extreme-persistent-disk) for details, including valid ranges for IOPS. | +| provisioned-throughput-on-create | string (int64 format). Values typically between 1 and 7,124 mb per second | | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation](TBD) for details, including valid ranges for throughput. | ### Topology diff --git a/charts/Chart.yaml b/charts/Chart.yaml index 547c4373..8c75338d 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -version: 1.1.0 -appVersion: "v1.1.2" +version: 1.2.0 +appVersion: "v1.2.0" description: GCP Compute Persistent Disk Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: gcp-compute-persistent-disk-csi-driver diff --git a/charts/templates/controller.yaml b/charts/templates/controller.yaml index 3b126510..efaaea59 100644 --- a/charts/templates/controller.yaml +++ b/charts/templates/controller.yaml @@ -45,7 +45,7 @@ spec: - "--feature-gates=Topology=true" - "--http-endpoint=:22011" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - - "--timeout=450s" + - "--timeout=250s" - "--extra-create-metadata" # - "--run-controller-service=false" # disable the controller service of the CSI driver # - "--run-node-service=false" # disable the node service of the CSI driver @@ -81,7 +81,7 @@ spec: - "--http-endpoint=:22012" - "--leader-election" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - - "--timeout=450s" + - "--timeout=250s" env: - name: PDCSI_NAMESPACE valueFrom: @@ -141,7 +141,7 @@ spec: - "--metrics-address=:22014" - "--leader-election" - "--leader-election-namespace=$(PDCSI_NAMESPACE)" - - "--timeout=450s" + - "--timeout=300s" env: - name: PDCSI_NAMESPACE valueFrom: diff --git a/charts/templates/storageclass_default.yaml b/charts/templates/storageclass_default.yaml index 1ff0af10..69bf2e97 100644 --- a/charts/templates/storageclass_default.yaml +++ b/charts/templates/storageclass_default.yaml @@ -6,7 +6,7 @@ metadata: storageclass.kubernetes.io/is-default-class: "true" name: encrypted-rwo parameters: - type: pd-standard + type: pd-balanced provisioner: gcp.csi.confidential.cloud allowVolumeExpansion: true reclaimPolicy: Delete diff --git a/charts/values.yaml b/charts/values.yaml index 477b1857..8083b193 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -1,28 +1,28 @@ image: csiProvisioner: - repo: k8s.gcr.io/sig-storage/csi-provisioner - tag: "v3.1.1" + repo: registry.k8s.io/sig-storage/csi-provisioner + tag: v3.4.0@sha256:e468dddcd275163a042ab297b2d8c2aca50d5e148d2d22f3b6ba119e2f31fa79 pullPolicy: IfNotPresent csiAttacher: - repo: k8s.gcr.io/sig-storage/csi-attacher - tag: "v3.5.0" + repo: registry.k8s.io/sig-storage/csi-attacher + tag: v4.2.0@sha256:34cf9b32736c6624fc9787fb149ea6e0fbeb45415707ac2f6440ac960f1116e6 pullPolicy: IfNotPresent csiResizer: - repo: k8s.gcr.io/sig-storage/csi-resizer - tag: "v1.5.0" + repo: registry.k8s.io/sig-storage/csi-resizer + tag: v1.7.0@sha256:3a7bdf5d105783d05d0962fa06ca53032b01694556e633f27366201c2881e01d pullPolicy: IfNotPresent csiSnapshotter: - repo: k8s.gcr.io/sig-storage/csi-snapshotter - tag: "v6.0.1" + repo: registry.k8s.io/sig-storage/csi-snapshotter + tag: v6.1.0@sha256:291334908ddf71a4661fd7f6d9d97274de8a5378a2b6fdfeb2ce73414a34f82f pullPolicy: IfNotPresent csiNodeRegistrar: - repo: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: "v2.5.1" + repo: registry.k8s.io/sig-storage/csi-node-driver-registrar + tag: v2.7.0@sha256:4a4cae5118c4404e35d66059346b7fa0835d7e6319ff45ed73f4bba335cf5183 pullPolicy: IfNotPresent gcepdDriver: repo: ghcr.io/edgelesssys/constellation/gcp-csi-driver # CSI driver version is independent of Constellation releases - tag: "v1.1.0" + tag: v1.2.0@sha256:84810096ab2244c77eeeab4c03445df69949e23f7f40d84bc434f122997a5692 pullPolicy: IfNotPresent csiController: diff --git a/cloudbuild.yaml b/cloudbuild.yaml index e875af03..d45b73c6 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -6,14 +6,14 @@ options: substitution_option: ALLOW_LOOSE steps: - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' + - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20221214-1b4dd4d69a' entrypoint: make env: - - GCE_PD_CSI_STAGING_IMAGE=gcr.io/${_STAGING_PROJECT}/gcp-compute-persistent-disk-csi-driver - - GCE_PD_CSI_STAGING_VERSION=${_PULL_BASE_REF} - # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx - # set the home to /root explicitly to if using docker buildx - - HOME=/root + - GCE_PD_CSI_STAGING_IMAGE=gcr.io/${_STAGING_PROJECT}/gcp-compute-persistent-disk-csi-driver + - GCE_PD_CSI_STAGING_VERSION=${_PULL_BASE_REF} + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root args: - build-and-push-multi-arch @@ -22,5 +22,5 @@ substitutions: _PULL_BASE_REF: 'master' tags: -- 'gcp-compute-persistent-disk-csi-driver' -- ${_PULL_BASE_REF} + - 'gcp-compute-persistent-disk-csi-driver' + - ${_PULL_BASE_REF} diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index ffea1f98..959fed8e 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -42,11 +42,12 @@ import ( "runtime" "time" - "k8s.io/klog" + "k8s.io/klog/v2" "github.com/edgelesssys/constellation/v2/csi/cryptmapper" cryptKms "github.com/edgelesssys/constellation/v2/csi/kms" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata" driver "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver" @@ -58,12 +59,16 @@ var ( constellationAddr = flag.String("kms-addr", "kms.kube-system:9000", "Address of the Constellation Coordinator's VPN API. Used to request keys (default: kms.kube-system:9000") cloudConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config") endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint") + computeEndpoint = flag.String("compute-endpoint", "", "If set, used as the endpoint for the GCE API.") runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)") runNodeService = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)") httpEndpoint = flag.String("http-endpoint", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.") metricsPath = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.") + grpcLogCharCap = flag.Int("grpc-log-char-cap", 10000, "The maximum amount of characters logged for every grpc responses") - extraVolumeLabelsStr = flag.String("extra-labels", "", "Extra labels to attach to each PD created. It is a comma separated list of key value pairs like '=,='. See https://cloud.google.com/compute/docs/labeling-resources for details") + errorBackoffInitialDurationMs = flag.Int("backoff-initial-duration-ms", 200, "The amount of ms for the initial duration of the backoff condition for controller publish/unpublish CSI operations. Default is 200.") + errorBackoffMaxDurationMs = flag.Int("backoff-max-duration-ms", 300000, "The amount of ms for the max duration of the backoff condition for controller publish/unpublish CSI operations. Default is 300000 (5m).") + extraVolumeLabelsStr = flag.String("extra-labels", "", "Extra labels to attach to each PD created. It is a comma separated list of key value pairs like '=,='. See https://cloud.google.com/compute/docs/labeling-resources for details") attachDiskBackoffDuration = flag.Duration("attach-disk-backoff-duration", 5*time.Second, "Duration for attachDisk backoff") attachDiskBackoffFactor = flag.Float64("attach-disk-backoff-factor", 0.0, "Factor for attachDisk backoff") @@ -124,7 +129,7 @@ func handle() { } extraVolumeLabels, err := common.ConvertLabelsStringToMap(*extraVolumeLabelsStr) if err != nil { - klog.Fatalf("Bad extra volume labels: %v", err) + klog.Fatalf("Bad extra volume labels: %v", err.Error()) } gceDriver := driver.GetGCEDriver() @@ -139,11 +144,13 @@ func handle() { //Initialize requirements for the controller service var controllerServer *driver.GCEControllerServer if *runControllerService { - cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath) + cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint) if err != nil { - klog.Fatalf("Failed to get cloud provider: %v", err) + klog.Fatalf("Failed to get cloud provider: %v", err.Error()) } - controllerServer = driver.NewControllerServer(gceDriver, cloudProvider) + initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond + maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond + controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration) } else if *cloudConfigFilePath != "" { klog.Warningf("controller service is disabled but cloud config given - it has no effect") } @@ -153,13 +160,13 @@ func handle() { if *runNodeService { mounter, err := mountmanager.NewSafeMounter() if err != nil { - klog.Fatalf("Failed to get safe mounter: %v", err) + klog.Fatalf("Failed to get safe mounter: %v", err.Error()) } - deviceUtils := mountmanager.NewDeviceUtils() + deviceUtils := deviceutils.NewDeviceUtils() statter := mountmanager.NewStatter(mounter) meta, err := metadataservice.NewMetadataService() if err != nil { - klog.Fatalf("Failed to set up metadata service: %v", err) + klog.Fatalf("Failed to set up metadata service: %v", err.Error()) } // [Edgeless] set up Constellation key management @@ -173,7 +180,7 @@ func handle() { err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, identityServer, controllerServer, nodeServer) if err != nil { - klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err) + klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) } gce.AttachDiskBackoff.Duration = *attachDiskBackoffDuration @@ -188,5 +195,5 @@ func handle() { gce.WaitForOpBackoff.Steps = *waitForOpBackoffSteps gce.WaitForOpBackoff.Cap = *waitForOpBackoffCap - gceDriver.Run(*endpoint) + gceDriver.Run(*endpoint, *grpcLogCharCap) } diff --git a/creds/cloud-sa.json b/creds/cloud-sa.json new file mode 100644 index 00000000..e69de29b diff --git a/deploy/common.sh b/deploy/common.sh index faab7899..ad170564 100644 --- a/deploy/common.sh +++ b/deploy/common.sh @@ -19,7 +19,7 @@ function ensure_var(){ function get_needed_roles() { - echo "roles/compute.storageAdmin roles/iam.serviceAccountUser projects/${PROJECT}/roles/gcp_compute_persistent_disk_csi_driver_custom_role" + echo "roles/editor roles/compute.storageAdmin roles/iam.serviceAccountUser projects/${PROJECT}/roles/gcp_compute_persistent_disk_csi_driver_custom_role" } # Installs kustomize in ${PKGDIR}/bin diff --git a/deploy/kubernetes/base/controller/kustomization.yaml b/deploy/kubernetes/base/controller/kustomization.yaml index 7d1c5065..77821685 100644 --- a/deploy/kubernetes/base/controller/kustomization.yaml +++ b/deploy/kubernetes/base/controller/kustomization.yaml @@ -6,3 +6,4 @@ resources: - cluster_setup.yaml - controller.yaml - v1_csidriver.yaml + diff --git a/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml b/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml index 0c5a10a2..8a985093 100644 --- a/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml +++ b/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml @@ -7,7 +7,7 @@ metadata: name: imagetag-csi-provisioner-prow-rc imageTag: name: k8s.gcr.io/sig-storage/csi-provisioner - newTag: "v3.1.1" + newTag: "v3.4.0" --- apiVersion: builtin kind: ImageTagTransformer @@ -15,7 +15,7 @@ metadata: name: imagetag-csi-attacher-prow-rc imageTag: name: k8s.gcr.io/sig-storage/csi-attacher - newTag: "v3.5.0" + newTag: "v4.2.0" --- apiVersion: builtin kind: ImageTagTransformer @@ -23,7 +23,7 @@ metadata: name: imagetag-csi-resize-prow-rc imageTag: name: k8s.gcr.io/sig-storage/csi-resizer - newTag: "v1.5.0" + newTag: "v1.7.0" --- apiVersion: builtin kind: ImageTagTransformer @@ -31,7 +31,7 @@ metadata: name: imagetag-csi-snapshotter-prow-head imageTag: name: k8s.gcr.io/sig-storage/csi-snapshotter - newTag: "v6.0.1" + newTag: "v6.1.0" --- apiVersion: builtin kind: ImageTagTransformer @@ -39,7 +39,7 @@ metadata: name: imagetag-csi-node-registrar-prow-rc imageTag: name: k8s.gcr.io/sig-storage/csi-node-driver-registrar - newTag: "v2.5.1" + newTag: "v2.7.0" --- apiVersion: builtin kind: ImageTagTransformer @@ -48,6 +48,6 @@ metadata: imageTag: name: k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.7.2-rc1" + newTag: "v1.9.0-rc1" --- diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index 310f68a2..2c05ad46 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -4,7 +4,7 @@ metadata: name: imagetag-csi-provisioner imageTag: name: k8s.gcr.io/sig-storage/csi-provisioner - newTag: "v3.1.0" + newTag: "v3.4.0" --- apiVersion: builtin @@ -13,7 +13,7 @@ metadata: name: imagetag-csi-attacher imageTag: name: k8s.gcr.io/sig-storage/csi-attacher - newTag: "v3.4.0" + newTag: "v4.2.0" --- apiVersion: builtin @@ -22,7 +22,7 @@ metadata: name: imagetag-csi-resizer imageTag: name: k8s.gcr.io/sig-storage/csi-resizer - newTag: "v1.4.0" + newTag: "v1.7.0" --- apiVersion: builtin @@ -31,7 +31,7 @@ metadata: name: imagetag-csi-snapshotter imageTag: name: k8s.gcr.io/sig-storage/csi-snapshotter - newTag: "v4.0.1" + newTag: "v6.1.0" --- apiVersion: builtin @@ -40,7 +40,7 @@ metadata: name: imagetag-csi-node-registrar imageTag: name: k8s.gcr.io/sig-storage/csi-node-driver-registrar - newTag: "v2.5.0" + newTag: "v2.7.0" --- apiVersion: builtin @@ -52,5 +52,5 @@ imageTag: # Don't change stable image without changing pdImagePlaceholder in # test/k8s-integration/main.go newName: k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.7.2" + newTag: "v1.9.0" --- diff --git a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/default-fstype.yaml b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/default-fstype.yaml new file mode 100644 index 00000000..28e0228c --- /dev/null +++ b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/default-fstype.yaml @@ -0,0 +1,5 @@ +# Set default-fstype for attacher sidecar. +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--default-fstype=ext4" + \ No newline at end of file diff --git a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml index c2351500..2063caa6 100644 --- a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml +++ b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml @@ -12,5 +12,17 @@ patchesJson6902: kind: Deployment name: csi-gce-pd-controller path: disk_labels.yaml +- path: max-grpc-log-length.yaml + target: + group: apps + kind: Deployment + name: csi-gce-pd-controller + version: v1 +- path: default-fstype.yaml + target: + group: apps + kind: Deployment + name: csi-gce-pd-controller + version: v1 transformers: - ../../images/prow-stable-sidecar-rc-master diff --git a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/max-grpc-log-length.yaml b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/max-grpc-log-length.yaml new file mode 100644 index 00000000..839e9cf8 --- /dev/null +++ b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/max-grpc-log-length.yaml @@ -0,0 +1,5 @@ +# Set max-grpc-log-length for attacher sidecar. +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--max-grpc-log-length=10000" + \ No newline at end of file diff --git a/deploy/kubernetes/overlays/stable-master/default-fstype.yaml b/deploy/kubernetes/overlays/stable-master/default-fstype.yaml new file mode 100644 index 00000000..28e0228c --- /dev/null +++ b/deploy/kubernetes/overlays/stable-master/default-fstype.yaml @@ -0,0 +1,5 @@ +# Set default-fstype for attacher sidecar. +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--default-fstype=ext4" + \ No newline at end of file diff --git a/deploy/kubernetes/overlays/stable-master/kustomization.yaml b/deploy/kubernetes/overlays/stable-master/kustomization.yaml index 9f35ef38..5286c01e 100644 --- a/deploy/kubernetes/overlays/stable-master/kustomization.yaml +++ b/deploy/kubernetes/overlays/stable-master/kustomization.yaml @@ -4,5 +4,18 @@ namespace: gce-pd-csi-driver resources: - ../../base/ +patchesJson6902: +- path: max-grpc-log-length.yaml + target: + group: apps + kind: Deployment + name: csi-gce-pd-controller + version: v1 +- path: default-fstype.yaml + target: + group: apps + kind: Deployment + name: csi-gce-pd-controller + version: v1 transformers: - ../../images/stable-master diff --git a/deploy/kubernetes/overlays/stable-master/max-grpc-log-length.yaml b/deploy/kubernetes/overlays/stable-master/max-grpc-log-length.yaml new file mode 100644 index 00000000..839e9cf8 --- /dev/null +++ b/deploy/kubernetes/overlays/stable-master/max-grpc-log-length.yaml @@ -0,0 +1,5 @@ +# Set max-grpc-log-length for attacher sidecar. +- op: add + path: /spec/template/spec/containers/1/args/- + value: "--max-grpc-log-length=10000" + \ No newline at end of file diff --git a/deploy/setup-project.sh b/deploy/setup-project.sh index 01e1f98b..c992dc1d 100755 --- a/deploy/setup-project.sh +++ b/deploy/setup-project.sh @@ -13,7 +13,8 @@ # PROJECT: GCP project # GCE_PD_SA_NAME: Name of the service account to create # GCE_PD_SA_DIR: Directory to save the service account key -# ENABLE_KMS: If true, it will enable Cloud KMS and configure IAM ACLs. +# ENABLE_KMS: Enable Cloud KMS and configure IAM ACLs. +# ENABLE_KMS_ADMIN: Add service account permissions to destroy Cloud KMS keys. # CREATE_SA_KEY: (Optional) If true, creates a new service account key and # exports it if creating a new service account @@ -27,6 +28,7 @@ source "${PKGDIR}/deploy/common.sh" ensure_var PROJECT ensure_var GCE_PD_SA_NAME ensure_var ENABLE_KMS +ensure_var ENABLE_KMS_ADMIN # Allow the user to pass CREATE_SA_KEY=false to skip the SA key creation # Ensure the SA directory set, if we're creating the SA_KEY @@ -35,9 +37,8 @@ if [ "${CREATE_SA_KEY}" = true ]; then ensure_var GCE_PD_SA_DIR fi -# If the project id includes the org name in the format "org-name:project", the -# gCloud api will format the project part of the iam email domain as -# "project.org-name" +# If the project ID includes the org name in the format "org-name:project", +# gcloud will format the project in the IAM email domain as "project.org-name" if [[ $PROJECT == *":"* ]]; then IFS=':' read -ra SPLIT <<< "$PROJECT" readonly IAM_PROJECT="${SPLIT[1]}.${SPLIT[0]}" @@ -90,14 +91,13 @@ fi # Create or Update Custom Role if gcloud iam roles describe gcp_compute_persistent_disk_csi_driver_custom_role --project "${PROJECT}"; then - gcloud iam roles update gcp_compute_persistent_disk_csi_driver_custom_role --quiet \ - --project "${PROJECT}" \ - --file "${PKGDIR}/deploy/gcp-compute-persistent-disk-csi-driver-custom-role.yaml" + action=update else - gcloud iam roles create gcp_compute_persistent_disk_csi_driver_custom_role --quiet \ - --project "${PROJECT}" \ - --file "${PKGDIR}/deploy/gcp-compute-persistent-disk-csi-driver-custom-role.yaml" + action=create fi +gcloud iam roles $action gcp_compute_persistent_disk_csi_driver_custom_role --quiet \ + --project "${PROJECT}" \ + --file "${PKGDIR}/deploy/gcp-compute-persistent-disk-csi-driver-custom-role.yaml" # Bind service account to roles for role in ${BIND_ROLES} @@ -113,6 +113,13 @@ then gcloud projects add-iam-policy-binding "${PROJECT}" --member serviceAccount:"service-${PROJECT_NUMBER}@compute-system.iam.gserviceaccount.com" --role "roles/cloudkms.cryptoKeyEncrypterDecrypter" fi +# Authorize SA to destroy Cloud KMS encryption keys. +if [ "${ENABLE_KMS_ADMIN}" = true ]; +then + gcloud services enable cloudkms.googleapis.com --project="${PROJECT}" + gcloud projects add-iam-policy-binding "${PROJECT}" --member serviceAccount:"${IAM_NAME}" --role "roles/cloudkms.admin" +fi + # Export key if needed if [ "${CREATE_SA}" = true ] && [ "${CREATE_SA_KEY}" = true ]; then diff --git a/docs/kubernetes/development.md b/docs/kubernetes/development.md index 6cdc19fc..abaca7ab 100644 --- a/docs/kubernetes/development.md +++ b/docs/kubernetes/development.md @@ -1,44 +1,190 @@ # Kubernetes Development -## Manual +## Manual deployment To build and install a development version of the driver: -``` -$ GCE_PD_CSI_STAGING_IMAGE=gcr.io/path/to/driver/image:dev # Location to push dev image to -$ make push-container +```sh +GCE_PD_CSI_STAGING_IMAGE=gcr.io/path/to/driver/image:dev # Location to push dev image to +make push-container # Modify controller.yaml and node.yaml in ./deploy/kubernetes/dev to use dev image -$ GCE_PD_DRIVER_VERSION=dev -$ ./deploy/kubernetes/deploy-driver.sh +GCE_PD_DRIVER_VERSION=dev +./deploy/kubernetes/deploy-driver.sh ``` To bring down driver: +```sh +./deploy/kubernetes/delete-driver.sh +``` + +## Testing + +### E2E tests: + +The following environment variables will be required for setup and all +subsequent test runs. + +```sh +export PROJECT=$USER-e2e # GCP project used for testing +export GCE_PD_SA_NAME=$PROJECT-pd-sa # Service account name +export IAM_NAME=$GCE_PD_SA_NAME@$PROJECT.iam.gserviceaccount.com # Service account IAM name +export GCE_PD_SA_DIR=~/credentials # Service account directory +export GOOGLE_APPLICATION_CREDENTIALS=$GCE_PD_SA_DIR/cloud-sa.json # Service account key +``` + +#### One time setup + +The following steps will need to be completed once per project. + +##### Create service account + +To create a service account with correct permissions, your test project must: + +1. Be assigned + [Project IAM Admin](https://cloud.google.com/iam/docs/understanding-roles#resourcemanager.projectIamAdmin) + and + [Role Administrator](https://cloud.google.com/iam/docs/understanding-roles#iam.roleAdmin) + IAM roles. +2. Be part of an organization that allows creation of service account keys. + - For example, a Google internal developer project will not work for E2E + testing because the org restricts SA key creation. + +```sh +export ENABLE_KMS=true +export ENABLE_KMS_ADMIN=true +./deploy/setup-project.sh ``` -$ ./deploy/kubernetes/delete-driver.sh + +##### Create SSH key + +1. Follow instructions to + [generate an SSH key](https://cloud.google.com/compute/docs/connect/create-ssh-keys#create_an_ssh_key_pair) + with the following parameters: + - `KEY_FILENAME: "google_compute_engine"` + - `USERNAME: "$USER"` +2. Add your public key (`cat ~/.ssh/google_compute_engine.pub`) to your + [GCP project metatdata](https://cloud.google.com/compute/docs/connect/add-ssh-keys#add_ssh_keys_to_project_metadata). + +#### Running tests + +```sh +./test/run-e2e-local.sh ``` +##### Debugging + +Local stderr will show you all GCE calls and SSH commands that were performed by +the test. + +###### GCE failures + +You may see errors during instance creation. It is likely your service account +is missing permissions. + +1. Make sure the resources are showing up in pantheon under [Compute Engine > + VM Instances]. +2. Look for errors in pantheon under [Logging > Logs Explorer]. +3. Double check and re-run the + [service account creation](#create-service-account) step. + +###### SSH failures + +You may see errors during driver deployment. + +1. Double check that you have an [SSH key file](#create-ssh-key) in the correct + directory. +2. `gcloud compute ssh $VM_NAME --zone $ZONE` into the VM instance and attempt + to execute the last few commands and look for errors. + - E.g. + [vm and driver have a dependency mismatch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1113)). + +### Sanity tests + +> ***NOTE:*** Sanity tests are currently failing, tracked by +> https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/issues/990. + +``` +./test/run-sanity.sh +``` + +### Unit tests: + +```sh +./test/run-unit.sh +``` + +### Performance tests + +Performance tests are run in automated testing frameworks, but you may wish to +run them locally to benchmark changes. For a given test configuration, you need +to modify the `test-config.yaml` file automatically generated from running one +of the `run-k8s-integration...` or `run-windows-k8s-integration.sh` scripts to +enable performance tests. + +```yaml +DriverInfo: + ... + PerformanceTestOptions: + ProvisioningOptions: # deploy 50 1Gi PVCs and measure latency/throughput + VolumeSize: 1Gi + Count: 50 + ExpectedMetrics: + AvgLatency: 10000000000 # 10 seconds + Throughput: 5 +``` + +You may modify the parameter values to customize the test. + +You also need to modify the `StorageClass` file pointed to in `test-config.yaml` +to set `volumeBindingMode: Immediate`, as the performance tests only support +this mode. + +We will be running the Kubernetes integration tests directly from its +repository. Install `kubetest` and set up a test cluster with the driver +installed. Now, cd into `$GOPATH/src/k8s.io/kubernetes` and run + +```sh +# pwd=k/k +kubetest \ + --check-version-skew=false \ + --test \ + --test_args="--ginkgo.focus=External.Storage.*volume-lifecycle-performance --allowed-not-ready-nodes=10 --node-os-distro= --storage.testdriver=" +``` + +## Dependency management + +Use [dep](https://github.com/golang/dep) + +```sh +dep ensure +``` + +To modify dependencies or versions change `./Gopkg.toml` + ## Debugging -We use https://github.com/go-delve/delve and its feature for remote debugging. This feature -is only available in the PD CSI Controller (which runs in a linux node). +We use https://github.com/go-delve/delve and its feature for remote debugging. +This feature is only available in the PD CSI Controller (which runs in a linux +node). Requirements: -- https://github.com/go-delve/delve +- https://github.com/go-delve/delve Steps: -- Build the PD CSI driver with additional compiler flags. +- Build the PD CSI driver with additional compiler flags. -``` +```sh export GCE_PD_CSI_STAGING_VERSION=latest export GCE_PD_CSI_STAGING_IMAGE=image/repo/gcp-compute-persistent-disk-csi-driver make build-and-push-multi-arch-debug ``` -- Update `deploy/kubernetes/overlays/noauth-debug/kustomization.yaml` to match the repo you wrote above e.g. +- Update `deploy/kubernetes/overlays/noauth-debug/kustomization.yaml` to match + the repo you wrote above e.g. ```yaml images: @@ -47,7 +193,7 @@ images: newTag: latest ``` -- Delete and deploy the driver with this overlay +- Delete and deploy the driver with this overlay ```sh ./deploy/kubernetes/delete-driver.sh && \ @@ -62,13 +208,13 @@ API server listening at: [::]:2345 2021-04-15T18:28:53Z debug layer=debugger continuing ``` -- Enable port forwading of the PD CSI controller of port 2345 +- Enable port forwading of the PD CSI controller of port 2345 ```sh kubectl -n gce-pd-csi-driver get pods | grep controller | awk '{print $1}' | xargs -I % kubectl -n gce-pd-csi-driver port-forward % 2345:2345 ``` -- Connect to the headless server and issue commands +- Connect to the headless server and issue commands ```sh dlv connect localhost:2345 @@ -102,5 +248,6 @@ Command failed: command not available (dlv) ``` -See https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/742 for the implementation details - +See +https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/742 +for the implementation details diff --git a/docs/kubernetes/user-guides/driver-install.md b/docs/kubernetes/user-guides/driver-install.md index a91e7953..727a9787 100644 --- a/docs/kubernetes/user-guides/driver-install.md +++ b/docs/kubernetes/user-guides/driver-install.md @@ -50,7 +50,7 @@ specified service account ```console $ GCE_PD_SA_DIR=/my/safe/credentials/directory # Directory to get the service account key -$ GCE_PD_DRIVER_VERSION=stable # Driver version to deploy +$ GCE_PD_DRIVER_VERSION=stable-master # Driver version to deploy $ ./deploy/kubernetes/deploy-driver.sh ``` diff --git a/docs/local-development.md b/docs/local-development.md deleted file mode 100644 index f9984e89..00000000 --- a/docs/local-development.md +++ /dev/null @@ -1,30 +0,0 @@ -# Local development -This page contains information on how to develop and test the driver locally. - -## Testing - -Running E2E Tests: -``` -$ PROJECT=my-project # GCP Project to run tests in -$ IAM_NAME=my-iam@project.iam.gserviceaccount.com # Existing IAM Account with GCE PD CSI Driver Permissions -$ ./test/run-e2e-local.sh -``` - -Running Sanity Tests: -``` -$ ./test/run-sanity.sh -``` - -Running Unit Tests: -``` -$ ./test/run-unit.sh -``` - -## Dependency Management - -Use [dep](https://github.com/golang/dep) -``` -$ dep ensure -``` - -To modify dependencies or versions change `./Gopkg.toml` diff --git a/edgeless/tests/README.md b/edgeless/tests/README.md index 44369a58..9838168d 100644 --- a/edgeless/tests/README.md +++ b/edgeless/tests/README.md @@ -1,31 +1,38 @@ # CSI driver e2e tests -Run end to end tests using the kubernetes `e2e.test` binary. +Run CSI e2e tests using [`sonobuoy`](https://github.com/vmware-tanzu/sonobuoy/releases/latest). + +## Generate test framework + +Generate CSI e2e test sonobuoy config: -Download the binary for the your Kubernetes version: ```shell -K8S_VER=1.23.0 -curl --location https://dl.k8s.io/v${K8S_VER}/kubernetes-test-linux-amd64.tar.gz | \ - tar --strip-components=3 -zxf - kubernetes/test/bin/e2e.test +KUBECONFIG= sonobuoy.yaml ``` -For an overview on how to run tests read the [Kubernetes blog post](https://kubernetes.io/blog/2020/01/08/testing-of-csi-drivers/#end-to-end-testing). +Apply driver patch: + +```shell +patch sonobuoy.yaml < patch.diff +``` ## Running the test suite -1. Set up the CSI driver +Start the test: - ```shell - kubectl apply -k ./deploy/kubernetes/overlays/edgeless/v1.0.0 - kubectl wait -n kube-system csi-gce-pd-controller --for condition=available - ``` +```shell +kubectl apply -f sonobuoy.yaml +``` -1. Run the tests +Wait for tests to complete: - ```shell - ./e2e.test \ - -ginkgo.v \ - -ginkgo.focus='External.Storage' \ - -ginkgo.skip='\[Disruptive\]' \ - -storage.testdriver=driver.yaml - ``` +```shell +sonobuoy wait +``` + +Analyze results: + +```shell +sonobuoy results $(sonobuoy retrieve) +``` diff --git a/edgeless/tests/patch.diff b/edgeless/tests/patch.diff new file mode 100644 index 00000000..3a398b80 --- /dev/null +++ b/edgeless/tests/patch.diff @@ -0,0 +1,58 @@ +--- o.yaml 2023-04-13 12:13:33.181285060 +0200 ++++ sonobuoy.yaml 2023-04-13 09:36:28.475532271 +0200 +@@ -64,6 +64,10 @@ + data: + plugin-0.yaml: |- + podSpec: ++ volumes: ++ - name: csi-driver-config-volume ++ configMap: ++ name: csi-driver-test-config + containers: [] + nodeSelector: + kubernetes.io/os: linux +@@ -86,7 +90,7 @@ + - /run_e2e.sh + env: + - name: E2E_EXTRA_ARGS +- value: --progress-report-url=http://localhost:8099/progress ++ value: --progress-report-url=http://localhost:8099/progress -storage.testdriver=/tmp/csi-cfg/driver.yaml + - name: E2E_FOCUS + value: External.Storage + - name: E2E_PARALLEL +@@ -113,6 +117,8 @@ + volumeMounts: + - mountPath: /tmp/sonobuoy/results + name: results ++ - mountPath: /tmp/csi-cfg ++ name: csi-driver-config-volume + plugin-1.yaml: |- + podSpec: + containers: [] +@@ -245,4 +251,28 @@ + sonobuoy-component: aggregator + type: ClusterIP + --- +- ++apiVersion: v1 ++kind: ConfigMap ++metadata: ++ name: csi-driver-test-config ++ namespace: sonobuoy ++data: ++ driver.yaml: | ++ StorageClass: ++ FromName: true ++ DriverInfo: ++ Name: gcp.csi.confidential.cloud ++ SupportedFsType: ++ ext4: {} ++ xfs: {} ++ Capabilities: ++ persistence: true ++ block: true ++ exec: true ++ nodeExpansion: true ++ controllerExpansion: true ++ onlineExpansion: true ++ topology: true diff --git a/go.mod b/go.mod index 90366d7e..2ae0cc4c 100644 --- a/go.mod +++ b/go.mod @@ -4,74 +4,76 @@ go 1.19 require ( github.com/GoogleCloudPlatform/k8s-cloud-provider v1.18.1-0.20220218231025-f11817397a1b - github.com/container-storage-interface/spec v1.6.0 + github.com/container-storage-interface/spec v1.7.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 - github.com/kubernetes-csi/csi-proxy/client v1.0.0 - github.com/kubernetes-csi/csi-test/v4 v4.2.0 - github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.1 - golang.org/x/oauth2 v0.2.0 - golang.org/x/sys v0.2.0 - google.golang.org/api v0.103.0 - google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 - google.golang.org/grpc v1.51.0 + github.com/kubernetes-csi/csi-proxy/client v1.1.1 + github.com/kubernetes-csi/csi-test/v4 v4.4.0 + github.com/onsi/ginkgo/v2 v2.7.1 + github.com/onsi/gomega v1.25.0 + golang.org/x/oauth2 v0.6.0 + golang.org/x/sys v0.6.0 + google.golang.org/api v0.114.0 + google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 + google.golang.org/grpc v1.53.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/warnings.v0 v0.1.2 // indirect - k8s.io/apimachinery v0.25.4 - k8s.io/client-go v0.25.4 - k8s.io/component-base v0.25.4 - k8s.io/kubernetes v1.25.4 - k8s.io/mount-utils v0.25.4 + k8s.io/apimachinery v0.26.3 + k8s.io/client-go v0.26.3 + k8s.io/component-base v0.26.3 + k8s.io/kubernetes v1.26.3 + k8s.io/mount-utils v0.26.3 k8s.io/test-infra v0.0.0-20220913174101-46ac1a6cf806 // indirect - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 ) require ( - cloud.google.com/go/compute/metadata v0.2.1 - cloud.google.com/go/kms v1.6.0 - github.com/edgelesssys/constellation/v2 v2.3.0-pre.0.20221130114107-f3e9a830000a - google.golang.org/protobuf v1.28.1 - k8s.io/klog v1.0.0 + cloud.google.com/go/compute/metadata v0.2.3 + cloud.google.com/go/kms v1.10.0 + github.com/edgelesssys/constellation/v2 v2.7.0 + google.golang.org/protobuf v1.29.1 + k8s.io/cloud-provider v0.0.0 + k8s.io/klog/v2 v2.90.0 sigs.k8s.io/boskos v0.0.0-20221128193344-1457a2165fda ) require ( - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/iam v0.12.0 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/moby/sys/mountinfo v0.6.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.13.0 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect @@ -79,53 +81,52 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.3.0 // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/term v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/time v0.2.0 // indirect - golang.org/x/tools v0.3.0 // indirect + golang.org/x/crypto v0.6.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.25.4 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/api v0.26.3 // indirect + k8s.io/kube-openapi v0.0.0-20230228151317-19cbebb19cb7 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( - k8s.io/api => k8s.io/api v0.25.3 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.3 - k8s.io/apimachinery => k8s.io/apimachinery v0.25.3 - k8s.io/apiserver => k8s.io/apiserver v0.25.3 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.25.3 - k8s.io/client-go => k8s.io/client-go v0.25.3 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.25.3 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.25.3 - k8s.io/code-generator => k8s.io/code-generator v0.25.3 - k8s.io/component-base => k8s.io/component-base v0.25.3 - k8s.io/component-helpers => k8s.io/component-helpers v0.25.3 - k8s.io/controller-manager => k8s.io/controller-manager v0.25.3 - k8s.io/cri-api => k8s.io/cri-api v0.25.3 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.25.3 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.25.3 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.25.3 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.25.3 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.25.3 - k8s.io/kubectl => k8s.io/kubectl v0.25.3 - k8s.io/kubelet => k8s.io/kubelet v0.25.3 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.25.3 - k8s.io/metrics => k8s.io/metrics v0.25.3 - k8s.io/mount-utils => k8s.io/mount-utils v0.25.3 - k8s.io/node-api => k8s.io/node-api v0.25.3 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.25.3 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.25.3 - k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.25.3 - k8s.io/sample-controller => k8s.io/sample-controller v0.25.3 + k8s.io/api => k8s.io/api v0.26.3 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.3 + k8s.io/apimachinery => k8s.io/apimachinery v0.26.3 + k8s.io/apiserver => k8s.io/apiserver v0.26.3 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.26.3 + k8s.io/client-go => k8s.io/client-go v0.26.3 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.26.3 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.26.3 + k8s.io/code-generator => k8s.io/code-generator v0.26.3 + k8s.io/component-base => k8s.io/component-base v0.26.3 + k8s.io/component-helpers => k8s.io/component-helpers v0.26.3 + k8s.io/controller-manager => k8s.io/controller-manager v0.26.3 + k8s.io/cri-api => k8s.io/cri-api v0.26.3 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.26.3 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.26.3 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.26.3 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.26.3 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.26.3 + k8s.io/kubectl => k8s.io/kubectl v0.26.3 + k8s.io/kubelet => k8s.io/kubelet v0.26.3 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.26.3 + k8s.io/metrics => k8s.io/metrics v0.26.3 + k8s.io/mount-utils => k8s.io/mount-utils v0.26.3 + k8s.io/node-api => k8s.io/node-api v0.26.3 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.3 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.3 + k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.26.3 + k8s.io/sample-controller => k8s.io/sample-controller v0.26.3 ) diff --git a/go.sum b/go.sum index 72b87809..d75c08e0 100644 --- a/go.sum +++ b/go.sum @@ -25,24 +25,24 @@ cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aD cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= -cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= -cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/kms v1.10.0 h1:Imrtp8792uqNP9bdfPrjtUkjjqOMBcAJ2bdFaAnLhnk= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -76,8 +76,9 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -85,18 +86,23 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= +github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/edgelesssys/constellation/v2 v2.3.0-pre.0.20221130114107-f3e9a830000a h1:zmBFvEIIY1Ahgp28eI1jWmNNTRG5soSl6E/t2vU5hEA= -github.com/edgelesssys/constellation/v2 v2.3.0-pre.0.20221130114107-f3e9a830000a/go.mod h1:1vuLOWnPbPduy8o8DkMZeMuB7BJibPm73n4gzPCzjR4= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/edgelesssys/constellation/v2 v2.7.0 h1:1yYqAecn42Pz/nNrVmuwtJNyfvrU87PIbbSTrNlSGVo= +github.com/edgelesssys/constellation/v2 v2.7.0/go.mod h1:BkAahvBkSZivWW+uhGzLB/yh0ZohTyWE3a0d03106mE= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -104,6 +110,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -122,17 +129,13 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -210,6 +213,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -217,14 +221,14 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= @@ -236,8 +240,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -258,24 +262,24 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kubernetes-csi/csi-proxy/client v1.0.0 h1:jqxpu9VidCtxnjgiIjT1xsIndX1wVRGzOp7R0mCZ+gI= -github.com/kubernetes-csi/csi-proxy/client v1.0.0/go.mod h1:URLOkEbRhOwKVvGvug6HSKRTpLSFuQ/Gt3xahDag8qc= -github.com/kubernetes-csi/csi-test/v4 v4.2.0 h1:uyFJMSN9vnOOuQwndB43Kp4Bi/dScuATdv4FMuGJJQ8= -github.com/kubernetes-csi/csi-test/v4 v4.2.0/go.mod h1:HuWP7lCCJzehodzd4kO170soxqgzSQHZ5Jbp1pKPlmA= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/csi-proxy/client v1.1.1 h1:GagYqArF85E/pQuzldTEQ3WmrcNVU/bYPNG+Ha8tTC4= +github.com/kubernetes-csi/csi-proxy/client v1.1.1/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU= +github.com/kubernetes-csi/csi-test/v4 v4.4.0 h1:r0mnAwDURI24Vw3a/LyA/ga11yD5ZGuU7+REO35Na9s= +github.com/kubernetes-csi/csi-test/v4 v4.4.0/go.mod h1:t1RzseMZJKy313nezI/d7TolbbiKpUZM3SXQvXxOX0w= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 h1:YDjLk3wsL5ZLhLC4TIwIvT2NkSCAdAV6pzzZaRfj4jk= github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6/go.mod h1:gZoZ0+POlM1ge/VUxWpMmZVNPzzMJ7l436CgkQ5+qzU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/sys/mountinfo v0.6.0 h1:gUDhXQx58YNrpHlK4nSL+7y2pxFZkUcXqzFDKWdC0Oo= -github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -288,20 +292,22 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.5/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.7.1 h1:YgLPk+gpqDtAPeRCWEmfO8oxE6ru3xcVSXAM7wn8w9I= +github.com/onsi/ginkgo/v2 v2.7.1/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -313,8 +319,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -334,7 +340,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -342,7 +347,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -361,8 +365,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -378,7 +382,7 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -387,8 +391,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -424,8 +428,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -459,18 +463,18 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -488,8 +492,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= -golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -519,7 +523,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -535,7 +538,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -544,7 +546,6 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -568,14 +569,15 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -585,13 +587,13 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -646,13 +648,12 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -683,8 +684,8 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -752,8 +753,8 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -779,8 +780,9 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -795,20 +797,20 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= @@ -824,6 +826,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -833,37 +836,37 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= -k8s.io/component-base v0.25.3 h1:UrsxciGdrCY03ULT1h/S/gXFCOPnLhUVwSyx+hM/zq4= -k8s.io/component-base v0.25.3/go.mod h1:WYoS8L+IlTZgU7rhAl5Ctpw0WdMxDfCC5dkxcEFa/TI= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU= +k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= +k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= +k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= +k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= +k8s.io/cloud-provider v0.26.3 h1:qN4f0+PvgOZyKIIct+90eedYj8/S4EKPpSD9S+3RCFE= +k8s.io/cloud-provider v0.26.3/go.mod h1:Hfxu0mvz426xlvz2tziw0m3ubr+sk05KKD4HPy2kwdY= +k8s.io/component-base v0.26.3 h1:oC0WMK/ggcbGDTkdcqefI4wIZRYdK3JySx9/HADpV0g= +k8s.io/component-base v0.26.3/go.mod h1:5kj1kZYwSC6ZstHJN7oHBqcJC6yyn41eR+Sqa/mQc8E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kubernetes v1.25.4 h1:M1+MR8IxE64zHhSSDn30twChLaOI+p0Kt77pvyQMKwU= -k8s.io/kubernetes v1.25.4/go.mod h1:lvEY+3iJhh+sGIK1LorGkI56rW0eLGsfalnp68wQwYU= -k8s.io/mount-utils v0.25.3 h1:Eb4MDClmozX3Vrz4ZtoG0bQ/pGhT5gyo28p3f+0r9EE= -k8s.io/mount-utils v0.25.3/go.mod h1:odpFnGwJfFjN3SRnjfGS0902ubcj/W6hDOrNDmSSINo= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= +k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230228151317-19cbebb19cb7 h1:GGNnJLiG6Dk4o75ZHMD40IVsodq/s4kiR0Va5orFZco= +k8s.io/kube-openapi v0.0.0-20230228151317-19cbebb19cb7/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kubernetes v1.26.3 h1:LtjNGNNpCTRyrWhDJMwTWDX+4h+GLwfULS8pu0xzSdk= +k8s.io/kubernetes v1.26.3/go.mod h1:NxzR7U7mS+OGa3J/qweI86Pek//mlfHqDgt6NNGdz8g= +k8s.io/mount-utils v0.26.3 h1:FxMDiPLCkrYgonfSaKHWltLNkyTg3Q/Xrwn94uwhd8k= +k8s.io/mount-utils v0.26.3/go.mod h1:95yx9K6N37y8YZ0/lUh9U6ITosMODNaW0/v4wvaa0Xw= k8s.io/test-infra v0.0.0-20220913174101-46ac1a6cf806 h1:TIT4tzITs2yaaFmBPy3gRVza0QEvE8Sra2cAyAbYrV0= k8s.io/test-infra v0.0.0-20220913174101-46ac1a6cf806/go.mod h1:1emAuQGuX9NvcVYeO10mx7v2TnE3dmHhMiP6rKLVjZM= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1EoULdbQBcAeNJkY= +k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/boskos v0.0.0-20221128193344-1457a2165fda h1:uPIZ+GmcDQKRgVHhRtNEcM8mgb+5OUqLR8Z+EMUJVPA= sigs.k8s.io/boskos v0.0.0-20221128193344-1457a2165fda/go.mod h1:SFfNgqHCE0MxoYAJ9MPL21Lt2q2W+rr5ntzBfd/6Uw0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index b2bde107..422498b0 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -23,10 +23,12 @@ import ( const ( // Parameters for StorageClass - ParameterKeyType = "type" - ParameterKeyReplicationType = "replication-type" - ParameterKeyDiskEncryptionKmsKey = "disk-encryption-kms-key" - ParameterKeyLabels = "labels" + ParameterKeyType = "type" + ParameterKeyReplicationType = "replication-type" + ParameterKeyDiskEncryptionKmsKey = "disk-encryption-kms-key" + ParameterKeyLabels = "labels" + ParameterKeyProvisionedIOPSOnCreate = "provisioned-iops-on-create" + ParameterKeyProvisionedThroughputOnCreate = "provisioned-throughput-on-create" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -75,6 +77,12 @@ type DiskParameters struct { // Values: {map[string]string} // Default: "" Labels map[string]string + // Values: {int64} + // Default: none + ProvisionedIOPSOnCreate int64 + // Values: {int64} + // Default: none + ProvisionedThroughputOnCreate int64 } // SnapshotParameters contains normalized and defaulted parameters for snapshots @@ -135,6 +143,18 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string for labelKey, labelValue := range paramLabels { p.Labels[labelKey] = labelValue } + case ParameterKeyProvisionedIOPSOnCreate: + paramProvisionedIOPSOnCreate, err := ConvertStringToInt64(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) + } + p.ProvisionedIOPSOnCreate = paramProvisionedIOPSOnCreate + case ParameterKeyProvisionedThroughputOnCreate: + paramProvisionedThroughputOnCreate, err := ConvertMiBStringToInt64(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) + } + p.ProvisionedThroughputOnCreate = paramProvisionedThroughputOnCreate default: return p, fmt.Errorf("parameters contains invalid option %q", k) } diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index c9832731..2d1a4b6b 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -74,6 +74,38 @@ func TestExtractAndDefaultParameters(t *testing.T) { }, }, }, + { + name: "values from parameters, checking pd-extreme", + parameters: map[string]string{ParameterKeyType: "pd-extreme", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyProvisionedIOPSOnCreate: "10k"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-extreme", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ProvisionedIOPSOnCreate: 10000, + }, + }, + { + name: "values from parameters, checking hyperdisk-throughput", + parameters: map[string]string{ParameterKeyType: "hyperdisk-throughput", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyProvisionedThroughputOnCreate: "1000Mi"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "hyperdisk-throughput", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ProvisionedThroughputOnCreate: 1000, + }, + }, { name: "values from parameters, checking balanced pd", parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "regional-pd", ParameterKeyDiskEncryptionKmsKey: "foo/key"}, diff --git a/pkg/common/utils.go b/pkg/common/utils.go index d5842ea3..86cc3f9d 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -22,7 +22,9 @@ import ( "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" + volumehelpers "k8s.io/cloud-provider/volume/helpers" ) const ( @@ -248,3 +250,21 @@ func ValidateSnapshotType(snapshotType string) error { return fmt.Errorf("invalid snapshot type %s", snapshotType) } } + +// ConvertStringToInt64 converts a string to int64 +func ConvertStringToInt64(str string) (int64, error) { + quantity, err := resource.ParseQuantity(str) + if err != nil { + return -1, err + } + return volumehelpers.RoundUpToB(quantity) +} + +// ConvertMiBStringToInt64 converts a GiB string to int64 +func ConvertMiBStringToInt64(str string) (int64, error) { + quantity, err := resource.ParseQuantity(str) + if err != nil { + return -1, err + } + return volumehelpers.RoundUpToMiB(quantity) +} diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index ca435caf..b08aee54 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -577,3 +577,213 @@ func TestSnapshotStorageLocations(t *testing.T) { }) } } + +func TestConvertStringToInt64(t *testing.T) { + tests := []struct { + desc string + inputStr string + expInt64 int64 + expectError bool + }{ + { + desc: "valid number string", + inputStr: "10000", + expInt64: 10000, + expectError: false, + }, + { + desc: "round M to number", + inputStr: "1M", + expInt64: 1000000, + expectError: false, + }, + { + desc: "round m to number", + inputStr: "1m", + expInt64: 1, + expectError: false, + }, + { + desc: "round k to number", + inputStr: "1k", + expInt64: 1000, + expectError: false, + }, + { + desc: "invalid empty string", + inputStr: "", + expInt64: 10000, + expectError: true, + }, + { + desc: "invalid string", + inputStr: "ew%65", + expInt64: 10000, + expectError: true, + }, + { + desc: "invalid KiB string", + inputStr: "10KiB", + expInt64: 10000, + expectError: true, + }, + { + desc: "invalid GB string", + inputStr: "10GB", + expInt64: 10000, + expectError: true, + }, + { + desc: "round Ki to number", + inputStr: "1Ki", + expInt64: 1024, + expectError: false, + }, + { + desc: "round k to number", + inputStr: "10k", + expInt64: 10000, + expectError: false, + }, + { + desc: "round Mi to number", + inputStr: "10Mi", + expInt64: 10485760, + expectError: false, + }, + { + desc: "round M to number", + inputStr: "10M", + expInt64: 10000000, + expectError: false, + }, + { + desc: "round G to number", + inputStr: "10G", + expInt64: 10000000000, + expectError: false, + }, + { + desc: "round Gi to number", + inputStr: "100Gi", + expInt64: 107374182400, + expectError: false, + }, + { + desc: "round decimal to number", + inputStr: "1.2Gi", + expInt64: 1288490189, + expectError: false, + }, + { + desc: "round big value to number", + inputStr: "8191Pi", + expInt64: 9222246136947933184, + expectError: false, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + actualInt64, err := ConvertStringToInt64(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to int64 %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to int64 %s; expect an error", tc.inputStr) + } + if err == nil && actualInt64 != tc.expInt64 { + t.Errorf("Got %d for converting string to int64; expect %d", actualInt64, tc.expInt64) + } + }) + } +} + +func TestConvertMiBStringToInt64(t *testing.T) { + tests := []struct { + desc string + inputStr string + expInt64 int64 + expectError bool + }{ + { + "valid number string", + "10000", + 1, + false, + }, + { + "round Ki to MiB", + "1000Ki", + 1, + false, + }, + { + "round k to MiB", + "1000k", + 1, + false, + }, + { + "round Mi to MiB", + "1000Mi", + 1000, + false, + }, + { + "round M to MiB", + "1000M", + 954, + false, + }, + { + "round G to MiB", + "1000G", + 953675, + false, + }, + { + "round Gi to MiB", + "10000Gi", + 10240000, + false, + }, + { + "round decimal to MiB", + "1.2Gi", + 1229, + false, + }, + { + "round big value to MiB", + "8191Pi", + 8795019280384, + false, + }, + { + "invalid empty string", + "", + 10000, + true, + }, + { + "invalid string", + "ew%65", + 10000, + true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + actualInt64, err := ConvertMiBStringToInt64(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to int64 %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to int64 %s; expect an error", tc.inputStr) + } + if err == nil && actualInt64 != tc.expInt64 { + t.Errorf("Got %d for converting string to int64; expect %d", actualInt64, tc.expInt64) + } + }) + } +} diff --git a/pkg/mount-manager/device-utils.go b/pkg/deviceutils/device-utils.go similarity index 79% rename from pkg/mount-manager/device-utils.go rename to pkg/deviceutils/device-utils.go index 66df1caa..f65a79ae 100644 --- a/pkg/mount-manager/device-utils.go +++ b/pkg/deviceutils/device-utils.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mountmanager +package deviceutils import ( "fmt" @@ -25,8 +25,9 @@ import ( "time" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" pathutils "k8s.io/utils/path" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" ) const ( @@ -84,6 +85,9 @@ type DeviceUtils interface { // DisableDevice performs necessary disabling prior to a device being // detached from a node. The path is that from GetDiskByIdPaths. DisableDevice(devicePath string) error + + // Resize returns whether or not a device needs resizing. + Resize(resizer resizefs.Resizefs, devicePath string, deviceMountPath string) (bool, error) } type deviceUtils struct { @@ -114,7 +118,7 @@ func (m *deviceUtils) GetDiskByIdPaths(deviceName string, partition string) []st func existingDevicePath(devicePaths []string) (string, error) { for _, devicePath := range devicePaths { if pathExists, err := pathExists(devicePath); err != nil { - return "", fmt.Errorf("error checking if path exists: %v", err) + return "", fmt.Errorf("error checking if path exists: %w", err) } else if pathExists { return devicePath, nil } @@ -133,7 +137,7 @@ func getScsiSerial(devicePath string) (string, error) { "--whitelisted", fmt.Sprintf("--device=%v", devicePath)).CombinedOutput() if err != nil { - return "", fmt.Errorf("scsi_id failed for device %q with output %s: %v", devicePath, string(out), err) + return "", fmt.Errorf("scsi_id failed for device %q with output %s: %w", devicePath, string(out), err) } return parseScsiSerial(string(out)) @@ -159,7 +163,7 @@ func getNvmeSerial(devicePath string) (string, error) { nvmeIdPath, fmt.Sprintf("-d%s", devicePath)).CombinedOutput() if err != nil { - return "", fmt.Errorf("google_nvme_id failed for device %q with output %v: %v", devicePath, out, err) + return "", fmt.Errorf("google_nvme_id failed for device %q with output %v: %w", devicePath, out, err) } return parseNvmeSerial(string(out)) @@ -178,7 +182,7 @@ func parseNvmeSerial(output string) (string, error) { func ensureUdevToolExists(toolPath string) error { exists, err := pathutils.Exists(pathutils.CheckFollowSymlink, toolPath) if err != nil { - return fmt.Errorf("failed to check existence of %q: %v", toolPath, err) + return fmt.Errorf("failed to check existence of %q: %w", toolPath, err) } if !exists { // The driver should be containerized with the tool so maybe something is @@ -215,22 +219,25 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) if err != nil { return "", err } - err = wait.Poll(pollInterval, pollTimeout, func() (bool, error) { var innerErr error - devicePath, innerErr = existingDevicePath(devicePaths) if innerErr != nil { - return false, fmt.Errorf("failed to check for existing device path: %v", innerErr) + e := fmt.Errorf("for disk %s failed to check for existing device path: %w", deviceName, innerErr) + klog.Errorf(e.Error()) + return false, e } if len(devicePath) == 0 { // Couldn't find a /dev/disk/by-id path for this deviceName, so we need to // find a /dev/* with a serial that matches deviceName. Then we attempt // to repair the symlink. + klog.Warningf("For disk %s couldn't find a device path, calling udevadmTriggerForDiskIfExists", deviceName) innerErr := udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { - return false, fmt.Errorf("failed to trigger udevadm fix of non existent disk for %q: %v", deviceName, innerErr) + e := fmt.Errorf("for disk %s failed to trigger udevadm fix of non existent device path: %w", deviceName, innerErr) + klog.Errorf(e.Error()) + return false, e } // Go to next retry loop to get the deviceName again after // potentially fixing it with the udev command @@ -241,13 +248,19 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) // expected disk at devicePath by matching device Serial to the disk name devFsPath, innerErr := filepath.EvalSymlinks(devicePath) if innerErr != nil { - return false, fmt.Errorf("filepath.EvalSymlinks(%q) failed with %v", devicePath, innerErr) + e := fmt.Errorf("filepath.EvalSymlinks(%q) failed: %w", devicePath, innerErr) + klog.Errorf(e.Error()) + return false, e } + klog.V(4).Infof("For disk %s the /dev/* path is %s for disk/by-id path %s", deviceName, devFsPath, devicePath) devFsSerial, innerErr := getDevFsSerial(devFsPath) if innerErr != nil { - return false, fmt.Errorf("couldn't get serial number for disk %s at path %s: %v", deviceName, devFsPath, innerErr) + e := fmt.Errorf("couldn't get serial number for disk %s at device path %s: %w", deviceName, devFsPath, innerErr) + klog.Errorf(e.Error()) + return false, e } + klog.V(4).Infof("For disk %s, device path %s, found serial number %s", deviceName, devFsPath, devFsSerial) // SUCCESS! devicePath points to a /dev/* path that has a serial // equivalent to our disk name if len(devFsSerial) != 0 && devFsSerial == deviceName { @@ -257,9 +270,12 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) // A /dev/* path exists, but is either not a recognized /dev prefix type // (/dev/nvme* or /dev/sd*) or devicePath is not mapped to the correct disk. // Attempt a repair + klog.Warningf("For disk %s and device path %s with mismatched serial number %q calling udevadmTriggerForDiskIfExists", deviceName, devFsPath, devFsSerial) innerErr = udevadmTriggerForDiskIfExists(deviceName) if innerErr != nil { - return false, fmt.Errorf("failed to trigger udevadm fix of misconfigured disk for %q: %v", deviceName, innerErr) + e := fmt.Errorf("failed to trigger udevadm fix of misconfigured disk for %q: %w", deviceName, innerErr) + klog.Errorf(e.Error()) + return false, e } // Go to next retry loop to get the deviceName again after // potentially fixing it with the udev command @@ -267,12 +283,16 @@ func (m *deviceUtils) VerifyDevicePath(devicePaths []string, deviceName string) }) if err != nil { - return "", fmt.Errorf("failed to find and re-link disk %s with udevadm after retrying for %v: %v", deviceName, pollTimeout, err) + return "", fmt.Errorf("failed to find and re-link disk %s with udevadm after retrying for %v: %w", deviceName, pollTimeout, err) } return devicePath, nil } +func (m *deviceUtils) Resize(resizer resizefs.Resizefs, devicePath string, deviceMountPath string) (bool, error) { + return resizer.Resize(devicePath, deviceMountPath) +} + // getDevFsSerial returns the serial number of the /dev/* path at devFsPath. // If devFsPath does not start with a known prefix, returns the empty string. func getDevFsSerial(devFsPath string) (string, error) { @@ -289,11 +309,11 @@ func getDevFsSerial(devFsPath string) (string, error) { func findAvailableDevFsPaths() ([]string, error) { diskSDPaths, err := filepath.Glob(diskSDPattern) if err != nil { - return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %v", diskSDPattern, err) + return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskSDPattern, err) } diskNvmePaths, err := filepath.Glob(diskNvmePattern) if err != nil { - return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %v", diskNvmePattern, err) + return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskNvmePattern, err) } return append(diskSDPaths, diskNvmePaths...), nil } @@ -309,23 +329,23 @@ func udevadmTriggerForDiskIfExists(deviceName string) error { if err != nil || len(devFsSerial) == 0 { // If we get an error, ignore. Either this isn't a block device, or it // isn't something we can get a serial number from - klog.V(7).Infof("failed to get Serial num for disk %s at path %s: %v", deviceName, devFsPath, err) + klog.Errorf("failed to get serial num for disk %s at device path %s: %v", deviceName, devFsPath, err.Error()) continue } + klog.V(4).Infof("device path %s, serial number %v", devFsPath, devFsSerial) devFsPathToSerial[devFsPath] = devFsSerial if devFsSerial == deviceName { // Found the disk that we're looking for so run a trigger on it // to resolve its /dev/by-id/ path - klog.Warningf("udevadm --trigger running to fix disk at path %s which has serial numberID %s", devFsPath, devFsSerial) + klog.Warningf("udevadm --trigger running to fix disk at path %s which has serial number %s", devFsPath, devFsSerial) err := udevadmChangeToDrive(devFsPath) if err != nil { - return fmt.Errorf("failed to fix disk which has serial numberID %s: %v", devFsSerial, err) + return fmt.Errorf("udevadm --trigger failed to fix device path %s which has serial number %s: %w", devFsPath, devFsSerial, err) } return nil } } - klog.Warningf("udevadm --trigger requested to fix disk %s but no such disk was found in %v", deviceName, devFsPathToSerial) - return fmt.Errorf("udevadm --trigger requested to fix disk %s but no such disk was found", deviceName) + return fmt.Errorf("udevadm --trigger requested to fix disk %s but no such disk was found in device path %v", deviceName, devFsPathToSerial) } // Calls "udevadm trigger --action=change" on the specified drive. drivePath @@ -338,13 +358,15 @@ func udevadmTriggerForDiskIfExists(deviceName string) error { // the change func udevadmChangeToDrive(devFsPath string) error { // Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/..." - out, err := exec.Command( + cmd := exec.Command( "udevadm", "trigger", "--action=change", - fmt.Sprintf("--property-match=DEVNAME=%s", devFsPath)).CombinedOutput() + fmt.Sprintf("--property-match=DEVNAME=%s", devFsPath)) + klog.V(4).Infof("Running command: %s", cmd.String()) + out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with output %s: %v.", devFsPath, string(out), err) + return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with output %s: %w", devFsPath, string(out), err) } return nil } diff --git a/pkg/mount-manager/device-utils_linux.go b/pkg/deviceutils/device-utils_linux.go similarity index 97% rename from pkg/mount-manager/device-utils_linux.go rename to pkg/deviceutils/device-utils_linux.go index 10476601..3e1cda5c 100644 --- a/pkg/mount-manager/device-utils_linux.go +++ b/pkg/deviceutils/device-utils_linux.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mountmanager +package deviceutils import ( "fmt" diff --git a/pkg/mount-manager/device-utils_test.go b/pkg/deviceutils/device-utils_test.go similarity index 98% rename from pkg/mount-manager/device-utils_test.go rename to pkg/deviceutils/device-utils_test.go index dd3d22b0..630ab4c7 100644 --- a/pkg/mount-manager/device-utils_test.go +++ b/pkg/deviceutils/device-utils_test.go @@ -1,4 +1,4 @@ -package mountmanager +package deviceutils import ( "testing" diff --git a/pkg/mount-manager/device-utils_windows.go b/pkg/deviceutils/device-utils_windows.go similarity index 97% rename from pkg/mount-manager/device-utils_windows.go rename to pkg/deviceutils/device-utils_windows.go index 5cf7a892..8b2055a1 100644 --- a/pkg/mount-manager/device-utils_windows.go +++ b/pkg/deviceutils/device-utils_windows.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mountmanager +package deviceutils func (_ *deviceUtils) DisableDevice(devicePath string) error { // No disabling is necessary on windows. diff --git a/pkg/mount-manager/fake-device-utils.go b/pkg/deviceutils/fake-device-utils.go similarity index 84% rename from pkg/mount-manager/fake-device-utils.go rename to pkg/deviceutils/fake-device-utils.go index be68f492..74734076 100644 --- a/pkg/mount-manager/fake-device-utils.go +++ b/pkg/deviceutils/fake-device-utils.go @@ -12,7 +12,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mountmanager +package deviceutils + +import "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" type fakeDeviceUtils struct { } @@ -38,3 +40,7 @@ func (_ *fakeDeviceUtils) DisableDevice(devicePath string) error { // No-op for testing. return nil } + +func (_ *fakeDeviceUtils) Resize(resizer resizefs.Resizefs, devicePath string, deviceMountPath string) (bool, error) { + return false, nil +} diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index 77111a51..9dedbaf3 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -43,7 +43,7 @@ import ( "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "k8s.io/apimachinery/pkg/util/sets" @@ -121,7 +121,7 @@ func (cloud *FakeCloudProvider) RepairUnderspecifiedVolumeKey(ctx context.Contex } r, err := common.GetRegionFromZones([]string{cloud.zone}) if err != nil { - return "", nil, fmt.Errorf("failed to get region from zones: %v", err) + return "", nil, fmt.Errorf("failed to get region from zones: %w", err) } volumeKey.Region = r return project, volumeKey, nil @@ -214,13 +214,14 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, } computeDisk := &computev1.Disk{ - Name: volKey.Name, - SizeGb: common.BytesToGbRoundUp(capBytes), - Description: "Disk created by GCE-PD CSI Driver", - Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), - SourceDiskId: volumeContentSourceVolumeID, - Status: cloud.mockDiskStatus, - Labels: params.Labels, + Name: volKey.Name, + SizeGb: common.BytesToGbRoundUp(capBytes), + Description: "Disk created by GCE-PD CSI Driver", + Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), + SourceDiskId: volumeContentSourceVolumeID, + Status: cloud.mockDiskStatus, + Labels: params.Labels, + ProvisionedIops: params.ProvisionedIOPSOnCreate, } if snapshotID != "" { @@ -442,6 +443,7 @@ func (cloud *FakeCloudProvider) CreateImage(ctx context.Context, project string, SourceType: "RAW", Status: "PENDING", StorageLocations: snapshotParams.StorageLocations, + Labels: snapshotParams.Labels, } switch volKey.Type() { diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index 1c14e600..0be8356d 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -28,7 +28,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" ) @@ -40,6 +40,8 @@ const ( cryptoKeyVerDelimiter = "/cryptoKeyVersions" ) +var hyperdiskTypes = []string{"hyperdisk-extreme", "hyperdisk-throughput"} + type GCEAPIVersion string const ( @@ -115,7 +117,7 @@ func (cloud *CloudProvider) GetDefaultZone() string { func (cloud *CloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, string, error) { region, err := common.GetRegionFromZones([]string{cloud.zone}) if err != nil { - return nil, "", fmt.Errorf("failed to get region from zones: %v", err) + return nil, "", fmt.Errorf("failed to get region from zones: %w", err) } zones, err := cloud.ListZones(ctx, region) if err != nil { @@ -162,7 +164,7 @@ func (cloud *CloudProvider) RepairUnderspecifiedVolumeKey(ctx context.Context, p } region, err := common.GetRegionFromZones([]string{cloud.zone}) if err != nil { - return "", nil, fmt.Errorf("failed to get region from zones: %v", err) + return "", nil, fmt.Errorf("failed to get region from zones: %w", err) } switch volumeKey.Type() { case meta.Zonal: @@ -216,7 +218,7 @@ func (cloud *CloudProvider) ListZones(ctx context.Context, region string) ([]str zones := []string{} zoneList, err := cloud.service.Zones.List(cloud.project).Filter(fmt.Sprintf("region eq .*%s$", region)).Do() if err != nil { - return nil, fmt.Errorf("failed to list zones in region %s: %v", region, err) + return nil, fmt.Errorf("failed to list zones in region %s: %w", region, err) } for _, zone := range zoneList.Items { zones = append(zones, zone.Name) @@ -387,7 +389,7 @@ func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey } } -func convertV1DiskToBetaDisk(v1Disk *computev1.Disk) *computebeta.Disk { +func convertV1DiskToBetaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCreate int64) *computebeta.Disk { var dek *computebeta.CustomerEncryptionKey = nil if v1Disk.DiskEncryptionKey != nil { @@ -395,7 +397,7 @@ func convertV1DiskToBetaDisk(v1Disk *computev1.Disk) *computebeta.Disk { } // Note: this is an incomplete list. It only includes the fields we use for disk creation. - return &computebeta.Disk{ + betaDisk := &computebeta.Disk{ Name: v1Disk.Name, SizeGb: v1Disk.SizeGb, Description: v1Disk.Description, @@ -404,6 +406,11 @@ func convertV1DiskToBetaDisk(v1Disk *computev1.Disk) *computebeta.Disk { ReplicaZones: v1Disk.ReplicaZones, DiskEncryptionKey: dek, } + if provisionedThroughputOnCreate > 0 { + betaDisk.ProvisionedThroughput = provisionedThroughputOnCreate + } + + return betaDisk } func (cloud *CloudProvider) insertRegionalDisk( @@ -429,11 +436,12 @@ func (cloud *CloudProvider) insertRegionalDisk( } diskToCreate := &computev1.Disk{ - Name: volKey.Name, - SizeGb: common.BytesToGbRoundUp(capBytes), - Description: description, - Type: cloud.GetDiskTypeURI(cloud.project, volKey, params.DiskType), - Labels: params.Labels, + Name: volKey.Name, + SizeGb: common.BytesToGbRoundUp(capBytes), + Description: description, + Type: cloud.GetDiskTypeURI(cloud.project, volKey, params.DiskType), + Labels: params.Labels, + ProvisionedIops: params.ProvisionedIOPSOnCreate, } if snapshotID != "" { _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) @@ -464,7 +472,7 @@ func (cloud *CloudProvider) insertRegionalDisk( if gceAPIVersion == GCEAPIVersionBeta { var insertOp *computebeta.Operation - betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate) + betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate, 0) betaDiskToCreate.MultiWriter = multiWriter insertOp, err = cloud.betaService.RegionDisks.Insert(project, volKey.Region, betaDiskToCreate).Context(ctx).Do() if insertOp != nil { @@ -493,7 +501,7 @@ func (cloud *CloudProvider) insertRegionalDisk( klog.Warningf("GCE PD %s already exists, reusing", volKey.Name) return nil } - return status.Error(codes.Internal, fmt.Sprintf("unknown Insert disk error: %v", err)) + return status.Error(codes.Internal, fmt.Sprintf("unknown Insert disk error: %v", err.Error())) } klog.V(5).Infof("InsertDisk operation %s for disk %s", opName, diskToCreate.Name) @@ -514,7 +522,7 @@ func (cloud *CloudProvider) insertRegionalDisk( klog.Warningf("GCE PD %s already exists after wait, reusing", volKey.Name) return nil } - return fmt.Errorf("unknown Insert disk operation error: %v", err) + return fmt.Errorf("unknown Insert disk operation error: %w", err) } return nil } @@ -536,7 +544,7 @@ func (cloud *CloudProvider) insertZonalDisk( gceAPIVersion = GCEAPIVersionV1 ) - if multiWriter { + if multiWriter || containsBetaDiskType(hyperdiskTypes, params.DiskType) { gceAPIVersion = GCEAPIVersionBeta } @@ -574,7 +582,7 @@ func (cloud *CloudProvider) insertZonalDisk( if gceAPIVersion == GCEAPIVersionBeta { var insertOp *computebeta.Operation - betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate) + betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate, params.ProvisionedThroughputOnCreate) betaDiskToCreate.MultiWriter = multiWriter insertOp, err = cloud.betaService.Disks.Insert(project, volKey.Zone, betaDiskToCreate).Context(ctx).Do() if insertOp != nil { @@ -604,7 +612,7 @@ func (cloud *CloudProvider) insertZonalDisk( klog.Warningf("GCE PD %s already exists, reusing", volKey.Name) return nil } - return fmt.Errorf("unknown Insert disk error: %v", err) + return fmt.Errorf("unknown Insert disk error: %w", err) } klog.V(5).Infof("InsertDisk operation %s for disk %s", opName, diskToCreate.Name) @@ -626,7 +634,7 @@ func (cloud *CloudProvider) insertZonalDisk( klog.Warningf("GCE PD %s already exists after wait, reusing", volKey.Name) return nil } - return fmt.Errorf("unknown Insert disk operation error: %v", err) + return fmt.Errorf("unknown Insert disk operation error: %w", err) } return nil } @@ -685,7 +693,7 @@ func (cloud *CloudProvider) AttachDisk(ctx context.Context, project string, volK deviceName, err := common.GetDeviceName(volKey) if err != nil { - return fmt.Errorf("failed to get device name: %v", err) + return fmt.Errorf("failed to get device name: %w", err) } attachedDiskV1 := &computev1.AttachedDisk{ DeviceName: deviceName, @@ -697,13 +705,13 @@ func (cloud *CloudProvider) AttachDisk(ctx context.Context, project string, volK op, err := cloud.service.Instances.AttachDisk(project, instanceZone, instanceName, attachedDiskV1).Context(ctx).Do() if err != nil { - return fmt.Errorf("failed cloud service attach disk call: %v", err) + return fmt.Errorf("failed cloud service attach disk call: %w", err) } klog.V(5).Infof("AttachDisk operation %s for disk %s", op.Name, attachedDiskV1.DeviceName) err = cloud.waitForZonalOp(ctx, project, op.Name, instanceZone) if err != nil { - return fmt.Errorf("failed when waiting for zonal op: %v", err) + return fmt.Errorf("failed when waiting for zonal op: %w", err) } return nil } @@ -814,7 +822,7 @@ func (cloud *CloudProvider) WaitForAttach(ctx context.Context, project string, v klog.V(6).Infof("Polling for attach of disk %v to instance %v to complete for %v", volKey.Name, instanceName, time.Since(start)) disk, err := cloud.GetDisk(ctx, project, volKey, GCEAPIVersionV1) if err != nil { - return false, fmt.Errorf("GetDisk failed to get disk: %v", err) + return false, fmt.Errorf("GetDisk failed to get disk: %w", err) } if disk == nil { @@ -924,6 +932,7 @@ func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, vol Name: imageName, StorageLocations: snapshotParams.StorageLocations, Description: description, + Labels: snapshotParams.Labels, } _, err = cloud.service.Images.Insert(project, image).Context(ctx).ForceCreate(true).Do() @@ -946,7 +955,7 @@ func (cloud *CloudProvider) waitForImageCreation(ctx context.Context, project, i klog.V(6).Infof("Checking GCE Image %s.", imageName) image, err := cloud.GetImage(ctx, project, imageName) if err != nil { - klog.Warningf("Error in getting image %s, %v", imageName, err) + klog.Warningf("Error in getting image %s, %v", imageName, err.Error()) } else if image != nil { if image.Status != "PENDING" { klog.V(6).Infof("Image %s status is %s", imageName, image.Status) @@ -1009,7 +1018,7 @@ func (cloud *CloudProvider) ResizeDisk(ctx context.Context, project string, volK klog.V(5).Infof("Resizing disk %v to size %v", volKey, requestBytes) cloudDisk, err := cloud.GetDisk(ctx, project, volKey, GCEAPIVersionV1) if err != nil { - return -1, fmt.Errorf("failed to get disk: %v", err) + return -1, fmt.Errorf("failed to get disk: %w", err) } sizeGb := cloudDisk.GetSizeGb() @@ -1037,13 +1046,13 @@ func (cloud *CloudProvider) resizeZonalDisk(ctx context.Context, project string, } op, err := cloud.service.Disks.Resize(project, volKey.Zone, volKey.Name, resizeReq).Context(ctx).Do() if err != nil { - return -1, fmt.Errorf("failed to resize zonal volume %v: %v", volKey.String(), err) + return -1, fmt.Errorf("failed to resize zonal volume %v: %w", volKey.String(), err) } klog.V(5).Infof("ResizeDisk operation %s for disk %s", op.Name, volKey.Name) err = cloud.waitForZonalOp(ctx, project, op.Name, volKey.Zone) if err != nil { - return -1, fmt.Errorf("failed waiting for op for zonal resize for %s: %v", volKey.String(), err) + return -1, fmt.Errorf("failed waiting for op for zonal resize for %s: %w", volKey.String(), err) } return requestGb, nil @@ -1056,13 +1065,13 @@ func (cloud *CloudProvider) resizeRegionalDisk(ctx context.Context, project stri op, err := cloud.service.RegionDisks.Resize(project, volKey.Region, volKey.Name, resizeReq).Context(ctx).Do() if err != nil { - return -1, fmt.Errorf("failed to resize regional volume %v: %v", volKey.String(), err) + return -1, fmt.Errorf("failed to resize regional volume %v: %w", volKey.String(), err) } klog.V(5).Infof("ResizeDisk operation %s for disk %s", op.Name, volKey.Name) err = cloud.waitForRegionalOp(ctx, project, op.Name, volKey.Region) if err != nil { - return -1, fmt.Errorf("failed waiting for op for regional resize for %s: %v", volKey.String(), err) + return -1, fmt.Errorf("failed waiting for op for regional resize for %s: %w", volKey.String(), err) } return requestGb, nil @@ -1114,7 +1123,7 @@ func (cloud *CloudProvider) waitForSnapshotCreation(ctx context.Context, project klog.V(6).Infof("Checking GCE Snapshot %s.", snapshotName) snapshot, err := cloud.GetSnapshot(ctx, project, snapshotName) if err != nil { - klog.Warningf("Error in getting snapshot %s, %v", snapshotName, err) + klog.Warningf("Error in getting snapshot %s, %v", snapshotName, err.Error()) } else if snapshot != nil { if snapshot.Status != "CREATING" { klog.V(6).Infof("Snapshot %s status is %s", snapshotName, snapshot.Status) @@ -1161,7 +1170,17 @@ func encodeTags(tags map[string]string) (string, error) { enc, err := json.Marshal(tags) if err != nil { - return "", fmt.Errorf("failed to encodeTags %v: %v", tags, err) + return "", fmt.Errorf("failed to encodeTags %v: %w", tags, err) } return string(enc), nil } + +func containsBetaDiskType(betaDiskTypes []string, diskType string) bool { + for _, betaDiskType := range betaDiskTypes { + if betaDiskType == diskType { + return true + } + } + + return false +} diff --git a/pkg/gce-cloud-provider/compute/gce.go b/pkg/gce-cloud-provider/compute/gce.go index 608feeae..26bf46a0 100644 --- a/pkg/gce-cloud-provider/compute/gce.go +++ b/pkg/gce-cloud-provider/compute/gce.go @@ -50,7 +50,7 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" ) const ( @@ -62,10 +62,6 @@ const ( regionURITemplate = "projects/%s/regions/%s" - GCEComputeAPIEndpoint = "https://www.googleapis.com/compute/v1/" - GCEComputeBetaAPIEndpoint = "https://www.googleapis.com/compute/beta/" - GCEComputeAlphaAPIEndpoint = "https://www.googleapis.com/compute/alpha/" - replicaZoneURITemplateSingleZone = "projects/%s/zones/%s" // {gce.projectID}/zones/{disk.Zone} ) @@ -91,7 +87,7 @@ type ConfigGlobal struct { Zone string `gcfg:"zone"` } -func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath string) (*CloudProvider, error) { +func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath string, computeEndpoint string) (*CloudProvider, error) { configFile, err := readConfig(configPath) if err != nil { return nil, err @@ -106,19 +102,19 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s return nil, err } - svc, err := createCloudService(ctx, vendorVersion, tokenSource) + svc, err := createCloudService(ctx, vendorVersion, tokenSource, computeEndpoint) if err != nil { return nil, err } - betasvc, err := createBetaCloudService(ctx, vendorVersion, tokenSource) + betasvc, err := createBetaCloudService(ctx, vendorVersion, tokenSource, computeEndpoint) if err != nil { return nil, err } project, zone, err := getProjectAndZone(configFile) if err != nil { - return nil, fmt.Errorf("Failed getting Project and Zone: %v", err) + return nil, fmt.Errorf("Failed getting Project and Zone: %w", err) } return &CloudProvider{ @@ -166,23 +162,29 @@ func readConfig(configPath string) (*ConfigFile, error) { reader, err := os.Open(configPath) if err != nil { - return nil, fmt.Errorf("couldn't open cloud provider configuration at %s: %v", configPath, err) + return nil, fmt.Errorf("couldn't open cloud provider configuration at %s: %w", configPath, err) } defer reader.Close() cfg := &ConfigFile{} if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil { - return nil, fmt.Errorf("couldn't read cloud provider configuration at %s: %v", configPath, err) + return nil, fmt.Errorf("couldn't read cloud provider configuration at %s: %w", configPath, err) } return cfg, nil } -func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource) (*computebeta.Service, error) { +func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*computebeta.Service, error) { client, err := newOauthClient(ctx, tokenSource) if err != nil { return nil, err } - service, err := computebeta.NewService(ctx, option.WithHTTPClient(client)) + + computeOpts := []option.ClientOption{option.WithHTTPClient(client)} + if computeEndpoint != "" { + betaEndpoint := fmt.Sprintf("%s/compute/beta/", computeEndpoint) + computeOpts = append(computeOpts, option.WithEndpoint(betaEndpoint)) + } + service, err := computebeta.NewService(ctx, computeOpts...) if err != nil { return nil, err } @@ -190,17 +192,23 @@ func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSour return service, nil } -func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) { - svc, err := createCloudServiceWithDefaultServiceAccount(ctx, vendorVersion, tokenSource) +func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) { + svc, err := createCloudServiceWithDefaultServiceAccount(ctx, vendorVersion, tokenSource, computeEndpoint) return svc, err } -func createCloudServiceWithDefaultServiceAccount(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource) (*compute.Service, error) { +func createCloudServiceWithDefaultServiceAccount(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) { client, err := newOauthClient(ctx, tokenSource) if err != nil { return nil, err } - service, err := compute.New(client) + + computeOpts := []option.ClientOption{option.WithHTTPClient(client)} + if computeEndpoint != "" { + v1Endpoint := fmt.Sprintf("%s/compute/v1/", computeEndpoint) + computeOpts = append(computeOpts, option.WithEndpoint(v1Endpoint)) + } + service, err := compute.NewService(ctx, computeOpts...) if err != nil { return nil, err } @@ -211,7 +219,7 @@ func createCloudServiceWithDefaultServiceAccount(ctx context.Context, vendorVers func newOauthClient(ctx context.Context, tokenSource oauth2.TokenSource) (*http.Client, error) { if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { if _, err := tokenSource.Token(); err != nil { - klog.Errorf("error fetching initial token: %v", err) + klog.Errorf("error fetching initial token: %v", err.Error()) return false, nil } return true, nil diff --git a/pkg/gce-cloud-provider/metadata/metadata.go b/pkg/gce-cloud-provider/metadata/metadata.go index 8e2d7c59..460bd00a 100644 --- a/pkg/gce-cloud-provider/metadata/metadata.go +++ b/pkg/gce-cloud-provider/metadata/metadata.go @@ -45,19 +45,19 @@ var _ MetadataService = &metadataServiceManager{} func NewMetadataService() (MetadataService, error) { zone, err := metadata.Zone() if err != nil { - return nil, fmt.Errorf("failed to get current zone: %v", err) + return nil, fmt.Errorf("failed to get current zone: %w", err) } projectID, err := metadata.ProjectID() if err != nil { - return nil, fmt.Errorf("failed to get project: %v", err) + return nil, fmt.Errorf("failed to get project: %w", err) } name, err := metadata.InstanceName() if err != nil { - return nil, fmt.Errorf("failed to get instance name: %v", err) + return nil, fmt.Errorf("failed to get instance name: %w", err) } fullMachineType, err := metadata.Get("instance/machine-type") if err != nil { - return nil, fmt.Errorf("failed to get machine-type: %v", err) + return nil, fmt.Errorf("failed to get machine-type: %w", err) } // Response format: "projects/[NUMERIC_PROJECT_ID]/machineTypes/[MACHINE_TYPE]" splits := strings.Split(fullMachineType, "/") diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index 19ccec2a..c9f3fb60 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -18,8 +18,8 @@ import ( "context" "fmt" "math/rand" + "regexp" "sort" - "strings" "time" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" @@ -31,17 +31,12 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/util/flowcontrol" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" ) -const ( - errorBackoffInitialDuration = 200 * time.Millisecond - errorBackoffMaxDuration = 5 * time.Minute -) - type GCEControllerServer struct { Driver *GCEDriver CloudProvider gce.GCECompute @@ -162,19 +157,19 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre capBytes, err := getRequestCapacity(capacityRange) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume Request Capacity is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume Request Capacity is invalid: %v", err.Error()) } err = validateVolumeCapabilities(volumeCapabilities) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapabilities is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error()) } // Apply Parameters (case-insensitive). We leave validation of // the values to the cloud provider. params, err := common.ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.name, gceCS.Driver.extraVolumeLabels) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err) + return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } // Determine multiWriter gceAPIVersion := gce.GCEAPIVersionV1 @@ -189,30 +184,30 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre case replicationTypeNone: zones, err = pickZones(ctx, gceCS, req.GetAccessibilityRequirements(), 1) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume failed to pick zones for disk: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to pick zones for disk: %v", err.Error()) } if len(zones) != 1 { - return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to pick exactly 1 zone for zonal disk, got %v instead", len(zones))) + return nil, status.Errorf(codes.Internal, "Failed to pick exactly 1 zone for zonal disk, got %v instead", len(zones)) } volKey = meta.ZonalKey(name, zones[0]) case replicationTypeRegionalPD: zones, err = pickZones(ctx, gceCS, req.GetAccessibilityRequirements(), 2) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume failed to pick zones for disk: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to pick zones for disk: %v", err.Error()) } region, err := common.GetRegionFromZones(zones) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume failed to get region from zones: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to get region from zones: %v", err.Error()) } volKey = meta.RegionalKey(name, region) default: - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume replication type '%s' is not supported", params.ReplicationType)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume replication type '%s' is not supported", params.ReplicationType) } volumeID, err := common.KeyToVolumeID(volKey, gceCS.CloudProvider.GetDefaultProject()) if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to convert volume key to volume ID: %v", err) + return nil, LoggedError("Failed to convert volume key to volume ID: ", err) } if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { return nil, status.Errorf(codes.Aborted, common.VolumeOperationAlreadyExistsFmt, volumeID) @@ -223,7 +218,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre existingDisk, err := gceCS.CloudProvider.GetDisk(ctx, gceCS.CloudProvider.GetDefaultProject(), volKey, gceAPIVersion) if err != nil { if !gce.IsGCEError(err, "notFound") { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume unknown get disk error when validating: %v", err)) + return nil, LoggedError("CreateVolume unknown get disk error when validating: ", err) } } if err == nil { @@ -233,15 +228,15 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre int64(capacityRange.GetLimitBytes()), multiWriter) if err != nil { - return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("CreateVolume disk already exists with same name and is incompatible: %v", err)) + return nil, status.Errorf(codes.AlreadyExists, "CreateVolume disk already exists with same name and is incompatible: %v", err.Error()) } ready, err := isDiskReady(existingDisk) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk %v had error checking ready status: %v", volKey, err)) + return nil, LoggedError("CreateVolume disk "+volKey.String()+" had error checking ready status: ", err) } if !ready { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume existing disk %v is not ready", volKey)) + return nil, status.Errorf(codes.Internal, "CreateVolume existing disk %v is not ready", volKey) } // If there is no validation error, immediately return success @@ -259,7 +254,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // Verify that snapshot exists sl, err := gceCS.getSnapshotByID(ctx, snapshotID) if err != nil { - return nil, status.Errorf(codes.Internal, "CreateVolume failed to get snapshot %s: %v", snapshotID, err) + return nil, LoggedError("CreateVolume failed to get snapshot "+snapshotID+": ", err) } else if len(sl.Entries) == 0 { return nil, status.Errorf(codes.NotFound, "CreateVolume source snapshot %s does not exist", snapshotID) } @@ -270,7 +265,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // Verify that the source VolumeID is in the correct format. project, sourceVolKey, err := common.VolumeIDToKey(volumeContentSourceVolumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume source volume id is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume source volume id is invalid: %v", err.Error()) } // Verify that the volume in VolumeContentSource exists. @@ -279,7 +274,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if gce.IsGCEError(err, "notFound") { return nil, status.Errorf(codes.NotFound, "CreateVolume source volume %s does not exist", volumeContentSourceVolumeID) } else { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume unknown get disk error when validating: %v", err)) + return nil, LoggedError("CreateVolume unknown get disk error when validating: ", err) } } @@ -317,10 +312,10 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // Verify the source disk is ready. ready, err := isDiskReady(diskFromSourceVolume) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk from source volume %v had error checking ready status: %v", sourceVolKey, err)) + return nil, LoggedError("CreateVolume disk from source volume "+sourceVolKey.String()+" had error checking ready status: ", err) } if !ready { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk from source volume %v is not ready", sourceVolKey)) + return nil, status.Errorf(codes.Internal, "CreateVolume disk from source volume %v is not ready", sourceVolKey) } } } else { // if VolumeContentSource is nil, validate access mode is not read only @@ -334,30 +329,30 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre switch params.ReplicationType { case replicationTypeNone: if len(zones) != 1 { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to get a single zone for creating zonal disk, instead got: %v", zones)) + return nil, status.Errorf(codes.Internal, "CreateVolume failed to get a single zone for creating zonal disk, instead got: %v", zones) } disk, err = createSingleZoneDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to create single zonal disk %#v: %v", name, err)) + return nil, LoggedError("CreateVolume failed to create single zonal disk "+name+": ", err) } case replicationTypeRegionalPD: if len(zones) != 2 { - return nil, status.Errorf(codes.Internal, fmt.Sprintf("CreateVolume failed to get a 2 zones for creating regional disk, instead got: %v", zones)) + return nil, status.Errorf(codes.Internal, "CreateVolume failed to get a 2 zones for creating regional disk, instead got: %v", zones) } disk, err = createRegionalDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to create regional disk %#v: %v", name, err)) + return nil, LoggedError("CreateVolume failed to create regional disk "+name+": ", err) } default: - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume replication type '%s' is not supported", params.ReplicationType)) + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume replication type '%s' is not supported", params.ReplicationType) } ready, err := isDiskReady(disk) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk %v had error checking ready status: %v", volKey, err)) + return nil, status.Errorf(codes.Internal, "CreateVolume disk %v had error checking ready status: %v", volKey, err.Error()) } if !ready { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk %v is not ready", volKey)) + return nil, status.Errorf(codes.Internal, "CreateVolume disk %v is not ready", volKey) } klog.V(4).Infof("CreateVolume succeeded for disk %v", volKey) @@ -376,17 +371,17 @@ func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.Del if err != nil { // Cannot find volume associated with this ID because VolumeID is not in // correct format, this is a success according to the Spec - klog.Warningf("DeleteVolume treating volume as deleted because volume id %s is invalid: %v", volumeID, err) + klog.Warningf("DeleteVolume treating volume as deleted because volume id %s is invalid: %v", volumeID, err.Error()) return &csi.DeleteVolumeResponse{}, nil } project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { - klog.Warningf("DeleteVolume treating volume as deleted because cannot find volume %v: %v", volumeID, err) + klog.Warningf("DeleteVolume treating volume as deleted because cannot find volume %v: %v", volumeID, err.Error()) return &csi.DeleteVolumeResponse{}, nil } - return nil, status.Errorf(codes.Internal, "DeleteVolume error repairing underspecified volume key: %v", err) + return nil, LoggedError("DeleteVolume error repairing underspecified volume key: ", err) } if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { @@ -396,7 +391,7 @@ func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.Del err = gceCS.CloudProvider.DeleteDisk(ctx, project, volKey) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete disk error: %v", err)) + return nil, LoggedError("unknown Delete disk error: ", err) } klog.V(4).Infof("DeleteVolume succeeded for disk %v", volKey) @@ -417,7 +412,7 @@ func (gceCS *GCEControllerServer) ControllerPublishVolume(ctx context.Context, r resp, err := gceCS.executeControllerPublishVolume(ctx, req) if err != nil { - klog.Infof("For node %s adding backoff due to error for volume %s: %v", req.NodeId, req.VolumeId, err) + klog.Infof("For node %s adding backoff due to error for volume %s: %v", req.NodeId, req.VolumeId, err.Error()) gceCS.errorBackoff.next(backoffId) } else { klog.Infof("For node %s clear backoff due to successful publish of volume %v", req.NodeId, req.VolumeId) @@ -443,12 +438,12 @@ func (gceCS *GCEControllerServer) validateControllerPublishVolumeRequest(ctx con project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerPublishVolume volume ID is invalid: %v", err)) + return "", nil, status.Errorf(codes.InvalidArgument, "ControllerPublishVolume volume ID is invalid: %v", err.Error()) } // TODO(#253): Check volume capability matches for ALREADY_EXISTS if err = validateVolumeCapability(volumeCapability); err != nil { - return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapabilities is invalid: %v", err)) + return "", nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error()) } return project, volKey, nil @@ -473,9 +468,9 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Errorf(codes.NotFound, "ControllerPublishVolume could not find volume with ID %v: %v", volumeID, err) + return nil, status.Errorf(codes.NotFound, "ControllerPublishVolume could not find volume with ID %v: %v", volumeID, err.Error()) } - return nil, status.Errorf(codes.Internal, "ControllerPublishVolume error repairing underspecified volume key: %v", err) + return nil, LoggedError("ControllerPublishVolume error repairing underspecified volume key: ", err) } // Acquires the lock for the volume on that node only, because we need to support the ability @@ -489,20 +484,20 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con _, err = gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find disk %v: %v", volKey.String(), err)) + return nil, status.Errorf(codes.NotFound, "Could not find disk %v: %v", volKey.String(), err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get disk error: %v", err)) + return nil, status.Errorf(codes.Internal, "Unknown get disk error: %v", err.Error()) } instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) if err != nil { - return nil, status.Error(codes.NotFound, fmt.Sprintf("could not split nodeID: %v", err)) + return nil, status.Errorf(codes.NotFound, "could not split nodeID: %v", err.Error()) } instance, err := gceCS.CloudProvider.GetInstanceOrError(ctx, instanceZone, instanceName) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find instance %v: %v", nodeID, err)) + return nil, status.Errorf(codes.NotFound, "Could not find instance %v: %v", nodeID, err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get instance error: %v", err)) + return nil, status.Errorf(codes.Internal, "Unknown get instance error: %v", err.Error()) } readWrite := "READ_WRITE" @@ -512,12 +507,12 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con deviceName, err := common.GetDeviceName(volKey) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("error getting device name: %v", err)) + return nil, status.Errorf(codes.Internal, "error getting device name: %v", err.Error()) } attached, err := diskIsAttachedAndCompatible(deviceName, instance, volumeCapability, readWrite) if err != nil { - return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Disk %v already published to node %v but incompatbile: %v", volKey.Name, nodeID, err)) + return nil, status.Errorf(codes.AlreadyExists, "Disk %v already published to node %v but incompatible: %v", volKey.Name, nodeID, err.Error()) } if attached { // Volume is attached to node. Success! @@ -526,16 +521,16 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con } instanceZone, instanceName, err = common.NodeIDToZoneAndName(nodeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("could not split nodeID: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()) } err = gceCS.CloudProvider.AttachDisk(ctx, project, volKey, readWrite, attachableDiskTypePersistent, instanceZone, instanceName) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Attach error: %v", err)) + return nil, status.Errorf(codes.Internal, "unknown Attach error: %v", err.Error()) } err = gceCS.CloudProvider.WaitForAttach(ctx, project, volKey, instanceZone, instanceName) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown WaitForAttach error: %v", err)) + return nil, status.Errorf(codes.Internal, "unknown WaitForAttach error: %v", err.Error()) } klog.V(4).Infof("ControllerPublishVolume succeeded for disk %v to instance %v", volKey, nodeID) return pubVolResp, nil @@ -577,7 +572,7 @@ func (gceCS *GCEControllerServer) validateControllerUnpublishVolumeRequest(ctx c project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return "", nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerUnpublishVolume Volume ID is invalid: %v", err)) + return "", nil, status.Errorf(codes.InvalidArgument, "ControllerUnpublishVolume Volume ID is invalid: %v", err.Error()) } return project, volKey, nil @@ -595,9 +590,10 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Errorf(codes.NotFound, "ControllerUnpublishVolume could not find volume with ID %v: %v", volumeID, err) + klog.Warningf("Treating volume %v as unpublished because it could not be found", volumeID) + return &csi.ControllerUnpublishVolumeResponse{}, nil } - return nil, status.Errorf(codes.Internal, "ControllerUnpublishVolume error repairing underspecified volume key: %v", err) + return nil, LoggedError("ControllerUnpublishVolume error repairing underspecified volume key: ", err) } // Acquires the lock for the volume on that node only, because we need to support the ability @@ -610,7 +606,7 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("could not split nodeID: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()) } instance, err := gceCS.CloudProvider.GetInstanceOrError(ctx, instanceZone, instanceName) if err != nil { @@ -619,12 +615,12 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C klog.Warningf("Treating volume %v as unpublished because node %v could not be found", volKey.String(), instanceName) return &csi.ControllerUnpublishVolumeResponse{}, nil } - return nil, status.Error(codes.Internal, fmt.Sprintf("error getting instance: %v", err)) + return nil, status.Errorf(codes.Internal, "error getting instance: %v", err.Error()) } deviceName, err := common.GetDeviceName(volKey) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("error getting device name: %v", err)) + return nil, status.Errorf(codes.Internal, "error getting device name: %v", err.Error()) } attached := diskIsAttached(deviceName, instance) @@ -637,7 +633,7 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C err = gceCS.CloudProvider.DetachDisk(ctx, project, deviceName, instanceZone, instanceName) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown detach error: %v", err)) + return nil, LoggedError("unknown detach error: ", err) } klog.V(4).Infof("ControllerUnpublishVolume succeeded for disk %v from node %v", volKey, nodeID) @@ -654,15 +650,15 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context } project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Volume ID is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "Volume ID is invalid: %v", err.Error()) } project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Errorf(codes.NotFound, "ValidateVolumeCapabilities could not find volume with ID %v: %v", volumeID, err) + return nil, status.Errorf(codes.NotFound, "ValidateVolumeCapabilities could not find volume with ID %v: %v", volumeID, err.Error()) } - return nil, status.Errorf(codes.Internal, "ValidateVolumeCapabilities error repairing underspecified volume key: %v", err) + return nil, LoggedError("ValidateVolumeCapabilities error repairing underspecified volume key: ", err) } if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { @@ -673,9 +669,9 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context disk, err := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find disk %v: %v", volKey.Name, err)) + return nil, status.Errorf(codes.NotFound, "Could not find disk %v: %v", volKey.Name, err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get disk error: %v", err)) + return nil, LoggedError("Unknown get disk error: ", err) } // Check Volume Context is Empty @@ -685,16 +681,16 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context // Check volume capabilities supported by PD. These are the same for any PD if err := validateVolumeCapabilities(req.GetVolumeCapabilities()); err != nil { - return generateFailedValidationMessage("VolumeCapabilities not valid: %v", err), nil + return generateFailedValidationMessage("VolumeCapabilities not valid: %v", err.Error()), nil } // Validate the disk parameters match the disk we GET params, err := common.ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.name, gceCS.Driver.extraVolumeLabels) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err) + return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } if err := gce.ValidateDiskParameters(disk, params); err != nil { - return generateFailedValidationMessage("Parameters %v do not match given disk %s: %v", req.GetParameters(), disk.GetName(), err), nil + return generateFailedValidationMessage("Parameters %v do not match given disk %s: %v", req.GetParameters(), disk.GetName(), err.Error()), nil } // Ignore secrets @@ -726,8 +722,8 @@ func generateFailedValidationMessage(format string, a ...interface{}) *csi.Valid func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { // https://cloud.google.com/compute/docs/reference/beta/disks/list if req.MaxEntries < 0 { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf( - "ListVolumes got max entries request %v. GCE only supports values >0", req.MaxEntries)) + return nil, status.Errorf(codes.InvalidArgument, + "ListVolumes got max entries request %v. GCE only supports values >0", req.MaxEntries) } offset := 0 @@ -736,16 +732,16 @@ func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.List diskList, _, err := gceCS.CloudProvider.ListDisks(ctx) if err != nil { if gce.IsGCEInvalidError(err) { - return nil, status.Error(codes.Aborted, fmt.Sprintf("ListVolumes error with invalid request: %v", err)) + return nil, status.Errorf(codes.Aborted, "ListVolumes error with invalid request: %v", err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list disk error: %v", err)) + return nil, LoggedError("Unknown list disk error: ", err) } gceCS.disks = diskList gceCS.seen = map[string]int{} } else { offset, ok = gceCS.seen[req.StartingToken] if !ok { - return nil, status.Error(codes.Aborted, fmt.Sprintf("ListVolumes error with invalid startingToken: %s", req.StartingToken)) + return nil, status.Errorf(codes.Aborted, "ListVolumes error with invalid startingToken: %s", req.StartingToken) } } @@ -807,7 +803,7 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C } project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateSnapshot Volume ID is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "CreateSnapshot Volume ID is invalid: %v", err.Error()) } if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { @@ -819,14 +815,14 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C _, err = gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Error(codes.NotFound, fmt.Sprintf("CreateSnapshot could not find disk %v: %v", volKey.String(), err)) + return nil, status.Errorf(codes.NotFound, "CreateSnapshot could not find disk %v: %v", volKey.String(), err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateSnapshot unknown get disk error: %v", err)) + return nil, LoggedError("CreateSnapshot unknown get disk error: ", err) } snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters(), gceCS.Driver.name) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid snapshot parameters: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "Invalid snapshot parameters: %v", err.Error()) } var snapshot *csi.Snapshot @@ -842,7 +838,7 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C return nil, err } default: - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid snapshot type: %s", snapshotParams.SnapshotType)) + return nil, status.Errorf(codes.InvalidArgument, "Invalid snapshot type: %s", snapshotParams.SnapshotType) } klog.V(4).Infof("CreateSnapshot succeeded for snapshot %s on volume %s", snapshot.SnapshotId, volumeID) @@ -852,7 +848,7 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C func (gceCS *GCEControllerServer) createPDSnapshot(ctx context.Context, project string, volKey *meta.Key, snapshotName string, snapshotParams common.SnapshotParameters) (*csi.Snapshot, error) { volumeID, err := common.KeyToVolumeID(volKey, project) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid volume key: %v", volKey)) + return nil, status.Errorf(codes.InvalidArgument, "Invalid volume key: %v", volKey) } // Check if PD snapshot already exists @@ -860,31 +856,31 @@ func (gceCS *GCEControllerServer) createPDSnapshot(ctx context.Context, project snapshot, err = gceCS.CloudProvider.GetSnapshot(ctx, project, snapshotName) if err != nil { if !gce.IsGCEError(err, "notFound") { - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get snapshot error: %v", err)) + return nil, status.Errorf(codes.Internal, "Unknown get snapshot error: %v", err.Error()) } // If we could not find the snapshot, we create a new one snapshot, err = gceCS.CloudProvider.CreateSnapshot(ctx, project, volKey, snapshotName, snapshotParams) if err != nil { if gce.IsGCEError(err, "notFound") { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find volume with ID %v: %v", volKey.String(), err)) + return nil, status.Errorf(codes.NotFound, "Could not find volume with ID %v: %v", volKey.String(), err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown create snapshot error: %v", err)) + return nil, LoggedError("Unknown create snapshot error: ", err) } } err = gceCS.validateExistingSnapshot(snapshot, volKey) if err != nil { - return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Error in creating snapshot: %v", err)) + return nil, status.Errorf(codes.AlreadyExists, "Error in creating snapshot: %v", err.Error()) } timestamp, err := parseTimestamp(snapshot.CreationTimestamp) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to covert creation timestamp: %v", err)) + return nil, status.Errorf(codes.Internal, "Failed to covert creation timestamp: %v", err.Error()) } ready, err := isCSISnapshotReady(snapshot.Status) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Snapshot had error checking ready status: %v", err)) + return nil, status.Errorf(codes.Internal, "Snapshot had error checking ready status: %v", err.Error()) } return &csi.Snapshot{ @@ -899,7 +895,7 @@ func (gceCS *GCEControllerServer) createPDSnapshot(ctx context.Context, project func (gceCS *GCEControllerServer) createImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*csi.Snapshot, error) { volumeID, err := common.KeyToVolumeID(volKey, project) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid volume key: %v", volKey)) + return nil, status.Errorf(codes.InvalidArgument, "Invalid volume key: %v", volKey) } // Check if image already exists @@ -907,31 +903,31 @@ func (gceCS *GCEControllerServer) createImage(ctx context.Context, project strin image, err = gceCS.CloudProvider.GetImage(ctx, project, imageName) if err != nil { if !gce.IsGCEError(err, "notFound") { - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get image error: %v", err)) + return nil, LoggedError("Unknown get image error: ", err) } // create a new image image, err = gceCS.CloudProvider.CreateImage(ctx, project, volKey, imageName, snapshotParams) if err != nil { if gce.IsGCEError(err, "notFound") { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find volume with ID %v: %v", volKey.String(), err)) + return nil, status.Errorf(codes.NotFound, "Could not find volume with ID %v: %v", volKey.String(), err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown create image error: %v", err)) + return nil, LoggedError("Unknown create image error: ", err) } } err = gceCS.validateExistingImage(image, volKey) if err != nil { - return nil, status.Errorf(codes.AlreadyExists, fmt.Sprintf("Error in creating snapshot: %v", err)) + return nil, status.Errorf(codes.AlreadyExists, "Error in creating snapshot: %v", err.Error()) } timestamp, err := parseTimestamp(image.CreationTimestamp) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to covert creation timestamp: %v", err)) + return nil, status.Errorf(codes.Internal, "Failed to covert creation timestamp: %v", err.Error()) } ready, err := isImageReady(image.Status) if err != nil { - return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to check image status: %v", err)) + return nil, status.Errorf(codes.Internal, "Failed to check image status: %v", err.Error()) } return &csi.Snapshot{ @@ -950,7 +946,7 @@ func (gceCS *GCEControllerServer) validateExistingImage(image *compute.Image, vo _, sourceKey, err := common.VolumeIDToKey(cleanSelfLink(image.SourceDisk)) if err != nil { - return fmt.Errorf("fail to get source disk key %s, %v", image.SourceDisk, err) + return fmt.Errorf("fail to get source disk key %s, %w", image.SourceDisk, err) } if sourceKey.String() != volKey.String() { @@ -1001,7 +997,7 @@ func (gceCS *GCEControllerServer) validateExistingSnapshot(snapshot *compute.Sna _, sourceKey, err := common.VolumeIDToKey(cleanSelfLink(snapshot.SourceDisk)) if err != nil { - return fmt.Errorf("fail to get source disk key %s, %v", snapshot.SourceDisk, err) + return fmt.Errorf("fail to get source disk key %s, %w", snapshot.SourceDisk, err) } if sourceKey.String() != volKey.String() { @@ -1045,15 +1041,15 @@ func (gceCS *GCEControllerServer) DeleteSnapshot(ctx context.Context, req *csi.D case common.DiskSnapshotType: err = gceCS.CloudProvider.DeleteSnapshot(ctx, project, key) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete snapshot error: %v", err)) + return nil, LoggedError("unknown Delete snapshot error: ", err) } case common.DiskImageType: err = gceCS.CloudProvider.DeleteImage(ctx, project, key) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete image error: %v", err)) + return nil, LoggedError("unknown Delete image error: ", err) } default: - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("unknown snapshot type %s", snapshotType)) + return nil, status.Errorf(codes.InvalidArgument, "unknown snapshot type %s", snapshotType) } return &csi.DeleteSnapshotResponse{}, nil @@ -1068,8 +1064,8 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li // case 2: no SnapshotId is set, so we return all the snapshots that satify the reqeust. var maxEntries int = int(req.MaxEntries) if maxEntries < 0 { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf( - "ListSnapshots got max entries request %v. GCE only supports values >0", maxEntries)) + return nil, status.Errorf(codes.InvalidArgument, + "ListSnapshots got max entries request %v. GCE only supports values >0", maxEntries) } var offset int @@ -1080,16 +1076,16 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li snapshotList, err := gceCS.getSnapshots(ctx, req) if err != nil { if gce.IsGCEInvalidError(err) { - return nil, status.Error(codes.Aborted, fmt.Sprintf("ListSnapshots error with invalid request: %v", err)) + return nil, status.Errorf(codes.Aborted, "ListSnapshots error with invalid request: %v", err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshots error: %v", err)) + return nil, LoggedError("Unknown list snapshots error: ", err) } gceCS.snapshots = snapshotList gceCS.snapshotTokens = map[string]int{} } else { offset, ok = gceCS.snapshotTokens[req.StartingToken] if !ok { - return nil, status.Error(codes.Aborted, fmt.Sprintf("ListSnapshots error with invalid startingToken: %s", req.StartingToken)) + return nil, status.Errorf(codes.Aborted, "ListSnapshots error with invalid startingToken: %s", req.StartingToken) } } @@ -1118,25 +1114,25 @@ func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, re capacityRange := req.GetCapacityRange() reqBytes, err := getRequestCapacity(capacityRange) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume capacity range is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "ControllerExpandVolume capacity range is invalid: %v", err.Error()) } project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume Volume ID is invalid: %v", err)) + return nil, status.Errorf(codes.InvalidArgument, "ControllerExpandVolume Volume ID is invalid: %v", err.Error()) } project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { - return nil, status.Errorf(codes.NotFound, "ControllerExpandVolume could not find volume with ID %v: %v", volumeID, err) + return nil, status.Errorf(codes.NotFound, "ControllerExpandVolume could not find volume with ID %v: %v", volumeID, err.Error()) } - return nil, status.Errorf(codes.Internal, "ControllerExpandVolume error repairing underspecified volume key: %v", err) + return nil, LoggedError("ControllerExpandVolume error repairing underspecified volume key: ", err) } resizedGb, err := gceCS.CloudProvider.ResizeDisk(ctx, project, volKey, reqBytes) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume failed to resize disk: %v", err)) + return nil, LoggedError("ControllerExpandVolume failed to resize disk: ", err) } klog.V(4).Infof("ControllerExpandVolume succeeded for disk %v to size %v", volKey, resizedGb) @@ -1157,17 +1153,17 @@ func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.Lis snapshots, _, err = gceCS.CloudProvider.ListSnapshots(ctx, filter) if err != nil { if gce.IsGCEError(err, "invalid") { - return nil, status.Error(codes.Aborted, fmt.Sprintf("Invalid error: %v", err)) + return nil, status.Errorf(codes.Aborted, "Invalid error: %v", err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshot error: %v", err)) + return nil, LoggedError("Unknown list snapshot error: ", err) } images, _, err = gceCS.CloudProvider.ListImages(ctx, filter) if err != nil { if gce.IsGCEError(err, "invalid") { - return nil, status.Error(codes.Aborted, fmt.Sprintf("Invalid error: %v", err)) + return nil, status.Errorf(codes.Aborted, "Invalid error: %v", err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list image error: %v", err)) + return nil, LoggedError("Unknown list image error: ", err) } entries := []*csi.ListSnapshotsResponse_Entry{} @@ -1175,7 +1171,7 @@ func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.Lis for _, snapshot := range snapshots { entry, err := generateDiskSnapshotEntry(snapshot) if err != nil { - return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) + return nil, fmt.Errorf("failed to generate snapshot entry: %w", err) } entries = append(entries, entry) } @@ -1183,7 +1179,7 @@ func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.Lis for _, image := range images { entry, err := generateDiskImageEntry(image) if err != nil { - return nil, fmt.Errorf("failed to generate image entry: %v", err) + return nil, fmt.Errorf("failed to generate image entry: %w", err) } entries = append(entries, entry) } @@ -1208,11 +1204,11 @@ func (gceCS *GCEControllerServer) getSnapshotByID(ctx context.Context, snapshotI // return empty list if no snapshot is found return &csi.ListSnapshotsResponse{}, nil } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshot error: %v", err)) + return nil, LoggedError("Unknown list snapshot error: ", err) } e, err := generateDiskSnapshotEntry(snapshot) if err != nil { - return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) + return nil, fmt.Errorf("failed to generate snapshot entry: %w", err) } entries = []*csi.ListSnapshotsResponse_Entry{e} case common.DiskImageType: @@ -1225,7 +1221,7 @@ func (gceCS *GCEControllerServer) getSnapshotByID(ctx context.Context, snapshotI } e, err := generateImageEntry(image) if err != nil { - return nil, fmt.Errorf("failed to generate image entry: %v", err) + return nil, fmt.Errorf("failed to generate image entry: %w", err) } entries = []*csi.ListSnapshotsResponse_Entry{e} } @@ -1242,7 +1238,7 @@ func generateDiskSnapshotEntry(snapshot *compute.Snapshot) (*csi.ListSnapshotsRe tp := timestamppb.New(t) if err := tp.CheckValid(); err != nil { - return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + return nil, fmt.Errorf("Failed to covert creation timestamp: %w", err) } // We ignore the error intentionally here since we are just listing snapshots @@ -1267,7 +1263,7 @@ func generateDiskImageEntry(image *compute.Image) (*csi.ListSnapshotsResponse_En tp := timestamppb.New(t) if err := tp.CheckValid(); err != nil { - return nil, fmt.Errorf("failed to covert creation timestamp: %v", err) + return nil, fmt.Errorf("failed to covert creation timestamp: %w", err) } ready, _ := isImageReady(image.Status) @@ -1287,7 +1283,7 @@ func generateDiskImageEntry(image *compute.Image) (*csi.ListSnapshotsResponse_En func generateImageEntry(image *compute.Image) (*csi.ListSnapshotsResponse_Entry, error) { timestamp, err := parseTimestamp(image.CreationTimestamp) if err != nil { - return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + return nil, fmt.Errorf("Failed to covert creation timestamp: %w", err) } // ignore the error intentionally here since we are just listing images @@ -1365,11 +1361,11 @@ func diskIsAttachedAndCompatible(deviceName string, instance *compute.Instance, func pickZonesFromTopology(top *csi.TopologyRequirement, numZones int) ([]string, error) { reqZones, err := getZonesFromTopology(top.GetRequisite()) if err != nil { - return nil, fmt.Errorf("could not get zones from requisite topology: %v", err) + return nil, fmt.Errorf("could not get zones from requisite topology: %w", err) } prefZones, err := getZonesFromTopology(top.GetPreferred()) if err != nil { - return nil, fmt.Errorf("could not get zones from preferred topology: %v", err) + return nil, fmt.Errorf("could not get zones from preferred topology: %w", err) } if numZones <= len(prefZones) { @@ -1407,7 +1403,7 @@ func getZonesFromTopology(topList []*csi.Topology) ([]string, error) { // GCE PD cloud provider Create has no restrictions so just create in top preferred zone zone, err := getZoneFromSegment(top.GetSegments()) if err != nil { - return nil, fmt.Errorf("could not get zone from preferred topology: %v", err) + return nil, fmt.Errorf("could not get zone from preferred topology: %w", err) } zones = append(zones, zone) } @@ -1436,12 +1432,12 @@ func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.Topolog if top != nil { zones, err = pickZonesFromTopology(top, numZones) if err != nil { - return nil, fmt.Errorf("failed to pick zones from topology: %v", err) + return nil, fmt.Errorf("failed to pick zones from topology: %w", err) } } else { zones, err = getDefaultZonesInRegion(ctx, gceCS, []string{gceCS.CloudProvider.GetDefaultZone()}, numZones) if err != nil { - return nil, fmt.Errorf("failed to get default %v zones in region: %v", numZones, err) + return nil, fmt.Errorf("failed to get default %v zones in region: %w", numZones, err) } klog.Warningf("No zones have been specified in either topology or params, picking default zone: %v", zones) @@ -1452,12 +1448,12 @@ func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.Topolog func getDefaultZonesInRegion(ctx context.Context, gceCS *GCEControllerServer, existingZones []string, numZones int) ([]string, error) { region, err := common.GetRegionFromZones(existingZones) if err != nil { - return nil, fmt.Errorf("failed to get region from zones: %v", err) + return nil, fmt.Errorf("failed to get region from zones: %w", err) } needToGet := numZones - len(existingZones) totZones, err := gceCS.CloudProvider.ListZones(ctx, region) if err != nil { - return nil, fmt.Errorf("failed to list zones from cloud provider: %v", err) + return nil, fmt.Errorf("failed to list zones from cloud provider: %w", err) } remainingZones := sets.NewString(totZones...).Difference(sets.NewString(existingZones...)) l := remainingZones.List() @@ -1526,16 +1522,15 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea } func cleanSelfLink(selfLink string) string { - temp := strings.TrimPrefix(selfLink, gce.GCEComputeAPIEndpoint) - temp = strings.TrimPrefix(temp, gce.GCEComputeBetaAPIEndpoint) - return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint) + r, _ := regexp.Compile("https:\\/\\/www.*apis.com\\/.*(v1|beta|alpha)\\/") + return r.ReplaceAllString(selfLink, "") } func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) (*gce.CloudDisk, error) { project := cloudProvider.GetDefaultProject() region, err := common.GetRegionFromZones(zones) if err != nil { - return nil, fmt.Errorf("failed to get region from zones: %v", err) + return nil, fmt.Errorf("failed to get region from zones: %w", err) } fullyQualifiedReplicaZones := []string{} @@ -1546,7 +1541,7 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name err = cloudProvider.InsertDisk(ctx, project, meta.RegionalKey(name, region), params, capBytes, capacityRange, fullyQualifiedReplicaZones, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { - return nil, fmt.Errorf("failed to insert regional disk: %v", err) + return nil, fmt.Errorf("failed to insert regional disk: %w", err) } gceAPIVersion := gce.GCEAPIVersionV1 @@ -1556,7 +1551,7 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name disk, err := cloudProvider.GetDisk(ctx, project, meta.RegionalKey(name, region), gceAPIVersion) if err != nil { - return nil, fmt.Errorf("failed to get disk after creating regional disk: %v", err) + return nil, fmt.Errorf("failed to get disk after creating regional disk: %w", err) } return disk, nil } @@ -1569,7 +1564,7 @@ func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, nam diskZone := zones[0] err := cloudProvider.InsertDisk(ctx, project, meta.ZonalKey(name, diskZone), params, capBytes, capacityRange, nil, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { - return nil, fmt.Errorf("failed to insert zonal disk: %v", err) + return nil, fmt.Errorf("failed to insert zonal disk: %w", err) } gceAPIVersion := gce.GCEAPIVersionV1 @@ -1597,8 +1592,8 @@ func pickRandAndConsecutive(slice []string, n int) ([]string, error) { return ret, nil } -func newCsiErrorBackoff() *csiErrorBackoff { - return &csiErrorBackoff{flowcontrol.NewBackOff(errorBackoffInitialDuration, errorBackoffMaxDuration)} +func newCsiErrorBackoff(initialDuration, errorBackoffMaxDuration time.Duration) *csiErrorBackoff { + return &csiErrorBackoff{flowcontrol.NewBackOff(initialDuration, errorBackoffMaxDuration)} } func (_ *csiErrorBackoff) backoffId(nodeId, volumeId string) csiErrorBackoffId { diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index 2963b80e..6ac139d2 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -60,12 +60,18 @@ var ( Segments: map[string]string{common.TopologyKeyZone: zone}, }, } - testVolumeID = fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, name) + + testVolumeID = fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, name) + underspecifiedVolumeID = fmt.Sprintf("projects/UNSPECIFIED/zones/UNSPECIFIED/disks/%s", name) + region, _ = common.GetRegionFromZones([]string{zone}) testRegionalID = fmt.Sprintf("projects/%s/regions/%s/disks/%s", project, region, name) testSnapshotID = fmt.Sprintf("projects/%s/global/snapshots/%s", project, name) testImageID = fmt.Sprintf("projects/%s/global/images/%s", project, name) testNodeID = fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone, node) + + errorBackoffInitialDuration = 200 * time.Millisecond + errorBackoffMaxDuration = 5 * time.Minute ) func TestCreateSnapshotArguments(t *testing.T) { @@ -74,7 +80,7 @@ func TestCreateSnapshotArguments(t *testing.T) { if err := tp.CheckValid(); err != nil { t.Fatalf("Unable to conver time to timestamp: %v", err) } - // Define test cases + testCases := []struct { name string req *csi.CreateSnapshotRequest @@ -183,7 +189,6 @@ func TestCreateSnapshotArguments(t *testing.T) { // Start Test resp, err := gceDriver.cs.CreateSnapshot(context.Background(), tc.req) - //check response if err != nil { serverError, ok := status.FromError(err) if !ok { @@ -211,6 +216,7 @@ func TestCreateSnapshotArguments(t *testing.T) { } } } + func TestDeleteSnapshot(t *testing.T) { testCases := []struct { name string @@ -249,7 +255,6 @@ func TestDeleteSnapshot(t *testing.T) { gceDriver := initGCEDriver(t, nil) _, err := gceDriver.cs.DeleteSnapshot(context.Background(), tc.req) - //check response if err != nil { serverError, ok := status.FromError(err) t.Logf("get server error %v", serverError) @@ -264,12 +269,10 @@ func TestDeleteSnapshot(t *testing.T) { if tc.expErrCode != codes.OK { t.Fatalf("Expected error: %v, got no error", tc.expErrCode) } - } } func TestListSnapshotsArguments(t *testing.T) { - // Define test cases testCases := []struct { name string req *csi.ListSnapshotsRequest @@ -372,7 +375,6 @@ func TestListSnapshotsArguments(t *testing.T) { // Start Test resp, err := gceDriver.cs.ListSnapshots(context.Background(), tc.req) - //check response if err != nil { serverError, ok := status.FromError(err) if !ok { @@ -389,7 +391,6 @@ func TestListSnapshotsArguments(t *testing.T) { // Make sure responses match snapshots := resp.GetEntries() - //expectsnapshots := expSnapshot.GetEntries() if (snapshots == nil || len(snapshots) == 0) && tc.numSnapshots == 0 { continue } @@ -406,7 +407,6 @@ func TestListSnapshotsArguments(t *testing.T) { } func TestCreateVolumeArguments(t *testing.T) { - // Define test cases testCases := []struct { name string req *csi.CreateVolumeRequest @@ -792,6 +792,56 @@ func TestCreateVolumeArguments(t *testing.T) { }, expErrCode: codes.InvalidArgument, }, + { + name: "success with provisionedIops parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-iops-on-create": "10000"}, + }, + expVol: &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: testVolumeID, + VolumeContext: nil, + AccessibleTopology: stdTopology, + }, + }, + { + name: "fail with malformed provisionedIops parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-iops-on-create": "dsfo3"}, + }, + expErrCode: codes.InvalidArgument, + }, + { + name: "success with provisionedThroughput parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-throughput-on-create": "10000"}, + }, + expVol: &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: testVolumeID, + VolumeContext: nil, + AccessibleTopology: stdTopology, + }, + }, + { + name: "fail with malformed provisionedThroughput parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-throughput-on-create": "dsfo3"}, + }, + expErrCode: codes.InvalidArgument, + }, } // Run test cases @@ -802,7 +852,6 @@ func TestCreateVolumeArguments(t *testing.T) { // Start Test resp, err := gceDriver.cs.CreateVolume(context.Background(), tc.req) - //check response if err != nil { serverError, ok := status.FromError(err) if !ok { @@ -964,7 +1013,6 @@ func TestListVolumeArgs(t *testing.T) { } func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { - // Define test cases testCases := []struct { name string project string @@ -1047,7 +1095,6 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { } resp, err := gceDriver.cs.CreateVolume(context.Background(), req) - //check response if err != nil { serverError, ok := status.FromError(err) if !ok { @@ -1095,7 +1142,6 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { }, } - // Define test cases testCases := []struct { name string volumeOnCloud bool @@ -2107,7 +2153,6 @@ func TestCreateVolumeDiskReady(t *testing.T) { }, } - // Run test cases for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { fcp, err := gce.CreateFakeCloudProvider(project, zone, nil) @@ -2119,9 +2164,9 @@ func TestCreateVolumeDiskReady(t *testing.T) { fcp.UpdateDiskStatus(tc.diskStatus) // Setup new driver each time so no interference gceDriver := initGCEDriverWithCloudProvider(t, fcp) + // Start Test resp, err := gceDriver.cs.CreateVolume(context.Background(), tc.req) - //check response if err != nil { serverError, ok := status.FromError(err) if !ok { @@ -2144,279 +2189,375 @@ func TestCreateVolumeDiskReady(t *testing.T) { } } -type backoffTesterConfig struct { - mockMissingInstance bool // used by the backoff tester to mock a missing instance scenario +type backoffDriverConfig struct { + mockMissingInstance bool + mockMissingDisk bool + + clock *clock.FakeClock + attachedDisks []*compute.AttachedDisk + readyToExecute chan chan gce.Signal } -func newFakeCsiErrorBackoff(tc *clock.FakeClock) *csiErrorBackoff { - return &csiErrorBackoff{flowcontrol.NewFakeBackOff(errorBackoffInitialDuration, errorBackoffMaxDuration, tc)} +func newFakeCSIErrorBackoff(tc *clock.FakeClock) *csiErrorBackoff { + backoff := flowcontrol.NewFakeBackOff(errorBackoffInitialDuration, errorBackoffMaxDuration, tc) + return &csiErrorBackoff{backoff} } func TestControllerUnpublishBackoff(t *testing.T) { - backoffTesterForUnpublish(t, &backoffTesterConfig{}) -} + for desc, tc := range map[string]struct { + config *backoffDriverConfig + }{ + "success": {}, + "missing instance": { + config: &backoffDriverConfig{ + mockMissingInstance: true, + }, + }, + } { + t.Run(desc, func(t *testing.T) { + if tc.config == nil { + tc.config = &backoffDriverConfig{} + } + tc.config.clock = clock.NewFakeClock(time.Now()) + tc.config.attachedDisks = []*compute.AttachedDisk{{DeviceName: name}} + tc.config.readyToExecute = make(chan chan gce.Signal) + driver := backoffDriver(t, tc.config) + + backoffId := driver.cs.errorBackoff.backoffId(testNodeID, testVolumeID) + step := 1 * time.Millisecond + + runUnpublishRequest := func(req *csi.ControllerUnpublishVolumeRequest, reportError bool) error { + response := make(chan error) + go func() { + _, err := driver.cs.ControllerUnpublishVolume(context.Background(), req) + response <- err + }() + go func() { + executeChan := <-tc.config.readyToExecute + executeChan <- gce.Signal{ReportError: reportError} + }() + return <-response + } -func TestControllerUnpublishBackoffMissingInstance(t *testing.T) { - backoffTesterForUnpublish(t, &backoffTesterConfig{ - mockMissingInstance: true, - }) -} + // Mock an active backoff condition on the node. + driver.cs.errorBackoff.next(backoffId) -func backoffTesterForUnpublish(t *testing.T, config *backoffTesterConfig) { - readyToExecute := make(chan chan gce.Signal) - cloudDisks := []*gce.CloudDisk{ - createZonalCloudDisk(name), - } - fcp, err := gce.CreateFakeCloudProvider(project, zone, cloudDisks) - if err != nil { - t.Fatalf("Failed to create fake cloud provider: %v", err) - } - fcpBlocking := &gce.FakeBlockingCloudProvider{ - FakeCloudProvider: fcp, - ReadyToExecute: readyToExecute, - } - instance := &compute.Instance{ - Name: node, - Disks: []*compute.AttachedDisk{ - {DeviceName: name}, // mock attached disks - }, - } - if !config.mockMissingInstance { - fcp.InsertInstance(instance, zone, node) - } + tc.config.clock.Step(step) + // A requst for a a different volume should succeed. This volume is not + // mounted on the node, so no GCE call will be made (ie, runUnpublishRequest + // doesn't need to be called, the request can be called directly). + differentUnpubReq := &csi.ControllerUnpublishVolumeRequest{ + VolumeId: testVolumeID + "-different", + NodeId: testNodeID, + } + if _, err := driver.cs.ControllerUnpublishVolume(context.Background(), differentUnpubReq); err != nil { + t.Errorf("expected no error on different unpublish, got %v", err) + } - driver := GetGCEDriver() - tc := clock.NewFakeClock(time.Now()) - driver.cs = &GCEControllerServer{ - Driver: driver, - CloudProvider: fcpBlocking, - seen: map[string]int{}, - volumeLocks: common.NewVolumeLocks(), - errorBackoff: newFakeCsiErrorBackoff(tc), + unpubreq := &csi.ControllerUnpublishVolumeRequest{ + VolumeId: testVolumeID, + NodeId: testNodeID, + } + // For the first 199 ms, the backoff condition is true. All controller publish request will be denied with 'Unavailable' error code. + for i := 0; i < 199; i++ { + var err error + _, err = driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) + if !isUnavailableError(err) { + t.Errorf("unexpected error %v", err) + } + tc.config.clock.Step(step) + } + + // At the 200th millisecond, the backoff condition is no longer true. The driver should return a success code, and the backoff condition should be cleared. + if tc.config.mockMissingInstance { + _, err := driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) + if err != nil { + t.Errorf("unexpected error %v", err) + } + // Driver is expected to remove the node key from the backoff map. + t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) + if t1 != 0 { + t.Error("unexpected delay") + } + return + } + + // Mock an error + if err := runUnpublishRequest(unpubreq, true); err == nil { + t.Errorf("expected error") + } + + // The above failure should cause driver to call Backoff.Next() again and a backoff duration of 400 ms duration is set starting at the 200th millisecond. + // For the 200-599 ms, the backoff condition is true, and new controller publish requests will be deined. + for i := 0; i < 399; i++ { + tc.config.clock.Step(step) + var err error + _, err = driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) + if !isUnavailableError(err) { + t.Errorf("unexpected error %v", err) + } + } + + // Mock clock tick for the 600th millisecond. So backoff condition is no longer true. + tc.config.clock.Step(step) + // Now mock a successful ControllerUnpublish request, where DetachDisk call succeeds. + if err := runUnpublishRequest(unpubreq, false); err != nil { + t.Errorf("unexpected error") + } + + // Driver is expected to remove the node key from the backoff map. + t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) + if t1 != 0 { + t.Error("unexpected delay") + } + }) } +} - backoffId := driver.cs.errorBackoff.backoffId(testNodeID, testVolumeID) - step := 1 * time.Millisecond +func TestControllerUnpublishSucceedsIfNotFound(t *testing.T) { + for desc, tc := range map[string]struct { + volumeID string + }{ + "full volume ID": { + volumeID: testVolumeID, + }, + "underspecified volume ID": { + volumeID: underspecifiedVolumeID, + }, + } { + t.Run(desc, func(t *testing.T) { + driver := backoffDriver(t, &backoffDriverConfig{ + mockMissingDisk: true, + clock: clock.NewFakeClock(time.Now()), + attachedDisks: []*compute.AttachedDisk{{DeviceName: name}}, + }) + + req := &csi.ControllerUnpublishVolumeRequest{ + VolumeId: tc.volumeID, + NodeId: testNodeID, + } - runUnpublishRequest := func(req *csi.ControllerUnpublishVolumeRequest, reportError bool) error { - response := make(chan error) - go func() { _, err := driver.cs.ControllerUnpublishVolume(context.Background(), req) - response <- err - }() - go func() { - executeChan := <-readyToExecute - executeChan <- gce.Signal{ReportError: reportError} - }() - return <-response + if err != nil { + t.Errorf("unexpected error: %v", err) + } + }) } +} - // Mock an active backoff condition on the node. - driver.cs.errorBackoff.next(backoffId) +func TestControllerPublishBackoff(t *testing.T) { + for desc, tc := range map[string]struct { + config *backoffDriverConfig + }{ + "success": {}, + "missing instance": { + config: &backoffDriverConfig{ + mockMissingInstance: true, + }, + }, + } { + t.Run(desc, func(t *testing.T) { + if tc.config == nil { + tc.config = &backoffDriverConfig{} + } + tc.config.clock = clock.NewFakeClock(time.Now()) + tc.config.readyToExecute = make(chan chan gce.Signal) + driver := backoffDriver(t, tc.config) + + backoffId := driver.cs.errorBackoff.backoffId(testNodeID, testVolumeID) + step := 1 * time.Millisecond + + // Mock an active backoff condition on the node. + driver.cs.errorBackoff.next(backoffId) + + // A detach request for a different disk should succeed. As this disk is not + // on the instance, the detach will succeed without calling the gce detach + // disk api so we don't have to go through the blocking cloud provider and + // and make the request directly. + if _, err := driver.cs.ControllerUnpublishVolume(context.Background(), &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeID + "different", NodeId: testNodeID}); err != nil { + t.Errorf("expected no error on different unpublish, got %v", err) + } - tc.Step(step) - // A requst for a a different volume should succeed. This volume is not - // mounted on the node, so no GCE call will be made (ie, runUnpublishRequest - // doesn't need to be called, the request can be called directly). - differentUnpubReq := &csi.ControllerUnpublishVolumeRequest{ - VolumeId: testVolumeID + "-different", - NodeId: testNodeID, - } - if _, err := driver.cs.ControllerUnpublishVolume(context.Background(), differentUnpubReq); err != nil { - t.Errorf("expected no error on different unpublish, got %v", err) - } + pubreq := &csi.ControllerPublishVolumeRequest{ + VolumeId: testVolumeID, + NodeId: testNodeID, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + } + // For the first 199 ms, the backoff condition is true. All controller publish request will be denied with 'Unavailable' error code. + for i := 0; i < 199; i++ { + tc.config.clock.Step(step) + var err error + _, err = driver.cs.ControllerPublishVolume(context.Background(), pubreq) + if !isUnavailableError(err) { + t.Errorf("unexpected error %v", err) + } + } - unpubreq := &csi.ControllerUnpublishVolumeRequest{ - VolumeId: testVolumeID, - NodeId: testNodeID, - } - // For the first 199 ms, the backoff condition is true. All controller publish request will be denied with 'Unavailable' error code. - for i := 0; i < 199; i++ { - var err error - _, err = driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) - if !isUnavailableError(err) { - t.Errorf("unexpected error %v", err) - } - tc.Step(step) - } + // Mock clock tick for the 200th millisecond. So backoff condition is no longer true. + tc.config.clock.Step(step) + runPublishRequest := func(req *csi.ControllerPublishVolumeRequest, reportError bool) error { + response := make(chan error) + go func() { + _, err := driver.cs.ControllerPublishVolume(context.Background(), req) + response <- err + }() + go func() { + executeChan := <-tc.config.readyToExecute + executeChan <- gce.Signal{ReportError: reportError} + }() + return <-response + } - // At the 200th millisecond, the backoff condition is no longer true. The driver should return a success code, and the backoff condition should be cleared. - if config.mockMissingInstance { - _, err = driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) - if err != nil { - t.Errorf("unexpected error %v", err) - } - // Driver is expected to remove the node key from the backoff map. - t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) - if t1 != 0 { - t.Error("unexpected delay") - } - return - } + // For a missing instance the driver should return error code, and the backoff condition should be set. + if tc.config.mockMissingInstance { + _, err := driver.cs.ControllerPublishVolume(context.Background(), pubreq) + if err == nil { + t.Errorf("unexpected error %v", err) + } - // mock an error - if err := runUnpublishRequest(unpubreq, true); err == nil { - t.Errorf("expected error") - } + t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) + if t1 == 0 { + t.Error("expected delay, got none") + } + return + } - // The above failure should cause driver to call Backoff.Next() again and a backoff duration of 400 ms duration is set starting at the 200th millisecond. - // For the 200-599 ms, the backoff condition is true, and new controller publish requests will be deined. - for i := 0; i < 399; i++ { - tc.Step(step) - var err error - _, err = driver.cs.ControllerUnpublishVolume(context.Background(), unpubreq) - if !isUnavailableError(err) { - t.Errorf("unexpected error %v", err) - } - } + // Mock an error + if err := runPublishRequest(pubreq, true); err == nil { + t.Errorf("expected error") + } - // Mock clock tick for the 600th millisecond. So backoff condition is no longer true. - tc.Step(step) - // Now mock a successful ControllerUnpublish request, where DetachDisk call succeeds. - if err := runUnpublishRequest(unpubreq, false); err != nil { - t.Errorf("unexpected error") - } + // The above failure should cause driver to call Backoff.Next() again and a backoff duration of 400 ms duration is set starting at the 200th millisecond. + // For the 200-599 ms, the backoff condition is true, and new controller publish requests will be deined. + for i := 0; i < 399; i++ { + tc.config.clock.Step(step) + var err error + _, err = driver.cs.ControllerPublishVolume(context.Background(), pubreq) + if !isUnavailableError(err) { + t.Errorf("unexpected error %v", err) + } + } + + // Mock clock tick for the 600th millisecond. So backoff condition is no longer true. + tc.config.clock.Step(step) + // Now mock a successful ControllerUnpublish request, where DetachDisk call succeeds. + if err := runPublishRequest(pubreq, false); err != nil { + t.Errorf("unexpected error") + } - // Driver is expected to remove the node key from the backoff map. - t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) - if t1 != 0 { - t.Error("unexpected delay") + // Driver is expected to remove the node key from the backoff map. + t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) + if t1 != 0 { + t.Error("unexpected delay") + } + }) } } -func TestControllerPublishBackoff(t *testing.T) { - backoffTesterForPublish(t, &backoffTesterConfig{}) -} +func TestCleanSelfLink(t *testing.T) { + testCases := []struct { + name string + in string + want string + }{ + { + name: "v1 full standard w/ endpoint prefix", + in: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "beta full standard w/ endpoint prefix", + in: "https://www.googleapis.com/compute/beta/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "alpha full standard w/ endpoint prefix", + in: "https://www.googleapis.com/compute/alpha/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "no prefix", + in: "projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, -func TestControllerPublishBackoffMissingInstance(t *testing.T) { - backoffTesterForPublish(t, &backoffTesterConfig{ - mockMissingInstance: true, - }) + { + name: "no prefix + project omitted", + in: "zones/zone/disks/disk", + want: "zones/zone/disks/disk", + }, + { + name: "Compute prefix, google api", + in: "https://www.compute.googleapis.com/compute/v1/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "Compute prefix, partner api", + in: "https://www.compute.PARTNERapis.com/compute/v1/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "Partner beta api", + in: "https://www.PARTNERapis.com/compute/beta/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + { + name: "Partner alpha api", + in: "https://www.partnerapis.com/compute/alpha/projects/project/zones/zone/disks/disk", + want: "projects/project/zones/zone/disks/disk", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := cleanSelfLink(tc.in) + if got != tc.want { + t.Errorf("Expected cleaned self link: %v, got: %v", tc.want, got) + } + }) + } } -func backoffTesterForPublish(t *testing.T, config *backoffTesterConfig) { - readyToExecute := make(chan chan gce.Signal) - cloudDisks := []*gce.CloudDisk{ - createZonalCloudDisk(name), +func backoffDriver(t *testing.T, config *backoffDriverConfig) *GCEDriver { + var cloudDisks []*gce.CloudDisk + if !config.mockMissingDisk { + cloudDisks = append(cloudDisks, createZonalCloudDisk(name)) } fcp, err := gce.CreateFakeCloudProvider(project, zone, cloudDisks) if err != nil { t.Fatalf("Failed to create fake cloud provider: %v", err) } - fcpBlocking := &gce.FakeBlockingCloudProvider{ - FakeCloudProvider: fcp, - ReadyToExecute: readyToExecute, - } instance := &compute.Instance{ Name: node, - Disks: []*compute.AttachedDisk{}, + Disks: config.attachedDisks, } if !config.mockMissingInstance { fcp.InsertInstance(instance, zone, node) } driver := GetGCEDriver() - tc := clock.NewFakeClock(time.Now()) driver.cs = &GCEControllerServer{ - Driver: driver, - CloudProvider: fcpBlocking, - seen: map[string]int{}, - volumeLocks: common.NewVolumeLocks(), - errorBackoff: newFakeCsiErrorBackoff(tc), - } - - backoffId := driver.cs.errorBackoff.backoffId(testNodeID, testVolumeID) - step := 1 * time.Millisecond - // Mock an active backoff condition on the node. - driver.cs.errorBackoff.next(backoffId) - - // A detach request for a different disk should succeed. As this disk is not - // on the instance, the detach will succeed without calling the gce detach - // disk api so we don't have to go through the blocking cloud provider and - // and make the request directly. - if _, err := driver.cs.ControllerUnpublishVolume(context.Background(), &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeID + "different", NodeId: testNodeID}); err != nil { - t.Errorf("expected no error on different unpublish, got %v", err) + Driver: driver, + seen: map[string]int{}, + volumeLocks: common.NewVolumeLocks(), + errorBackoff: newFakeCSIErrorBackoff(config.clock), } - pubreq := &csi.ControllerPublishVolumeRequest{ - VolumeId: testVolumeID, - NodeId: testNodeID, - VolumeCapability: &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - }, - } - // For the first 199 ms, the backoff condition is true. All controller publish request will be denied with 'Unavailable' error code. - for i := 0; i < 199; i++ { - tc.Step(step) - var err error - _, err = driver.cs.ControllerPublishVolume(context.Background(), pubreq) - if !isUnavailableError(err) { - t.Errorf("unexpected error %v", err) - } - } - - // Mock clock tick for the 200th millisecond. So backoff condition is no longer true. - tc.Step(step) - runPublishRequest := func(req *csi.ControllerPublishVolumeRequest, reportError bool) error { - response := make(chan error) - go func() { - _, err := driver.cs.ControllerPublishVolume(context.Background(), req) - response <- err - }() - go func() { - executeChan := <-readyToExecute - executeChan <- gce.Signal{ReportError: reportError} - }() - return <-response - } - - // For a missing instance the driver should return error code, and the backoff condition should be set. - if config.mockMissingInstance { - _, err = driver.cs.ControllerPublishVolume(context.Background(), pubreq) - if err == nil { - t.Errorf("unexpected error %v", err) - } - - t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) - if t1 == 0 { - t.Error("expected delay, got none") + driver.cs.CloudProvider = fcp + if config.readyToExecute != nil { + driver.cs.CloudProvider = &gce.FakeBlockingCloudProvider{ + FakeCloudProvider: fcp, + ReadyToExecute: config.readyToExecute, } - return - } - - // mock an error - if err := runPublishRequest(pubreq, true); err == nil { - t.Errorf("expected error") - } - - // The above failure should cause driver to call Backoff.Next() again and a backoff duration of 400 ms duration is set starting at the 200th millisecond. - // For the 200-599 ms, the backoff condition is true, and new controller publish requests will be deined. - for i := 0; i < 399; i++ { - tc.Step(step) - var err error - _, err = driver.cs.ControllerPublishVolume(context.Background(), pubreq) - if !isUnavailableError(err) { - t.Errorf("unexpected error %v", err) - } - } - - // Mock clock tick for the 600th millisecond. So backoff condition is no longer true. - tc.Step(step) - // Now mock a successful ControllerUnpublish request, where DetachDisk call succeeds. - if err := runPublishRequest(pubreq, false); err != nil { - t.Errorf("unexpected error") - } - - // Driver is expected to remove the node key from the backoff map. - t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) - if t1 != 0 { - t.Error("unexpected delay") } + return driver } func isUnavailableError(err error) bool { diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index 3533247c..251bc942 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -34,18 +34,22 @@ package gceGCEDriver import ( "fmt" + "time" csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/mount-utils" common "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata" mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" ) +var maxLogChar int + type GCEDriver struct { name string vendorVersion string @@ -155,7 +159,7 @@ func NewIdentityServer(gceDriver *GCEDriver) *GCEIdentityServer { } } -func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils mountmanager.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, mapper cryptMapper, evalSymLinks func(string) (string, error)) *GCENodeServer { +func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, mapper cryptMapper, evalSymLinks func(string) (string, error)) *GCENodeServer { return &GCENodeServer{ Driver: gceDriver, Mounter: mounter, @@ -168,19 +172,21 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi } } -func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute) *GCEControllerServer { +func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration) *GCEControllerServer { return &GCEControllerServer{ Driver: gceDriver, CloudProvider: cloudProvider, seen: map[string]int{}, volumeLocks: common.NewVolumeLocks(), - errorBackoff: newCsiErrorBackoff(), + errorBackoff: newCsiErrorBackoff(errorBackoffInitialDuration, errorBackoffMaxDuration), } } -func (gceDriver *GCEDriver) Run(endpoint string) { +func (gceDriver *GCEDriver) Run(endpoint string, grpcLogCharCap int) { + maxLogChar = grpcLogCharCap + klog.V(4).Infof("Driver: %v", gceDriver.name) - //Start the nonblocking GRPC + // Start the nonblocking GRPC s := NewNonBlockingGRPCServer() // TODO(#34): Only start specific servers based on a flag. // In the future have this only run specific combinations of servers depending on which version this is. diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go index 7aa2fd86..b32bc64b 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go @@ -16,6 +16,7 @@ package gceGCEDriver import ( "testing" + "time" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" ) @@ -43,7 +44,10 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver { vendorVersion := "test-vendor" gceDriver := GetGCEDriver() - controllerServer := NewControllerServer(gceDriver, cloudProvider) + errorBackoffInitialDuration := 200 * time.Millisecond + errorBackoffMaxDuration := 5 * time.Minute + + controllerServer := NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration) err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, controllerServer, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index 587f49a6..3b74af47 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -44,11 +44,12 @@ import ( csi "github.com/container-storage-interface/spec/lib/go/csi" - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/mount-utils" "github.com/edgelesssys/constellation/v2/csi/cryptmapper" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata" mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" @@ -64,7 +65,7 @@ type cryptMapper interface { type GCENodeServer struct { Driver *GCEDriver Mounter *mount.SafeFormatAndMount - DeviceUtils mountmanager.DeviceUtils + DeviceUtils deviceutils.DeviceUtils VolumeStatter mountmanager.Statter MetadataService metadataservice.MetadataService CryptMapper cryptMapper @@ -98,6 +99,7 @@ func getDefaultFsType() string { return defaultLinuxFsType } } + func (ns *GCENodeServer) isVolumePathMounted(path string) bool { notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(path) klog.V(4).Infof("NodePublishVolume check volume path %s is mounted %t: error %v", path, !notMnt, err) @@ -107,7 +109,6 @@ func (ns *GCENodeServer) isVolumePathMounted(path string) bool { 1) Target Path MUST be the vol referenced by vol ID 2) TODO(#253): Check volume capability matches for ALREADY_EXISTS 3) Readonly MUST match - */ return true } @@ -140,7 +141,7 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub defer ns.volumeLocks.Release(volumeID) if err := validateVolumeCapability(volumeCapability); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err.Error())) } if ns.isVolumePathMounted(targetPath) { @@ -170,7 +171,7 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub sourcePath = stagingTargetPath if err := preparePublishPath(targetPath, ns.Mounter); err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", targetPath, err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", targetPath, err.Error())) } } else if blk := volumeCapability.GetBlock(); blk != nil { @@ -179,20 +180,20 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub // [Edgeless] use the mapped device created by NodeStageVolume _, volumeKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodePublishVolume Volume ID is invalid: %v", err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err.Error())) } sourcePath, err = ns.evalSymLinks(filepath.Join("/dev/mapper", volumeKey.Name)) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume can not evaluate source path: %v", err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume can not evaluate source path: %v", err.Error())) } // Expose block volume as file at target path err = makeFile(targetPath) if err != nil { if removeErr := os.Remove(targetPath); removeErr != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Error removing block file at target path %v: %v, mounti error: %v", targetPath, removeErr, err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("Error removing block file at target path %v: %v, mounti error: %v", targetPath, removeErr, err.Error())) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create block file at target path %v: %v", targetPath, err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create block file at target path %v: %v", targetPath, err.Error())) } } else { return nil, status.Error(codes.InvalidArgument, "NodePublishVolume volume capability must specify either mount or block mode") @@ -200,35 +201,35 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub err = ns.Mounter.Interface.Mount(sourcePath, targetPath, fstype, options) if err != nil { - klog.Errorf("Mount of disk %s failed: %v", targetPath, err) + klog.Errorf("Mount of disk %s failed: %v", targetPath, err.Error()) notMnt, mntErr := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath) if mntErr != nil { - klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to check whether target path is a mount point: %v", err)) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr.Error()) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to check whether target path is a mount point: %v", err.Error())) } if !notMnt { // TODO: check the logic here again. If mntErr == nil & notMnt == false, it means volume is actually mounted. // Why need to unmount? klog.Warningf("Although volume mount failed, but IsLikelyNotMountPoint returns volume %s is mounted already at %s", volumeID, targetPath) if mntErr = ns.Mounter.Interface.Unmount(targetPath); mntErr != nil { - klog.Errorf("Failed to unmount: %v", mntErr) - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to unmount target path: %v", err)) + klog.Errorf("Failed to unmount: %v", mntErr.Error()) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to unmount target path: %v", err.Error())) } notMnt, mntErr := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath) if mntErr != nil { - klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to check whether target path is a mount point: %v", err)) + klog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr.Error()) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume failed to check whether target path is a mount point: %v", err.Error())) } if !notMnt { // This is very odd, we don't expect it. We'll try again next sync loop. klog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", targetPath) - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume something is wrong with mounting: %v", err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume something is wrong with mounting: %v", err.Error())) } } if err := os.Remove(targetPath); err != nil { - klog.Errorf("failed to remove targetPath %s: %v", targetPath, err) + klog.Errorf("failed to remove targetPath %s: %v", targetPath, err.Error()) } - return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume mount of disk failed: %v", err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume mount of disk failed: %v", err.Error())) } klog.V(4).Infof("NodePublishVolume succeeded on volume %v to %s", volumeID, targetPath) @@ -237,12 +238,12 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub func makeFile(path string) error { // Create file - newFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0750) + newFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o750) if err != nil { - return fmt.Errorf("failed to open file %s: %v", path, err) + return fmt.Errorf("failed to open file %s: %w", path, err) } if err := newFile.Close(); err != nil { - return fmt.Errorf("failed to close file %s: %v", path, err) + return fmt.Errorf("failed to close file %s: %w", path, err) } return nil } @@ -264,7 +265,7 @@ func (ns *GCENodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU defer ns.volumeLocks.Release(volumeID) if err := cleanupPublishPath(targetPath, ns.Mounter); err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Unmount failed: %v\nUnmounting arguments: %s\n", err, targetPath)) + return nil, status.Error(codes.Internal, fmt.Sprintf("Unmount failed: %v\nUnmounting arguments: %s\n", err.Error(), targetPath)) } klog.V(4).Infof("NodeUnpublishVolume succeeded on %v from %s", volumeID, targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil @@ -291,14 +292,14 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage defer ns.volumeLocks.Release(volumeID) if err := validateVolumeCapability(volumeCapability); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err.Error())) } // TODO(#253): Check volume capability matches for ALREADY_EXISTS _, volumeKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeStageVolume Volume ID is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeStageVolume Volume ID is invalid: %v", err.Error())) } // Part 1: Get device path of attached device @@ -308,9 +309,8 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage partition = part } devicePath, err := getDevicePath(ns, volumeID, partition) - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err.Error())) } klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath) @@ -322,7 +322,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage } if err := prepareStagePath(stagingTargetPath, ns.Mounter); err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", stagingTargetPath, err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", stagingTargetPath, err.Error())) } // Get FS type @@ -384,7 +384,15 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage } return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to format and mount device from (%q) to (%q) with fstype (%q) and options (%q): %v", - devicePath, stagingTargetPath, fstype, options, err)) + devicePath, stagingTargetPath, fstype, options, err.Error())) + } + + // Part 4: Resize filesystem. + // https://github.com/kubernetes/kubernetes/issues/94929 + resizer := resizefs.NewResizeFs(ns.Mounter) + _, err = ns.DeviceUtils.Resize(resizer, devicePath, stagingTargetPath) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("error when resizing volume %s from device '%s' at path '%s': %v", volumeID, devicePath, stagingTargetPath, err.Error())) } klog.V(4).Infof("NodeStageVolume succeeded on %v to %s", volumeID, stagingTargetPath) @@ -408,21 +416,21 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns defer ns.volumeLocks.Release(volumeID) if err := cleanupStagePath(stagingTargetPath, ns.Mounter); err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed: %v\nUnmounting arguments: %s\n", err, stagingTargetPath)) + return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed: %v\nUnmounting arguments: %s\n", err.Error(), stagingTargetPath)) } _, volumeKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeUnstageVolume failed: Volume ID is invalid: %s", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeUnstageVolume failed: Volume ID is invalid: %s", err.Error())) } deviceName, err := common.GetDeviceName(volumeKey) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeUnstageVolume failed: getting device name: %s", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("NodeUnstageVolume failed: getting device name: %s", err.Error())) } // [Edgeless] Unmap the crypt device so we can properly remove the device from the node if err := ns.CryptMapper.CloseCryptDevice(deviceName); err != nil { - return nil, status.Errorf(codes.Internal, "NodeUnstageVolume failed to close mapped crypt device for disk %s (%v)", stagingTargetPath, err) + return nil, status.Errorf(codes.Internal, "NodeUnstageVolume failed to close mapped crypt device for disk %s: %s", stagingTargetPath, err.Error()) } klog.V(4).Infof("NodeUnstageVolume succeeded on %v from %s", volumeID, stagingTargetPath) @@ -465,17 +473,17 @@ func (ns *GCENodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGe if os.IsNotExist(err) { return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath) } - return nil, status.Errorf(codes.Internal, "unknown error when stat on %s: %v", req.VolumePath, err) + return nil, status.Errorf(codes.Internal, "unknown error when stat on %s: %v", req.VolumePath, err.Error()) } isBlock, err := ns.VolumeStatter.IsBlockDevice(req.VolumePath) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to determine whether %s is block device: %v", req.VolumePath, err) + return nil, status.Errorf(codes.Internal, "failed to determine whether %s is block device: %v", req.VolumePath, err.Error()) } if isBlock { bcap, err := getBlockSizeBytes(req.VolumePath, ns.Mounter) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get block capacity on path %s: %v", req.VolumePath, err) + return nil, status.Errorf(codes.Internal, "failed to get block capacity on path %s: %v", req.VolumePath, err.Error()) } return &csi.NodeGetVolumeStatsResponse{ Usage: []*csi.VolumeUsage{ @@ -488,7 +496,7 @@ func (ns *GCENodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGe } available, capacity, used, inodesFree, inodes, inodesUsed, err := ns.VolumeStatter.StatFS(req.VolumePath) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get fs info on path %s: %v", req.VolumePath, err) + return nil, status.Errorf(codes.Internal, "failed to get fs info on path %s: %v", req.VolumePath, err.Error()) } return &csi.NodeGetVolumeStatsResponse{ @@ -517,7 +525,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa capacityRange := req.GetCapacityRange() reqBytes, err := getRequestCapacity(capacityRange) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("capacity range is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("capacity range is invalid: %v", err.Error())) } reqBytes = reqBytes - cryptmapper.LUKSHeaderSize // LUKS2 header is 16MiB, subtract from request size to get expected value @@ -528,14 +536,14 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa _, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("volume ID is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("volume ID is invalid: %v", err.Error())) } volumeCapability := req.GetVolumeCapability() if volumeCapability != nil { // VolumeCapability is optional, if specified, validate it if err := validateVolumeCapability(volumeCapability); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err)) + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err.Error())) } } @@ -561,7 +569,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa resizer := resizefs.NewResizeFs(ns.Mounter) _, err = resizer.Resize(devicePath, volumePath) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("error when resizing volume %s: %v", volKey.String(), err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("error when resizing volume %s: %v", volKey.String(), err.Error())) } diskSizeBytes, err := getBlockSizeBytes(devicePath, ns.Mounter) @@ -582,12 +590,12 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa /* format, err := ns.Mounter.GetDiskFormat(devicePath) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error checking format for device %s: %v", devicePath, err)) + return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error checking format for device %s: %v", devicePath, err.Error())) } gotSizeBytes, err = ns.getFSSizeBytes(devicePath) if err != nil { - return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize could not get fs size of %s: %v", volumePath, err) + return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize could not get fs size of %s: %v", volumePath, err.Error()) } if gotSizeBytes != reqBytes { return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize requested for size %v but after resize volume was size %v", reqBytes, gotSizeBytes) diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go index 812b1f20..599a60e3 100644 --- a/pkg/gce-pd-csi-driver/node_test.go +++ b/pkg/gce-pd-csi-driver/node_test.go @@ -16,7 +16,6 @@ along with this program. If not, see . This file incorporates work covered by the following copyright and permission notice: - Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,22 +34,29 @@ package gceGCEDriver import ( "context" + "fmt" "io/ioutil" "os" "path/filepath" "testing" + "k8s.io/utils/exec" + testingexec "k8s.io/utils/exec/testing" + csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/mount-utils" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata" mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" ) -const defaultVolumeID = "project/test001/zones/c1/disks/testDisk" -const defaultTargetPath = "/mnt/test" -const defaultStagingPath = "/staging" +const ( + defaultVolumeID = "project/test001/zones/c1/disks/testDisk" + defaultTargetPath = "/mnt/test" + defaultStagingPath = "/staging" +) type fakeCryptMapper struct { deviceName string @@ -77,10 +83,14 @@ func fakeEvalSymlinks(path string) (string, error) { } func getTestGCEDriver(t *testing.T) *GCEDriver { - return getCustomTestGCEDriver(t, mountmanager.NewFakeSafeMounter(), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService()) + return getCustomTestGCEDriver(t, mountmanager.NewFakeSafeMounter(), deviceutils.NewFakeDeviceUtils(), metadataservice.NewFakeService()) +} + +func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAndMount) *GCEDriver { + return getCustomTestGCEDriver(t, mounter, deviceutils.NewFakeDeviceUtils(), metadataservice.NewFakeService()) } -func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils mountmanager.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { +func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { gceDriver := GetGCEDriver() nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}, fakeEvalSymlinks) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nodeServer) @@ -93,7 +103,7 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev func getTestBlockingGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver { gceDriver := GetGCEDriver() mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) - nodeServer := NewNodeServer(gceDriver, mounter, mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}, fakeEvalSymlinks) + nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}, fakeEvalSymlinks) err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) @@ -101,6 +111,15 @@ func getTestBlockingGCEDriver(t *testing.T, readyToExecute chan chan struct{}) * return gceDriver } +func makeFakeCmd(fakeCmd *testingexec.FakeCmd, cmd string, args ...string) testingexec.FakeCommandAction { + c := cmd + a := args + return func(cmd string, args ...string) exec.Cmd { + command := testingexec.InitFakeCmd(fakeCmd, c, a...) + return command + } +} + func TestNodeGetVolumeStats(t *testing.T) { gceDriver := getTestGCEDriver(t) ns := gceDriver.ns @@ -156,7 +175,6 @@ func TestNodeGetVolumeStats(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - req := &csi.NodeGetVolumeStatsRequest{ VolumeId: tc.volumeID, VolumePath: tc.volumePath, @@ -173,7 +191,6 @@ func TestNodeGetVolumeStats(t *testing.T) { } func TestNodeGetVolumeLimits(t *testing.T) { - gceDriver := getTestGCEDriver(t) ns := gceDriver.ns req := &csi.NodeGetInfoRequest{} @@ -389,8 +406,6 @@ func TestNodeUnpublishVolume(t *testing.T) { } func TestNodeStageVolume(t *testing.T) { - gceDriver := getTestGCEDriver(t) - ns := gceDriver.ns volumeID := "project/test001/zones/c1/disks/testDisk" blockCap := &csi.VolumeCapability_Block{ Block: &csi.VolumeCapability_BlockVolume{}, @@ -474,6 +489,61 @@ func TestNodeStageVolume(t *testing.T) { } for _, tc := range testCases { t.Logf("Test case: %s", tc.name) + actionList := []testingexec.FakeCommandAction{ + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte(fmt.Sprintf("DEVNAME=/dev/sdb\nTYPE=ext4")), nil, nil + }, + }, + }, + "blkid", + ), + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte("1"), nil, nil + }, + }, + }, + "blockdev", + ), + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte("1"), nil, nil + }, + }, + }, + "blockdev", + ), + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte(fmt.Sprintf("DEVNAME=/dev/sdb\nTYPE=ext4")), nil, nil + }, + }, + }, + "blkid", + ), + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte(fmt.Sprintf("block size: 1\nblock count: 1")), nil, nil + }, + }, + }, + "dumpe2fs", + ), + } + mounter := mountmanager.NewFakeSafeMounterWithCustomExec(&testingexec.FakeExec{CommandScript: actionList}) + gceDriver := getTestGCEDriverWithCustomMounter(t, mounter) + ns := gceDriver.ns _, err := ns.NodeStageVolume(context.Background(), tc.req) if err != nil { serverError, ok := status.FromError(err) diff --git a/pkg/gce-pd-csi-driver/server.go b/pkg/gce-pd-csi-driver/server.go index 573fa800..48f3af69 100644 --- a/pkg/gce-pd-csi-driver/server.go +++ b/pkg/gce-pd-csi-driver/server.go @@ -22,7 +22,7 @@ import ( "sync" "google.golang.org/grpc" - "k8s.io/klog" + "k8s.io/klog/v2" csi "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -106,7 +106,7 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c klog.V(4).Infof("Start listening with scheme %v, addr %v", u.Scheme, addr) listener, err := net.Listen(u.Scheme, addr) if err != nil { - klog.Fatalf("Failed to listen: %v", err) + klog.Fatalf("Failed to listen: %v", err.Error()) } server := grpc.NewServer(opts...) @@ -125,7 +125,7 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c klog.V(4).Infof("Listening for connections on address: %#v", listener.Addr()) if err := server.Serve(listener); err != nil { - klog.Fatalf("Failed to serve: %v", err) + klog.Fatalf("Failed to serve: %v", err.Error()) } } diff --git a/pkg/gce-pd-csi-driver/utils.go b/pkg/gce-pd-csi-driver/utils.go index 4862613a..c81fc5be 100644 --- a/pkg/gce-pd-csi-driver/utils.go +++ b/pkg/gce-pd-csi-driver/utils.go @@ -19,12 +19,16 @@ package gceGCEDriver import ( "errors" "fmt" + "net/http" "context" csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/api/googleapi" "google.golang.org/grpc" - "k8s.io/klog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" ) const ( @@ -67,9 +71,13 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h klog.V(4).Infof("%s called with request: %s", info.FullMethod, req) resp, err := handler(ctx, req) if err != nil { - klog.Errorf("%s returned with error: %v", info.FullMethod, err) + klog.Errorf("%s returned with error: %v", info.FullMethod, err.Error()) } else { - klog.V(4).Infof("%s returned with response: %s", info.FullMethod, resp) + cappedStr := fmt.Sprintf("%v", resp) + if len(cappedStr) > maxLogChar { + cappedStr = cappedStr[:maxLogChar] + fmt.Sprintf(" [response body too large, log capped to %d chars]", maxLogChar) + } + klog.V(4).Infof("%s returned with response: %s", info.FullMethod, cappedStr) } return resp, err } @@ -209,3 +217,36 @@ func containsZone(zones []string, zone string) bool { return false } + +// CodeForError returns a pointer to the grpc error code that maps to the http +// error code for the passed in user googleapi error. Returns codes.Internal if +// the given error is not a googleapi error caused by the user. The following +// http error codes are considered user errors: +// (1) http 400 Bad Request, returns grpc InvalidArgument, +// (2) http 403 Forbidden, returns grpc PermissionDenied, +// (3) http 404 Not Found, returns grpc NotFound +// (4) http 429 Too Many Requests, returns grpc ResourceExhausted +func CodeForError(err error) *codes.Code { + internalErrorCode := codes.Internal + // Upwrap the error + var apiErr *googleapi.Error + if !errors.As(err, &apiErr) { + return &internalErrorCode + } + + userErrors := map[int]codes.Code{ + http.StatusForbidden: codes.PermissionDenied, + http.StatusBadRequest: codes.InvalidArgument, + http.StatusTooManyRequests: codes.ResourceExhausted, + http.StatusNotFound: codes.NotFound, + } + if code, ok := userErrors[apiErr.Code]; ok { + return &code + } + return &internalErrorCode +} + +func LoggedError(msg string, err error) error { + klog.Errorf(msg+"%v", err.Error()) + return status.Errorf(*CodeForError(err), msg+"%v", err.Error()) +} diff --git a/pkg/gce-pd-csi-driver/utils_linux.go b/pkg/gce-pd-csi-driver/utils_linux.go index cce6bdeb..a1afe786 100644 --- a/pkg/gce-pd-csi-driver/utils_linux.go +++ b/pkg/gce-pd-csi-driver/utils_linux.go @@ -33,7 +33,7 @@ func getDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error } deviceName, err := common.GetDeviceName(volumeKey) if err != nil { - return "", fmt.Errorf("error getting device name: %v", err) + return "", fmt.Errorf("error getting device name: %w", err) } if deviceName == "" { return "", fmt.Errorf("device name is empty") @@ -76,7 +76,7 @@ func cleanupStagePath(path string, m *mount.SafeFormatAndMount) error { func getBlockSizeBytes(devicePath string, m *mount.SafeFormatAndMount) (int64, error) { output, err := m.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput() if err != nil { - return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", devicePath, string(output), err) + return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %w", devicePath, string(output), err) } strOut := strings.TrimSpace(string(output)) gotSizeBytes, err := strconv.ParseInt(strOut, 10, 64) diff --git a/pkg/gce-pd-csi-driver/utils_test.go b/pkg/gce-pd-csi-driver/utils_test.go index 2a0fd5f4..ab92e45e 100644 --- a/pkg/gce-pd-csi-driver/utils_test.go +++ b/pkg/gce-pd-csi-driver/utils_test.go @@ -18,9 +18,13 @@ limitations under the License. package gceGCEDriver import ( + "errors" + "net/http" "testing" csi "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" ) var ( @@ -291,3 +295,37 @@ func TestGetReadOnlyFromCapabilities(t *testing.T) { } } } + +func TestCodeForError(t *testing.T) { + internalErrorCode := codes.Internal + userErrorCode := codes.InvalidArgument + testCases := []struct { + name string + inputErr error + expCode *codes.Code + }{ + { + name: "Not googleapi.Error", + inputErr: errors.New("I am not a googleapi.Error"), + expCode: &internalErrorCode, + }, + { + name: "User error", + inputErr: &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"}, + expCode: &userErrorCode, + }, + { + name: "googleapi.Error but not a user error", + inputErr: &googleapi.Error{Code: http.StatusInternalServerError, Message: "Internal error"}, + expCode: &internalErrorCode, + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + actualCode := *CodeForError(tc.inputErr) + if *tc.expCode != actualCode { + t.Fatalf("Expected error code '%v' but got '%v'", tc.expCode, actualCode) + } + } +} diff --git a/pkg/gce-pd-csi-driver/utils_windows.go b/pkg/gce-pd-csi-driver/utils_windows.go index 5d32ac7a..4661cd89 100644 --- a/pkg/gce-pd-csi-driver/utils_windows.go +++ b/pkg/gce-pd-csi-driver/utils_windows.go @@ -83,7 +83,7 @@ func getDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error } deviceName, err := common.GetDeviceName(volumeKey) if err != nil { - return "", fmt.Errorf("error getting device name: %v", err) + return "", fmt.Errorf("error getting device name: %w", err) } proxy, ok := ns.Mounter.Interface.(mounter.CSIProxyMounter) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d8dfdb94..cae37923 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,7 +22,7 @@ import ( "os" "k8s.io/component-base/metrics" - "k8s.io/klog" + "k8s.io/klog/v2" ) const ( @@ -101,7 +101,7 @@ func (mm *metricsManager) InitializeHttpHandler(address, path string) { go func() { klog.Infof("Metric server listening at %q", address) if err := http.ListenAndServe(address, mux); err != nil { - klog.Fatalf("Failed to start metric server at specified address (%q) and path (%q): %s", address, path, err) + klog.Fatalf("Failed to start metric server at specified address (%q) and path (%q): %v", address, path, err.Error()) } }() } diff --git a/pkg/mount-manager/safe-mounter-v1_windows.go b/pkg/mount-manager/safe-mounter-v1_windows.go index ee9e8613..68233837 100644 --- a/pkg/mount-manager/safe-mounter-v1_windows.go +++ b/pkg/mount-manager/safe-mounter-v1_windows.go @@ -34,7 +34,7 @@ import ( volumeapi "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1" volumeclient "github.com/kubernetes-csi/csi-proxy/client/groups/volume/v1" - "k8s.io/klog" + "k8s.io/klog/v2" mount "k8s.io/mount-utils" ) diff --git a/pkg/mount-manager/safe-mounter-v1beta_windows.go b/pkg/mount-manager/safe-mounter-v1beta_windows.go index 1b4c30cc..70ce52ee 100644 --- a/pkg/mount-manager/safe-mounter-v1beta_windows.go +++ b/pkg/mount-manager/safe-mounter-v1beta_windows.go @@ -34,7 +34,7 @@ import ( volumeapi "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1beta1" volumeclient "github.com/kubernetes-csi/csi-proxy/client/groups/volume/v1beta1" - "k8s.io/klog" + "k8s.io/klog/v2" mount "k8s.io/mount-utils" ) diff --git a/pkg/mount-manager/safe-mounter_windows.go b/pkg/mount-manager/safe-mounter_windows.go index b97857b3..23e6dd36 100644 --- a/pkg/mount-manager/safe-mounter_windows.go +++ b/pkg/mount-manager/safe-mounter_windows.go @@ -17,7 +17,7 @@ limitations under the License. package mountmanager import ( - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" ) @@ -67,7 +67,7 @@ func NewSafeMounter() (*mount.SafeFormatAndMount, error) { Exec: utilexec.New(), }, nil } - klog.V(4).Infof("failed to connect to csi-proxy v1 with error=%v, will try with v1Beta", err) + klog.V(4).Infof("failed to connect to csi-proxy v1 with error=%v, will try with v1Beta", err.Error()) csiProxyMounterV1Beta, err := NewCSIProxyMounterV1Beta() if err == nil { @@ -77,6 +77,6 @@ func NewSafeMounter() (*mount.SafeFormatAndMount, error) { Exec: utilexec.New(), }, nil } - klog.V(4).Infof("failed to connect to csi-proxy v1beta with error=%v", err) + klog.V(4).Infof("failed to connect to csi-proxy v1beta with error=%v", err.Error()) return nil, err } diff --git a/pkg/mount-manager/statter_linux.go b/pkg/mount-manager/statter_linux.go index a4e0b194..d75c39b3 100644 --- a/pkg/mount-manager/statter_linux.go +++ b/pkg/mount-manager/statter_linux.go @@ -45,7 +45,7 @@ func (*realStatter) StatFS(path string) (available, capacity, used, inodesFree, statfs := &unix.Statfs_t{} err = unix.Statfs(path, statfs) if err != nil { - err = fmt.Errorf("failed to get fs info on path %s: %v", path, err) + err = fmt.Errorf("failed to get fs info on path %s: %w", path, err) return } diff --git a/pkg/resizefs/resizefs_linux.go b/pkg/resizefs/resizefs_linux.go index 674b785c..206c0135 100644 --- a/pkg/resizefs/resizefs_linux.go +++ b/pkg/resizefs/resizefs_linux.go @@ -19,9 +19,10 @@ limitations under the License. package resizefs import ( - "fmt" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/mount-utils" ) @@ -38,51 +39,22 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *resizeFs { } // Resize perform resize of file system -func (resizefs *resizeFs) Resize(devicePath, deviceMountPath string) (bool, error) { - format, err := resizefs.mounter.GetDiskFormat(devicePath) +func (resizefs *resizeFs) Resize(devicePath, deviceMountPath string) (needResize bool, err error) { + resizer := mount.NewResizeFs(resizefs.mounter.Exec) - if err != nil { - formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err) - return false, formatErr + klog.V(4).Infof("Checking if filesystem needs to be resized. Device: %s Mountpoint: %s", devicePath, deviceMountPath) + if needResize, err = resizer.NeedResize(devicePath, deviceMountPath); err != nil { + err = status.Errorf(codes.Internal, "Could not determine if filesystem %q needs to be resized: %v", deviceMountPath, err) + return } - // If disk has no format, there is no need to resize the disk because mkfs.* - // by default will use whole disk anyways. - if format == "" { - return false, nil + if needResize { + klog.V(4).Infof("Resizing filesystem. Device: %s Mountpoint: %s", devicePath, deviceMountPath) + if _, err = resizer.Resize(devicePath, deviceMountPath); err != nil { + err = status.Errorf(codes.Internal, "Failed to resize filesystem %q: %v", deviceMountPath, err) + return + } } - klog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) - switch format { - case "ext3", "ext4": - return resizefs.extResize(devicePath) - case "xfs": - return resizefs.xfsResize(deviceMountPath) - } - return false, fmt.Errorf("ResizeFS.Resize - resize of format %s is not supported for device %s mounted at %s", format, devicePath, deviceMountPath) -} - -func (resizefs *resizeFs) extResize(devicePath string) (bool, error) { - output, err := resizefs.mounter.Exec.Command("resize2fs", devicePath).CombinedOutput() - if err == nil { - klog.V(2).Infof("Device %s resized successfully", devicePath) - return true, nil - } - - resizeError := fmt.Errorf("resize of device %s failed: %v. resize2fs output: %s", devicePath, err, string(output)) - return false, resizeError - -} - -func (resizefs *resizeFs) xfsResize(deviceMountPath string) (bool, error) { - args := []string{"-d", deviceMountPath} - output, err := resizefs.mounter.Exec.Command("xfs_growfs", args...).CombinedOutput() - - if err == nil { - klog.V(2).Infof("Device %s resized successfully", deviceMountPath) - return true, nil - } - - resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", deviceMountPath, err, string(output)) - return false, resizeError + return } diff --git a/pkg/resizefs/resizefs_windows.go b/pkg/resizefs/resizefs_windows.go index 1b39ced8..a1643899 100644 --- a/pkg/resizefs/resizefs_windows.go +++ b/pkg/resizefs/resizefs_windows.go @@ -25,7 +25,7 @@ import ( volumeapiv1 "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1" volumeapiv1beta1 "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1beta1" - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/mount-utils" mounter "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" ) diff --git a/release-tools/KUBERNETES_CSI_OWNERS_ALIASES b/release-tools/KUBERNETES_CSI_OWNERS_ALIASES index 2f3e2dfc..344fbe4d 100644 --- a/release-tools/KUBERNETES_CSI_OWNERS_ALIASES +++ b/release-tools/KUBERNETES_CSI_OWNERS_ALIASES @@ -27,6 +27,7 @@ aliases: - jingxu97 - jsafrane - pohly + - RaunakShah - xing-yang # This documents who previously contributed to Kubernetes-CSI diff --git a/release-tools/filter-junit.go b/release-tools/filter-junit.go index cf1cb4ab..5454092b 100644 --- a/release-tools/filter-junit.go +++ b/release-tools/filter-junit.go @@ -35,10 +35,18 @@ var ( ) /* - * TestSuite represents a JUnit file. Due to how encoding/xml works, we have + * TestResults represents a JUnit file. Due to how encoding/xml works, we have * represent all fields that we want to be passed through. It's therefore * not a complete solution, but good enough for Ginkgo + Spyglass. + * + * Before Kubernetes 1.25 and ginkgo v2, we directly had in the + * JUnit file. Now we get and inside it the . */ +type TestResults struct { + XMLName string `xml:"testsuites"` + TestSuite TestSuite `xml:"testsuite"` +} + type TestSuite struct { XMLName string `xml:"testsuite"` TestCases []TestCase `xml:"testcase"` @@ -93,7 +101,15 @@ func main() { } } if err := xml.Unmarshal(data, &junit); err != nil { - panic(err) + if err.Error() != "expected element type but have " { + panic(err) + } + // Fall back to Ginkgo v2 format. + var junitv2 TestResults + if err := xml.Unmarshal(data, &junitv2); err != nil { + panic(err) + } + junit = junitv2.TestSuite } } diff --git a/release-tools/go-get-kubernetes.sh b/release-tools/go-get-kubernetes.sh index 0a960c47..2b46c187 100755 --- a/release-tools/go-get-kubernetes.sh +++ b/release-tools/go-get-kubernetes.sh @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + # This script can be used while converting a repo from "dep" to "go mod" # by calling it after "go mod init" or to update the Kubernetes packages # in a repo that has already been converted. Only packages that are diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 74cc017a..55c3231d 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -86,14 +86,17 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.18" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.19" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below # ginkgo test runner version to use. If the pre-installed version is -# different, the desired version is built from source. +# different, the desired version is built from source. For Kubernetes, +# the version built via "make WHAT=vendor/github.com/onsi/ginkgo/ginkgo" is +# used, which is guaranteed to match what the Kubernetes e2e.test binary +# needs. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" # Ginkgo runs the E2E test in parallel. The default is based on the number @@ -118,7 +121,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.22.0 "Kubernetes" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST @@ -138,7 +141,7 @@ kind_version_default () { latest|master) echo main;; *) - echo v0.11.1;; + echo v0.14.0;; esac } @@ -149,16 +152,13 @@ configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind" # kind images to use. Must match the kind version. # The release notes of each kind release list the supported images. -configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.23.0@sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac -kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047 -kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6 -kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9 -kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729 -kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c -kindest/node:v1.17.17@sha256:66f1d0d91a88b8a001811e2f1054af60eef3b669a9a74f9b6db871f2f1eeed00 -kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861 -kindest/node:v1.15.12@sha256:b920920e1eda689d9936dfcf7332701e80be12566999152626b2c9d730397a95 -kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e54de8c6f7219f8" "kind images" +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.24.0@sha256:0866296e693efe1fed79d5e6c7af8df71fc73ae45e3679af05342239cdc5bc8e +kindest/node:v1.23.6@sha256:b1fa224cc6c7ff32455e0b1fd9cbfd3d3bc87ecaa8fcb06961ed1afb3db0f9ae +kindest/node:v1.22.9@sha256:8135260b959dfe320206eb36b3aeda9cffcb262f4b44cda6b33f7bb73f453105 +kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207 +kindest/node:v1.20.15@sha256:6f2d011dffe182bad80b85f6c00e8ca9d86b5b8922cdf433d53575c4c5212248 +kindest/node:v1.19.16@sha256:d9c819e8668de8d5030708e484a9fdff44d95ec4675d136ef0a0a584e587f65c +kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fcdbd126188e6d7" "kind images" # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on @@ -196,7 +196,7 @@ kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e5 # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.8.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" @@ -228,13 +228,16 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION} configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" +# Local path for e2e tests. Set to "none" to disable. +configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" + # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain # csi.sock as a TCP service for use by the csi-sanity command, which runs outside # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.3.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v5.0.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -282,13 +285,18 @@ tests_enabled () { sanity_enabled () { [ "${CSI_PROW_TESTS_SANITY}" = "sanity" ] && tests_enabled "sanity" } + +sidecar_tests_enabled () { + [ "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" != "none" ] +} + tests_need_kind () { tests_enabled "parallel" "serial" "serial-alpha" "parallel-alpha" || - sanity_enabled + sanity_enabled || sidecar_tests_enabled } tests_need_non_alpha_cluster () { tests_enabled "parallel" "serial" || - sanity_enabled + sanity_enabled || sidecar_tests_enabled } tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" @@ -346,15 +354,23 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +configvar CSI_PROW_E2E_GATES_LATEST '' "non alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_GATES "$(get_versioned_variable CSI_PROW_E2E_GATES "${csi_prow_kubernetes_version_suffix}")" "non alpha E2E feature gates" + +# Focus for local tests run in the sidecar E2E repo. Only used if CSI_PROW_SIDECAR_E2E_IMPORT_PATH +# is not set to "none". If empty, all tests in the sidecar repo will be run. +configvar CSI_PROW_SIDECAR_E2E_FOCUS '' "tags for local E2E tests" +configvar CSI_PROW_SIDECAR_E2E_SKIP '' "local tests that need to be skipped" + # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment default_csi_snapshotter_version () { if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo "master" else - echo "v3.0.2" + echo "v4.0.0" fi } configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag" @@ -365,7 +381,7 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP '\[Disruptive\]|\[Feature:SELinux\]' "tests that need to be skipped" # This creates directories that are required for testing. ensure_paths () { @@ -437,6 +453,10 @@ install_kind () { # Ensure that we have the desired version of the ginkgo test runner. install_ginkgo () { + if [ -e "${CSI_PROW_BIN}/ginkgo" ]; then + return + fi + # CSI_PROW_GINKGO_VERSION contains the tag with v prefix, the command line output does not. if [ "v$(ginkgo version 2>/dev/null | sed -e 's/.* //')" = "${CSI_PROW_GINKGO_VERSION}" ]; then return @@ -935,12 +955,17 @@ install_e2e () { return fi + if sidecar_tests_enabled; then + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" go test -c -o "${CSI_PROW_WORK}/e2e-local.test" "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" + fi git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" + ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" && + run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/ginkgo" "${CSI_PROW_BIN}" else run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e" fi @@ -988,8 +1013,13 @@ run_e2e () ( } trap move_junit EXIT - cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + if [ "${name}" == "local" ]; then + cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local + else + cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + fi ) # Run csi-sanity against installed CSI driver. @@ -1254,7 +1284,8 @@ main () { fi if tests_need_non_alpha_cluster; then - start_cluster || die "starting the non-alpha cluster failed" + # Need to (re)create the cluster. + start_cluster "${CSI_PROW_E2E_GATES}" || die "starting the non-alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller install_snapshot_crds @@ -1300,11 +1331,24 @@ main () { ret=1 fi fi + + if sidecar_tests_enabled; then + if ! run_e2e local \ + -focus="${CSI_PROW_SIDECAR_E2E_FOCUS}" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E sidecar failed" + ret=1 + fi + fi fi delete_cluster_inside_prow_job non-alpha fi - if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then + # If the cluster for alpha tests doesn't need any feature gates, then we + # could reuse the same cluster as for the other tests. But that would make + # the flow in this script harder and wouldn't help in practice because + # we have separate Prow jobs for alpha and non-alpha tests. + if tests_need_alpha_cluster; then # Need to (re)create the cluster. start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index 6dae9c1a..5149c4d8 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -20,11 +20,11 @@ import ( "strings" csi "github.com/container-storage-interface/spec/lib/go/csi" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" @@ -157,7 +157,7 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In testFile := filepath.Join(a.publishDir, "testfile") err := testutils.WriteFile(instance, testFile, testFileContents) if err != nil { - return fmt.Errorf("Failed to write file: %v", err) + return fmt.Errorf("Failed to write file: %v", err.Error()) } } return nil @@ -168,7 +168,7 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In secondTestFile := filepath.Join(a.publishDir, "testfile") readContents, err := testutils.ReadFile(instance, secondTestFile) if err != nil { - return fmt.Errorf("ReadFile failed with error: %v", err) + return fmt.Errorf("ReadFile failed with error: %v", err.Error()) } if strings.TrimSpace(string(readContents)) != testFileContents { return fmt.Errorf("wanted test file content: %s, got content: %s", testFileContents, readContents) @@ -184,14 +184,14 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst // Attach Disk err = client.ControllerPublishVolume(volID, instance.GetNodeID()) if err != nil { - return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err) + return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()) } defer func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() @@ -206,19 +206,19 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst //err = client.NodeStageExt4Volume(volID, stageDir) if err != nil { - return fmt.Errorf("NodeStageExt4Volume failed with error: %v", err) + return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err) } defer func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() @@ -232,11 +232,11 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst } if err != nil { - return fmt.Errorf("NodePublishVolume failed with error: %v", err) + return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) } err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") if err != nil { - return fmt.Errorf("Chmod failed with error: %v", err) + return fmt.Errorf("Chmod failed with error: %v", err.Error()) } a := verifyArgs{ @@ -251,7 +251,7 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst // Unmount Disk err = client.NodeUnpublishVolume(volID, publishDir) if err != nil { - return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err) + return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) } if secondMountVerify != nil { @@ -263,7 +263,7 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst err = client.NodePublishVolume(volID, stageDir, secondPublishDir) } if err != nil { - return fmt.Errorf("NodePublishVolume failed with error: %v", err) + return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) } err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") if err != nil { @@ -275,13 +275,13 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst } err = secondMountVerify(b) if err != nil { - return fmt.Errorf("failed to verify after second mount to %s: %v", publishDir, err) + return fmt.Errorf("failed to verify after second mount to %s: %v", publishDir, err.Error()) } // Unmount Disk err = client.NodeUnpublishVolume(volID, secondPublishDir) if err != nil { - return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err) + return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) } } diff --git a/test/e2e/tests/resize_e2e_test.go b/test/e2e/tests/resize_e2e_test.go index 48241267..f6126475 100644 --- a/test/e2e/tests/resize_e2e_test.go +++ b/test/e2e/tests/resize_e2e_test.go @@ -17,12 +17,12 @@ package tests import ( "path/filepath" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" csi "github.com/container-storage-interface/spec/lib/go/csi" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" @@ -74,7 +74,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() @@ -87,12 +87,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() @@ -105,7 +105,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unmount Disk err = client.NodeUnpublishVolume(volID, publishDir) if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %v", err) + klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } }() @@ -196,7 +196,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() @@ -210,12 +210,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() @@ -228,21 +228,16 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unmount Disk err = client.NodeUnpublishVolume(volID, publishDir) if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %v", err) + klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } }() - // Verify pre-resize fs size - sizeGb, err := testutils.GetFSSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get FSSize in GB") - Expect(sizeGb).To(Equal(defaultSizeGb)) - // Resize node _, err = client.NodeExpandVolume(volID, publishDir, newSizeGb) Expect(err).To(BeNil(), "Node expand volume failed") // Verify disk size - sizeGb, err = testutils.GetFSSizeInGb(instance, publishDir) + sizeGb, err := testutils.GetFSSizeInGb(instance, publishDir) Expect(err).To(BeNil(), "Failed to get FSSize in GB") Expect(sizeGb).To(Equal(newSizeGb)) @@ -293,7 +288,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() @@ -307,12 +302,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() @@ -325,7 +320,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unmount Disk err = client.NodeUnpublishVolume(volID, publishDir) if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %v", err) + klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } }() diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 9a7932ff..a1988c34 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -24,11 +24,12 @@ import ( "time" cloudkms "cloud.google.com/go/kms/apiv1" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" computealpha "google.golang.org/api/compute/v0.alpha" + computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" - "k8s.io/klog" + "k8s.io/klog/v2" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" ) @@ -46,6 +47,7 @@ var ( testContexts = []*remote.TestContext{} computeService *compute.Service computeAlphaService *computealpha.Service + computeBetaService *computebeta.Service kmsClient *cloudkms.KeyManagementClient ) @@ -54,7 +56,6 @@ func init() { } func TestE2E(t *testing.T) { - flag.Parse() RegisterFailHandler(Fail) RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests") @@ -75,6 +76,9 @@ var _ = BeforeSuite(func() { computeAlphaService, err = remote.GetComputeAlphaClient() Expect(err).To(BeNil()) + computeBetaService, err = remote.GetComputeBetaClient() + Expect(err).To(BeNil()) + // Create the KMS client. kmsClient, err = cloudkms.NewKeyManagementClient(context.Background()) Expect(err).To(BeNil()) @@ -86,53 +90,23 @@ var _ = BeforeSuite(func() { Expect(*project).ToNot(BeEmpty(), "Project should not be empty") Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty") - klog.Infof("Running in project %v with service account %v\n\n", *project, *serviceAccount) + klog.Infof("Running in project %v with service account %v", *project, *serviceAccount) for _, zone := range zones { go func(curZone string) { defer GinkgoRecover() - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", curZone) - klog.Infof("Setting up node %s\n", nodeID) - - i, err := remote.SetupInstance(*project, *architecture, curZone, nodeID, *machineType, *serviceAccount, *imageURL, computeService) - if err != nil { - klog.Fatalf("Failed to setup instance %v: %v", nodeID, err) - } - - err = testutils.MkdirAll(i, "/lib/udev_containerized") - if err != nil { - klog.Fatalf("Could not make scsi_id containerized directory: %v", err) - } - - err = testutils.CopyFile(i, "/lib/udev/scsi_id", "/lib/udev_containerized/scsi_id") - if err != nil { - klog.Fatalf("could not copy scsi_id to containerized directory: %v", err) - } - - err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id") - if err != nil { - klog.Fatalf("could not copy google_nvme_id to containerized directory: %v", err) - } - - klog.Infof("Creating new driver and client for node %s\n", i.GetName()) - // Create new driver and client - testContext, err := testutils.GCEClientAndDriverSetup(i) - if err != nil { - klog.Fatalf("Failed to set up Test Context for instance %v: %v", i.GetName(), err) - } - tcc <- testContext + tcc <- NewTestContext(curZone) }(zone) } for i := 0; i < len(zones); i++ { tc := <-tcc - klog.Infof("Test Context for node %s set up\n", tc.Instance.GetName()) testContexts = append(testContexts, tc) + klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) } }) var _ = AfterSuite(func() { - for _, tc := range testContexts { err := remote.TeardownDriverAndClient(tc) Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") @@ -142,6 +116,40 @@ var _ = AfterSuite(func() { } }) +func NewTestContext(zone string) *remote.TestContext { + nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) + klog.Infof("Setting up node %s", nodeID) + + i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService) + if err != nil { + klog.Fatalf("Failed to setup instance %v: %w", nodeID, err) + } + + err = testutils.MkdirAll(i, "/lib/udev_containerized") + if err != nil { + klog.Fatalf("Failed to make scsi_id containerized directory: %w", err) + } + + err = testutils.CopyFile(i, "/lib/udev/scsi_id", "/lib/udev_containerized/scsi_id") + if err != nil { + klog.Fatalf("Failed to copy scsi_id to containerized directory: %w", err) + } + + err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id") + if err != nil { + klog.Fatalf("Failed to copy google_nvme_id to containerized directory: %w", err) + } + + klog.Infof("Creating new driver and client for node %s", i.GetName()) + tc, err := testutils.GCEClientAndDriverSetup(i, "") + if err != nil { + klog.Fatalf("Failed to set up TestContext for instance %v: %w", i.GetName(), err) + } + + klog.Infof("Finished creating TestContext for node %s", tc.Instance.GetName()) + return tc +} + func getRandomTestContext() *remote.TestContext { Expect(testContexts).ToNot(BeEmpty()) rn := rand.Intn(len(testContexts)) diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index abacc564..7c8d7e05 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -18,22 +18,24 @@ import ( "context" "fmt" "path/filepath" + "regexp" "strings" "time" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" - mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" csi "github.com/container-storage-interface/spec/lib/go/csi" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + compute "google.golang.org/api/compute/v1" "google.golang.org/api/iterator" kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1" fieldmask "google.golang.org/genproto/protobuf/field_mask" @@ -42,12 +44,15 @@ import ( const ( testNamePrefix = "gcepd-csi-e2e-" - defaultSizeGb int64 = 5 - defaultRepdSizeGb int64 = 200 - defaultMwSizeGb int64 = 200 - readyState = "READY" - standardDiskType = "pd-standard" - defaultVolumeLimit int64 = 127 + defaultSizeGb int64 = 5 + defaultRepdSizeGb int64 = 200 + defaultMwSizeGb int64 = 200 + defaultVolumeLimit int64 = 127 + readyState = "READY" + standardDiskType = "pd-standard" + extremeDiskType = "pd-extreme" + provisionedIOPSOnCreate = "100000" + provisionedIOPSOnCreateInt = int64(100000) defaultEpsilon = 500000000 // 500M ) @@ -70,7 +75,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -95,7 +100,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -115,13 +120,13 @@ var _ = Describe("GCE PD CSI Driver", func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() // MESS UP THE symlink - devicePaths := mountmanager.NewDeviceUtils().GetDiskByIdPaths(volName, "") + devicePaths := deviceutils.NewDeviceUtils().GetDiskByIdPaths(volName, "") for _, devicePath := range devicePaths { err = testutils.RmAll(instance, devicePath) Expect(err).To(BeNil(), "failed to remove /dev/by-id folder") @@ -149,12 +154,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() }) @@ -167,7 +172,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -187,13 +192,13 @@ var _ = Describe("GCE PD CSI Driver", func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { - klog.Errorf("Failed to detach disk: %v", err) + klog.Errorf("Failed to detach disk: %w", err) } }() // DELETE THE symlink - devicePaths := mountmanager.NewDeviceUtils().GetDiskByIdPaths(volName, "") + devicePaths := deviceutils.NewDeviceUtils().GetDiskByIdPaths(volName, "") for _, devicePath := range devicePaths { err = testutils.RmAll(instance, devicePath) Expect(err).To(BeNil(), "failed to remove /dev/by-id folder") @@ -219,12 +224,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { - klog.Errorf("Failed to unstage volume: %v", err) + klog.Errorf("Failed to unstage volume: %w", err) } fp := filepath.Join("/tmp/", volName) err = testutils.RmAll(instance, fp) if err != nil { - klog.Errorf("Failed to rm file path %s: %v", fp, err) + klog.Errorf("Failed to rm file path %s: %w", fp, err) } }() }) @@ -256,35 +261,75 @@ var _ = Describe("GCE PD CSI Driver", func() { _, err = computeService.Disks.Get(p, zone, volName).Do() Expect(err).To(BeNil(), "Could not find disk in correct zone") } - }) - It("Should complete entire disk lifecycle with underspecified volume ID", func() { - testContext := getRandomTestContext() + DescribeTable("Should complete entire disk lifecycle with underspecified volume ID", + func(diskType string) { + testContext := getRandomTestContext() - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + instance := testContext.Instance - volName, _ := createAndValidateUniqueZonalDisk(client, p, z) + volName, _ := createAndValidateUniqueZonalDisk(client, p, z, diskType) - underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) + underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) - defer func() { - // Delete Disk - err := client.DeleteVolume(underSpecifiedID) - Expect(err).To(BeNil(), "DeleteVolume failed") + defer func() { + // Delete Disk + err := client.DeleteVolume(underSpecifiedID) + Expect(err).To(BeNil(), "DeleteVolume failed") - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() - // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") + // Attach Disk + err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle") + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) - }) + DescribeTable("Should complete publish/unpublish lifecycle with underspecified volume ID and missing volume", + func(diskType string) { + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + instance := testContext.Instance + + // Create Disk + volName, _ := createAndValidateUniqueZonalDisk(client, p, z, diskType) + underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) + + defer func() { + // Detach Disk + err := instance.DetachDisk(volName) + Expect(err).To(BeNil(), "DetachDisk failed") + + // Delete Disk + err = client.DeleteVolume(underSpecifiedID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + + // Unpublish Disk + err = client.ControllerUnpublishVolume(underSpecifiedID, instance.GetNodeID()) + Expect(err).To(BeNil(), "ControllerUnpublishVolume failed") + }() + + // Attach Disk + err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID()) + Expect(err).To(BeNil(), "ControllerPublishVolume failed") + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) It("Should successfully create RePD in two zones in the drivers region when none are specified", func() { Expect(testContexts).ToNot(BeEmpty()) @@ -331,80 +376,86 @@ var _ = Describe("GCE PD CSI Driver", func() { }() }) - It("Should create and delete disk with default zone", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, nil, defaultSizeGb, nil, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + DescribeTable("Should create and delete disk with default zone", + func(diskType string) { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client - defer func() { - // Delete Disk - client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) + // Create Disk + disk := typeToDisk[diskType] + volName := testNamePrefix + string(uuid.NewUUID()) + volID, err := client.CreateVolume(volName, disk.params, defaultSizeGb, nil, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - It("Should create and delete disk with labels", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + disk.validate(cloudDisk) - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client + defer func() { + // Delete Disk + client.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - params := map[string]string{ - common.ParameterKeyLabels: "key1=value1,key2=value2", - } - volID, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Labels).To(Equal(map[string]string{ - "key1": "value1", - "key2": "value2", - // The label below is added as an --extra-label driver command line argument. - testutils.DiskLabelKey: testutils.DiskLabelValue, - })) - Expect(cloudDisk.Name).To(Equal(volName)) + DescribeTable("Should create and delete disk with labels", + func(diskType string) { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) + // Create Disk + disk := typeToDisk[diskType] + volName := testNamePrefix + string(uuid.NewUUID()) + params := merge(disk.params, map[string]string{ + common.ParameterKeyLabels: "key1=value1,key2=value2", + }) + volID, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Labels).To(Equal(map[string]string{ + "key1": "value1", + "key2": "value2", + // The label below is added as an --extra-label driver command line argument. + testutils.DiskLabelKey: testutils.DiskLabelValue, + })) + Expect(cloudDisk.Name).To(Equal(volName)) + disk.validate(cloudDisk) - // Test volume already exists idempotency + defer func() { + // Delete Disk + err := client.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") - // Test volume with op pending + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) It("Should create and delete snapshot for the volume with default zone", func() { Expect(testContexts).ToNot(BeEmpty()) @@ -413,7 +464,7 @@ var _ = Describe("GCE PD CSI Driver", func() { p, z, _ := testContext.Instance.GetIdentity() client := testContext.Client - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -454,161 +505,119 @@ var _ = Describe("GCE PD CSI Driver", func() { }() }) - It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() { - ctx := context.Background() - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client + DescribeTable("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", + func(diskType string) { + ctx := context.Background() + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() - p, z, _ := controllerInstance.GetIdentity() - locationID := "global" + controllerInstance := testContext.Instance + controllerClient := testContext.Client - // The resource name of the key rings. - parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID) - keyRingId := "gce-pd-csi-test-ring" + p, z, _ := controllerInstance.GetIdentity() + locationID := "global" - // Create KeyRing - ringReq := &kmspb.CreateKeyRingRequest{ - Parent: parentName, - KeyRingId: keyRingId, - } - keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq) - if !gce.IsGCEError(err, "alreadyExists") { - getKeyRingReq := &kmspb.GetKeyRingRequest{ - Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId), - } - keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq) + // The resource name of the key rings. + parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID) + keyRingId := "gce-pd-csi-test-ring" - } - Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId) - - // Create CryptoKey in KeyRing - keyId := "test-key-" + string(uuid.NewUUID()) - keyReq := &kmspb.CreateCryptoKeyRequest{ - Parent: keyRing.Name, - CryptoKeyId: keyId, - CryptoKey: &kmspb.CryptoKey{ - Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT, - VersionTemplate: &kmspb.CryptoKeyVersionTemplate{ - Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION, - }, - }, - } - key, err := kmsClient.CreateCryptoKey(ctx, keyReq) - Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name) + key, keyVersions := setupKeyRing(ctx, parentName, keyRingId) - keyVersions := []string{} - keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{ - Parent: key.Name, - } - - it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq) - - for { - keyVersion, err := it.Next() - if err == iterator.Done { - break - } - Expect(err).To(BeNil(), "Failed to list crypto key versions") - - keyVersions = append(keyVersions, keyVersion.Name) - } - - // Defer deletion of all key versions - // https://cloud.google.com/kms/docs/destroy-restore - defer func() { - - for _, keyVersion := range keyVersions { - destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{ - Name: keyVersion, + // Defer deletion of all key versions + // https://cloud.google.com/kms/docs/destroy-restore + defer func() { + for _, keyVersion := range keyVersions { + destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{ + Name: keyVersion, + } + _, err := kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) + Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion) } - _, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) - Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion) - } - - }() + }() - // Go through volume lifecycle using CMEK-ed PD - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyDiskEncryptionKmsKey: key.Name, - }, defaultSizeGb, - &csi.TopologyRequirement{ + // Go through volume lifecycle using CMEK-ed PD Create Disk + disk := typeToDisk[diskType] + volName := testNamePrefix + string(uuid.NewUUID()) + params := merge(disk.params, map[string]string{ + common.ParameterKeyDiskEncryptionKmsKey: key.Name, + }) + topology := &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { Segments: map[string]string{common.TopologyKeyZone: z}, }, }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + } + volID, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, topology, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + disk.validate(cloudDisk) - defer func() { - // Delete Disk - err = controllerClient.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") + defer func() { + // Delete Disk + err = controllerClient.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() - // Test disk works - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") + // Test disk works + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") - // Revoke CMEK key - // https://cloud.google.com/kms/docs/enable-disable + // Revoke CMEK key + // https://cloud.google.com/kms/docs/enable-disable - for _, keyVersion := range keyVersions { - disableReq := &kmspb.UpdateCryptoKeyVersionRequest{ - CryptoKeyVersion: &kmspb.CryptoKeyVersion{ - Name: keyVersion, - State: kmspb.CryptoKeyVersion_DISABLED, - }, - UpdateMask: &fieldmask.FieldMask{ - Paths: []string{"state"}, - }, + for _, keyVersion := range keyVersions { + disableReq := &kmspb.UpdateCryptoKeyVersionRequest{ + CryptoKeyVersion: &kmspb.CryptoKeyVersion{ + Name: keyVersion, + State: kmspb.CryptoKeyVersion_DISABLED, + }, + UpdateMask: &fieldmask.FieldMask{ + Paths: []string{"state"}, + }, + } + _, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq) + Expect(err).To(BeNil(), "Failed to disable crypto key") } - _, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq) - Expect(err).To(BeNil(), "Failed to disable crypto key") - } - // Make sure attach of PD fails - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") + // Make sure attach of PD fails + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") - // Restore CMEK key - for _, keyVersion := range keyVersions { - enableReq := &kmspb.UpdateCryptoKeyVersionRequest{ - CryptoKeyVersion: &kmspb.CryptoKeyVersion{ - Name: keyVersion, - State: kmspb.CryptoKeyVersion_ENABLED, - }, - UpdateMask: &fieldmask.FieldMask{ - Paths: []string{"state"}, - }, + // Restore CMEK key + for _, keyVersion := range keyVersions { + enableReq := &kmspb.UpdateCryptoKeyVersionRequest{ + CryptoKeyVersion: &kmspb.CryptoKeyVersion{ + Name: keyVersion, + State: kmspb.CryptoKeyVersion_ENABLED, + }, + UpdateMask: &fieldmask.FieldMask{ + Paths: []string{"state"}, + }, + } + _, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq) + Expect(err).To(BeNil(), "Failed to enable crypto key") } - _, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq) - Expect(err).To(BeNil(), "Failed to enable crypto key") - } - // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. - time.Sleep(time.Second) - // Make sure attach of PD succeeds - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") - }) + // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. + time.Sleep(time.Second) + // Make sure attach of PD succeeds + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) It("Should create disks, attach them places, and verify List returns correct results", func() { Expect(testContexts).ToNot(BeEmpty()) @@ -619,10 +628,10 @@ var _ = Describe("GCE PD CSI Driver", func() { nodeID := testContext.Instance.GetNodeID() - _, volID := createAndValidateUniqueZonalDisk(client, p, z) + _, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer deleteVolumeOrError(client, volID) - _, secondVolID := createAndValidateUniqueZonalDisk(client, p, z) + _, secondVolID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer deleteVolumeOrError(client, secondVolID) // Attach volID to current instance @@ -721,7 +730,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client instance := testContext.Instance - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -736,7 +745,7 @@ var _ = Describe("GCE PD CSI Driver", func() { verifyVolumeStats := func(a verifyArgs) error { available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) if err != nil { - return fmt.Errorf("failed to get node volume stats: %v", err) + return fmt.Errorf("failed to get node volume stats: %v", err.Error()) } if available != 0 || capacity != common.GbToBytes(defaultSizeGb) || used != 0 || inodesFree != 0 || inodes != 0 || inodesUsed != 0 { @@ -758,7 +767,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client instance := testContext.Instance - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -773,7 +782,7 @@ var _ = Describe("GCE PD CSI Driver", func() { verifyVolumeStats := func(a verifyArgs) error { available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) if err != nil { - return fmt.Errorf("failed to get node volume stats: %v", err) + return fmt.Errorf("failed to get node volume stats: %v", err.Error()) } if !equalWithinEpsilon(available, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(capacity, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(used, 0, defaultEpsilon) || inodesFree == 0 || inodes == 0 || inodesUsed == 0 { @@ -800,7 +809,7 @@ var _ = Describe("GCE PD CSI Driver", func() { zone := "us-east1-a" // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, zone) + volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, zone, standardDiskType) defer func() { // Delete Disk @@ -822,7 +831,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -839,14 +848,14 @@ var _ = Describe("GCE PD CSI Driver", func() { writeFunc := func(a verifyArgs) error { err := testutils.WriteBlock(instance, a.publishDir, testFileContents) if err != nil { - return fmt.Errorf("Failed to write file: %v", err) + return fmt.Errorf("Failed to write file: %v", err.Error()) } return nil } verifyReadFunc := func(a verifyArgs) error { readContents, err := testutils.ReadBlock(instance, a.publishDir, len(testFileContents)) if err != nil { - return fmt.Errorf("ReadFile failed with error: %v", err) + return fmt.Errorf("ReadFile failed with error: %v", err.Error()) } if strings.TrimSpace(string(readContents)) != testFileContents { return fmt.Errorf("wanted test file content: %s, got content: %s", testFileContents, readContents) @@ -857,42 +866,49 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(err).To(BeNil(), "Failed to go through volume lifecycle") }) - It("Should successfully create disk with PVC/PV tags", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() + DescribeTable("Should successfully create disk with PVC/PV tags", + func(diskType string) { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() - controllerInstance := testContext.Instance - controllerClient := testContext.Client + controllerInstance := testContext.Instance + controllerClient := testContext.Client - p, z, _ := controllerInstance.GetIdentity() + p, z, _ := controllerInstance.GetIdentity() - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyPVCName: "test-pvc", - common.ParameterKeyPVCNamespace: "test-pvc-namespace", - common.ParameterKeyPVName: "test-pv-name", - }, defaultSizeGb, nil /* topReq */, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + // Create Disk + disk := typeToDisk[diskType] + volName := testNamePrefix + string(uuid.NewUUID()) + params := merge(disk.params, map[string]string{ + common.ParameterKeyPVCName: "test-pvc", + common.ParameterKeyPVCNamespace: "test-pvc-namespace", + common.ParameterKeyPVName: "test-pv-name", + }) + volID, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, nil /* topReq */, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + Expect(cloudDisk.Description).To(Equal("{\"kubernetes.io/created-for/pv/name\":\"test-pv-name\",\"kubernetes.io/created-for/pvc/name\":\"test-pvc\",\"kubernetes.io/created-for/pvc/namespace\":\"test-pvc-namespace\",\"storage.gke.io/created-by\":\"pd.csi.storage.gke.io\"}")) + disk.validate(cloudDisk) - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(cloudDisk.Description).To(Equal("{\"kubernetes.io/created-for/pv/name\":\"test-pv-name\",\"kubernetes.io/created-for/pvc/name\":\"test-pvc\",\"kubernetes.io/created-for/pvc/namespace\":\"test-pvc-namespace\",\"storage.gke.io/created-by\":\"pd.csi.storage.gke.io\"}")) - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") + defer func() { + // Delete Disk + controllerClient.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }, + Entry("on pd-standard", standardDiskType), + Entry("on pd-extreme", extremeDiskType), + ) // Use the region of the test location. It("Should successfully create snapshot with storage locations", func() { @@ -902,7 +918,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -963,7 +979,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -1024,7 +1040,7 @@ var _ = Describe("GCE PD CSI Driver", func() { p, z, _ := controllerInstance.GetIdentity() // Create Source Disk - _, srcVolID := createAndValidateUniqueZonalDisk(controllerClient, p, z) + _, srcVolID := createAndValidateUniqueZonalDisk(controllerClient, p, z, standardDiskType) // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) @@ -1123,6 +1139,40 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") }() }) + + It("Should pass/fail if valid/invalid compute endpoint is passed in", func() { + // gets instance set up w/o compute-endpoint set from test setup + _, err := getRandomTestContext().Client.ListVolumes() + Expect(err).To(BeNil(), "no error expected when passed valid compute url") + + zone := "us-central1-c" + nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) + i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService) + + if err != nil { + klog.Fatalf("Failed to setup instance %v: %w", nodeID, err) + } + + klog.Infof("Creating new driver and client for node %s\n", i.GetName()) + + // Create new driver and client w/ invalid endpoint + tcInvalid, err := testutils.GCEClientAndDriverSetup(i, "invalid-string") + if err != nil { + klog.Fatalf("Failed to set up Test Context for instance %v: %w", i.GetName(), err) + } + + _, err = tcInvalid.Client.ListVolumes() + Expect(err.Error()).To(ContainSubstring("no such host"), "expected error when passed invalid compute url") + + // Create new driver and client w/ valid, passed-in endpoint + tcValid, err := testutils.GCEClientAndDriverSetup(i, "https://compute.googleapis.com") + if err != nil { + klog.Fatalf("Failed to set up Test Context for instance %v: %w", i.GetName(), err) + } + _, err = tcValid.Client.ListVolumes() + + Expect(err).To(BeNil(), "no error expected when passed valid compute url") + }) }) func equalWithinEpsilon(a, b, epsiolon int64) bool { @@ -1132,11 +1182,11 @@ func equalWithinEpsilon(a, b, epsiolon int64) bool { return b-a < epsiolon } -func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string) (volName, volID string) { +func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) { // Create Disk - var err error - volName = testNamePrefix + string(uuid.NewUUID()) - volID, err = client.CreateVolume(volName, nil, defaultSizeGb, + disk := typeToDisk[diskType] + volName := testNamePrefix + string(uuid.NewUUID()) + volID, err := client.CreateVolume(volName, disk.params, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -1149,11 +1199,12 @@ func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone st // Validate Disk Created cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do() Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) Expect(cloudDisk.Status).To(Equal(readyState)) Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) Expect(cloudDisk.Name).To(Equal(volName)) - return + disk.validate(cloudDisk) + + return volName, volID } func deleteVolumeOrError(client *remote.CsiClient, volID string) { @@ -1168,10 +1219,11 @@ func deleteVolumeOrError(client *remote.CsiClient, volID string) { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") } -func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string) (string, string) { +func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) { // Create Disk + disk := typeToDisk[diskType] volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolumeWithCaps(volName, nil, defaultMwSizeGb, + volID, err := client.CreateVolumeWithCaps(volName, disk.params, defaultMwSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -1192,19 +1244,108 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created - cloudDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) + cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do() + Expect(err).To(BeNil(), "Failed to get cloud disk") Expect(cloudDisk.Status).To(Equal(readyState)) Expect(cloudDisk.SizeGb).To(Equal(defaultMwSizeGb)) Expect(cloudDisk.Name).To(Equal(volName)) - Expect(cloudDisk.MultiWriter).To(Equal(true)) + disk.validate(cloudDisk) + + alphaDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do() + Expect(err).To(BeNil(), "Failed to get cloud disk using alpha API") + Expect(alphaDisk.MultiWriter).To(Equal(true)) return volName, volID } func cleanSelfLink(selfLink string) string { - temp := strings.TrimPrefix(selfLink, gce.GCEComputeAPIEndpoint) - temp = strings.TrimPrefix(temp, gce.GCEComputeBetaAPIEndpoint) - return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint) + r, _ := regexp.Compile("https:\\/\\/www.*apis.com\\/.*(v1|beta|alpha)\\/") + return r.ReplaceAllString(selfLink, "") +} + +func setupKeyRing(ctx context.Context, parentName string, keyRingId string) (*kmspb.CryptoKey, []string) { + // Create KeyRing + ringReq := &kmspb.CreateKeyRingRequest{ + Parent: parentName, + KeyRingId: keyRingId, + } + keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq) + if !gce.IsGCEError(err, "alreadyExists") { + getKeyRingReq := &kmspb.GetKeyRingRequest{ + Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId), + } + keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq) + + } + Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId) + + // Create CryptoKey in KeyRing + keyId := "test-key-" + string(uuid.NewUUID()) + keyReq := &kmspb.CreateCryptoKeyRequest{ + Parent: keyRing.Name, + CryptoKeyId: keyId, + CryptoKey: &kmspb.CryptoKey{ + Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT, + VersionTemplate: &kmspb.CryptoKeyVersionTemplate{ + Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION, + }, + }, + } + key, err := kmsClient.CreateCryptoKey(ctx, keyReq) + Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name) + + keyVersions := []string{} + keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{ + Parent: key.Name, + } + + it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq) + + for { + keyVersion, err := it.Next() + if err == iterator.Done { + break + } + Expect(err).To(BeNil(), "Failed to list crypto key versions") + + keyVersions = append(keyVersions, keyVersion.Name) + } + return key, keyVersions +} + +type disk struct { + params map[string]string + validate func(disk *compute.Disk) +} + +var typeToDisk = map[string]*disk{ + standardDiskType: { + params: map[string]string{ + common.ParameterKeyType: standardDiskType, + }, + validate: func(disk *compute.Disk) { + Expect(disk.Type).To(ContainSubstring(standardDiskType)) + }, + }, + extremeDiskType: { + params: map[string]string{ + common.ParameterKeyType: extremeDiskType, + common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, + }, + validate: func(disk *compute.Disk) { + Expect(disk.Type).To(ContainSubstring(extremeDiskType)) + Expect(disk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) + }, + }, +} + +func merge(a, b map[string]string) map[string]string { + res := map[string]string{} + for k, v := range a { + res[k] = v + } + for k, v := range b { + res[k] = v + } + return res } diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 6d0198eb..8a80501e 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -27,7 +27,7 @@ import ( "golang.org/x/oauth2/google" cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - "k8s.io/klog" + "k8s.io/klog/v2" boskosclient "sigs.k8s.io/boskos/client" "sigs.k8s.io/boskos/common" utilcommon "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" @@ -43,7 +43,7 @@ var ( boskos, _ = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos", "", "") ) -func GCEClientAndDriverSetup(instance *remote.InstanceInfo) (*remote.TestContext, error) { +func GCEClientAndDriverSetup(instance *remote.InstanceInfo, computeEndpoint string) (*remote.TestContext, error) { port := fmt.Sprintf("%v", 1024+rand.Intn(10000)) goPath, ok := os.LookupEnv("GOPATH") if !ok { @@ -53,10 +53,14 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo) (*remote.TestContext binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver") endpoint := fmt.Sprintf("tcp://localhost:%s", port) + computeFlag := "" + if computeEndpoint != "" { + computeFlag = fmt.Sprintf("--compute-endpoint %s", computeEndpoint) + } workspace := remote.NewWorkspaceDir("gce-pd-e2e-") - driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver -v=4 --endpoint=%s --extra-labels=%s=%s 2> %s/prog.out < /dev/null > /dev/null &'", - workspace, endpoint, DiskLabelKey, DiskLabelValue, workspace) + driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver -v=4 --endpoint=%s %s --extra-labels=%s=%s 2> %s/prog.out < /dev/null > /dev/null &'", + workspace, endpoint, computeFlag, DiskLabelKey, DiskLabelValue, workspace) config := &remote.ClientConfig{ PkgPath: pkgPath, @@ -88,7 +92,7 @@ func getBoskosProject(resourceType string) *common.Resource { case <-ticker.C: p, err := boskos.Acquire(resourceType, "free", "busy") if err != nil { - klog.Warningf("boskos failed to acquire project: %v", err) + klog.Warningf("boskos failed to acquire project: %w", err) } else if p == nil { klog.Warningf("boskos does not have a free %s at the moment", resourceType) } else { @@ -120,17 +124,17 @@ func SetupProwConfig(resourceType string) (project, serviceAccount string) { c, err := google.DefaultClient(context.Background(), cloudresourcemanager.CloudPlatformScope) if err != nil { - klog.Fatalf("Failed to get Google Default Client: %v", err) + klog.Fatalf("Failed to get Google Default Client: %w", err) } cloudresourcemanagerService, err := cloudresourcemanager.New(c) if err != nil { - klog.Fatalf("Failed to create new cloudresourcemanager: %v", err) + klog.Fatalf("Failed to create new cloudresourcemanager: %w", err) } resp, err := cloudresourcemanagerService.Projects.Get(project).Do() if err != nil { - klog.Fatalf("Failed to get project %v from Cloud Resource Manager: %v", project, err) + klog.Fatalf("Failed to get project %v from Cloud Resource Manager: %w", project, err) } // Default Compute Engine service account @@ -143,19 +147,19 @@ func SetupProwConfig(resourceType string) (project, serviceAccount string) { func ForceChmod(instance *remote.InstanceInfo, filePath string, perms string) error { originalumask, err := instance.SSHNoSudo("umask") if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", originalumask, err) + return fmt.Errorf("failed to umask. Output: %v, errror: %v", originalumask, err.Error()) } output, err := instance.SSHNoSudo("umask", "0000") if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err) + return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err.Error()) } output, err = instance.SSH("chmod", "-R", perms, filePath) if err != nil { - return fmt.Errorf("failed to chmod file %s. Output: %v, errror: %v", filePath, output, err) + return fmt.Errorf("failed to chmod file %s. Output: %v, errror: %v", filePath, output, err.Error()) } output, err = instance.SSHNoSudo("umask", originalumask) if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err) + return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err.Error()) } return nil } @@ -163,7 +167,7 @@ func ForceChmod(instance *remote.InstanceInfo, filePath string, perms string) er func WriteFile(instance *remote.InstanceInfo, filePath, fileContents string) error { output, err := instance.SSHNoSudo("echo", fileContents, ">", filePath) if err != nil { - return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", filePath, output, err) + return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", filePath, output, err.Error()) } return nil } @@ -171,7 +175,7 @@ func WriteFile(instance *remote.InstanceInfo, filePath, fileContents string) err func ReadFile(instance *remote.InstanceInfo, filePath string) (string, error) { output, err := instance.SSHNoSudo("cat", filePath) if err != nil { - return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", filePath, output, err) + return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", filePath, output, err.Error()) } return output, nil } @@ -179,7 +183,7 @@ func ReadFile(instance *remote.InstanceInfo, filePath string) (string, error) { func WriteBlock(instance *remote.InstanceInfo, path, fileContents string) error { output, err := instance.SSHNoSudo("echo", fileContents, "|", "dd", "of="+path) if err != nil { - return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", path, output, err) + return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", path, output, err.Error()) } return nil } @@ -188,7 +192,7 @@ func ReadBlock(instance *remote.InstanceInfo, path string, length int) (string, lengthStr := strconv.Itoa(length) output, err := instance.SSHNoSudo("dd", "if="+path, "bs="+lengthStr, "count=1", "2>", "/dev/null") if err != nil { - return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", path, output, err) + return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", path, output, err.Error()) } return output, nil } @@ -196,7 +200,7 @@ func ReadBlock(instance *remote.InstanceInfo, path string, length int) (string, func GetFSSizeInGb(instance *remote.InstanceInfo, mountPath string) (int64, error) { output, err := instance.SSH("df", "--output=size", "-BG", mountPath, "|", "awk", "'NR==2'") if err != nil { - return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", mountPath, output, err) + return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", mountPath, output, err.Error()) } output = strings.TrimSuffix(strings.TrimSpace(output), "G") n, err := strconv.ParseInt(output, 10, 64) @@ -209,7 +213,7 @@ func GetFSSizeInGb(instance *remote.InstanceInfo, mountPath string) (int64, erro func GetBlockSizeInGb(instance *remote.InstanceInfo, devicePath string) (int64, error) { output, err := instance.SSH("blockdev", "--getsize64", devicePath) if err != nil { - return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", devicePath, output, err) + return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", devicePath, output, err.Error()) } n, err := strconv.ParseInt(strings.TrimSpace(output), 10, 64) if err != nil { @@ -221,7 +225,7 @@ func GetBlockSizeInGb(instance *remote.InstanceInfo, devicePath string) (int64, func Symlink(instance *remote.InstanceInfo, src, dest string) error { output, err := instance.SSH("ln", "-s", src, dest) if err != nil { - return fmt.Errorf("failed to symlink from %s to %s. Output: %v, errror: %v", src, dest, output, err) + return fmt.Errorf("failed to symlink from %s to %s. Output: %v, errror: %v", src, dest, output, err.Error()) } return nil } @@ -229,7 +233,7 @@ func Symlink(instance *remote.InstanceInfo, src, dest string) error { func RmAll(instance *remote.InstanceInfo, filePath string) error { output, err := instance.SSH("rm", "-rf", filePath) if err != nil { - return fmt.Errorf("failed to delete all %s. Output: %v, errror: %v", filePath, output, err) + return fmt.Errorf("failed to delete all %s. Output: %v, errror: %v", filePath, output, err.Error()) } return nil } @@ -237,7 +241,7 @@ func RmAll(instance *remote.InstanceInfo, filePath string) error { func MkdirAll(instance *remote.InstanceInfo, dir string) error { output, err := instance.SSH("mkdir", "-p", dir) if err != nil { - return fmt.Errorf("failed to mkdir -p %s. Output: %v, errror: %v", dir, output, err) + return fmt.Errorf("failed to mkdir -p %s. Output: %v, errror: %v", dir, output, err.Error()) } return nil } @@ -245,7 +249,7 @@ func MkdirAll(instance *remote.InstanceInfo, dir string) error { func CopyFile(instance *remote.InstanceInfo, src, dest string) error { output, err := instance.SSH("cp", src, dest) if err != nil { - return fmt.Errorf("failed to copy %s to %s. Output: %v, errror: %v", src, dest, output, err) + return fmt.Errorf("failed to copy %s to %s. Output: %v, errror: %v", src, dest, output, err.Error()) } return nil } @@ -269,7 +273,7 @@ func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName str devFsPath, err := instance.SSH("find", link, "-printf", "'%l'") if err != nil { - return false, fmt.Errorf("failed to find symbolic link for %s. Output: %v, errror: %v", link, devFsPath, err) + return false, fmt.Errorf("failed to find symbolic link for %s. Output: %v, errror: %v", link, devFsPath, err.Error()) } if len(devFsPath) == 0 { return false, nil @@ -278,7 +282,7 @@ func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName str fullDevPath := path.Join("/dev/", string(sdx)) scsiIDOut, err := instance.SSH("/lib/udev_containerized/scsi_id", "--page=0x83", "--whitelisted", fmt.Sprintf("--device=%v", fullDevPath)) if err != nil { - return false, fmt.Errorf("failed to find %s's SCSI ID. Output: %v, errror: %v", devFsPath, scsiIDOut, err) + return false, fmt.Errorf("failed to find %s's SCSI ID. Output: %v, errror: %v", devFsPath, scsiIDOut, err.Error()) } scsiID := scsiRegex.FindStringSubmatch(scsiIDOut) if len(scsiID) == 0 { @@ -292,7 +296,7 @@ func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName str fullDevPath := path.Join("/dev/", string(nvmex)) nvmeIDOut, err := instance.SSH("/lib/udev_containerized/google_nvme_id", fmt.Sprintf("-d%v", fullDevPath)) if err != nil { - return false, fmt.Errorf("failed to find %s's NVME ID. Output: %v, errror: %v", devFsPath, nvmeIDOut, err) + return false, fmt.Errorf("failed to find %s's NVME ID. Output: %v, errror: %v", devFsPath, nvmeIDOut, err.Error()) } nvmeID := nvmeSerialRegex.FindStringSubmatch(nvmeIDOut) if len(nvmeID) == 0 { diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index 38257c29..e3425935 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -13,7 +13,7 @@ import ( apimachineryversion "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog" + "k8s.io/klog/v2" ) func gkeLocationArgs(gceZone, gceRegion string) (locationArg, locationVal string, err error) { @@ -54,7 +54,7 @@ func clusterDownGKE(gceZone, gceRegion string) error { locationArg, locationVal, "--quiet") err = runCommand("Bringing Down E2E Cluster on GKE", cmd) if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err) + return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err.Error()) } return nil } @@ -64,7 +64,7 @@ func buildKubernetes(k8sDir, command string) error { cmd.Env = os.Environ() err := runCommand(fmt.Sprintf("Running command in kubernetes/kubernetes path=%s", k8sDir), cmd) if err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err) + return fmt.Errorf("failed to build Kubernetes: %w", err) } return nil } @@ -76,7 +76,7 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, ima // Set kubectl to the one bundled in the k8s tar for versioning err = os.Setenv("GCE_PD_KUBECTL", kshPath) if err != nil { - return fmt.Errorf("failed to set cluster specific kubectl: %v", err) + return fmt.Errorf("failed to set cluster specific kubectl: %w", err) } } else { klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath) @@ -85,14 +85,14 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, ima if len(*kubeFeatureGates) != 0 { err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates) if err != nil { - return fmt.Errorf("failed to set kubernetes feature gates: %v", err) + return fmt.Errorf("failed to set kubernetes feature gates: %w", err) } klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) } err = setImageTypeEnvs(imageType) if err != nil { - return fmt.Errorf("failed to set image type environment variables: %v", err) + return fmt.Errorf("failed to set image type environment variables: %w", err) } err = os.Setenv("NUM_NODES", strconv.Itoa(numNodes)) @@ -126,7 +126,7 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, ima cmd.Env = os.Environ() err = runCommand("Starting E2E Cluster on GCE", cmd) if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err) + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err.Error()) } return nil @@ -170,7 +170,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, fmt.Sprintf("name=%s", *gkeTestClusterName)).CombinedOutput() if err != nil { - return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out) + return fmt.Errorf("failed to check for previous test cluster: %v %s", err.Error(), out) } if len(out) > 0 { klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", *gkeTestClusterName) @@ -203,7 +203,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, cmd = exec.Command("gcloud", cmdParams...) err = runCommand("Starting E2E Cluster on GKE", cmd) if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err.Error()) } // Because gcloud cannot disable addons on cluster create, the deployment has @@ -216,7 +216,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, "--update-addons", "GcePersistentDiskCsiDriver=DISABLED") err = runCommand("Updating E2E Cluster on GKE to disable driver deployment", cmd) if err != nil { - return fmt.Errorf("failed to update kubernetes e2e cluster on gke: %v", err) + return fmt.Errorf("failed to update kubernetes e2e cluster on gke: %v", err.Error()) } } @@ -245,7 +245,7 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { klog.Info("cloning k8s master") out, err := exec.Command("git", "clone", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput() if err != nil { - return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err) + return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error()) } } else { // Shallow clone of a release branch. @@ -253,7 +253,7 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { klog.Infof("shallow clone of k8s %s", vKubeVersion) out, err := exec.Command("git", "clone", "--depth", "1", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput() if err != nil { - return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err) + return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err.Error()) } } return nil @@ -289,7 +289,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) cmd := exec.Command("gcloud", "config", "get-value", "project") project, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("failed to get current project: %v", err) + return nil, fmt.Errorf("failed to get current project: %v", err.Error()) } // kubetest arguments @@ -353,7 +353,7 @@ func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) { func getKubeClusterVersion() (string, error) { out, err := exec.Command("kubectl", "version", "-o=json").Output() if err != nil { - return "", fmt.Errorf("failed to obtain cluster version, error: %v; output was %s", err, out) + return "", fmt.Errorf("failed to obtain cluster version, error: %v; output was %s", err.Error(), out) } type version struct { ClientVersion *apimachineryversion.Info `json:"clientVersion,omitempty" yaml:"clientVersion,omitempty"` @@ -363,7 +363,7 @@ func getKubeClusterVersion() (string, error) { var v version err = json.Unmarshal(out, &v) if err != nil { - return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err) + return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err.Error()) } return v.ServerVersion.GitVersion, nil @@ -372,7 +372,7 @@ func getKubeClusterVersion() (string, error) { func mustGetKubeClusterVersion() string { ver, err := getKubeClusterVersion() if err != nil { - klog.Fatalf("Error: %v", err) + klog.Fatalf("Error: %v", err.Error()) } return ver } @@ -401,11 +401,11 @@ func getKubeClient() (kubernetes.Interface, error) { } config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { - return nil, fmt.Errorf("failed to create config: %v", err) + return nil, fmt.Errorf("failed to create config: %v", err.Error()) } kubeClient, err := kubernetes.NewForConfig(config) if err != nil { - return nil, fmt.Errorf("failed to create client: %v", err) + return nil, fmt.Errorf("failed to create client: %v", err.Error()) } return kubeClient, nil } diff --git a/test/k8s-integration/config/sc-extreme.yaml b/test/k8s-integration/config/sc-extreme.yaml new file mode 100644 index 00000000..7f930c24 --- /dev/null +++ b/test/k8s-integration/config/sc-extreme.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-extreme + provisioned-iops-on-create: '10000' + # Add labels for testing. + labels: key1=value1,key2=value2 +volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index b3a8cf9b..25cdef47 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -127,7 +127,9 @@ func generateDriverConfigFile(testParams *testParameters) (string, error) { snapshotClassName = "no-volumesnapshotclass" } - caps = append(caps, "pvcDataSource") + if !strings.Contains(testParams.storageClassFile, "sc-extreme") { + caps = append(caps, "pvcDataSource") + } minimumVolumeSize := "5Gi" numAllowedTopologies := 1 if testParams.storageClassFile == regionalPDStorageClass { diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go index 179984d7..a8e7c209 100644 --- a/test/k8s-integration/driver.go +++ b/test/k8s-integration/driver.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "k8s.io/klog" + "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e/storage/podlogs" ) @@ -22,14 +22,14 @@ func installDriver(testParams *testParameters, stagingImage, deployOverlayName s klog.Infof("Installing kustomize") out, err := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput() if err != nil { - return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err) + return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err.Error()) } // Edit ci kustomization to use given image tag overlayDir := getOverlayDir(testParams.pkgDir, deployOverlayName) err = os.Chdir(overlayDir) if err != nil { - return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err) + return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err.Error()) } // TODO (#138): in a local environment this is going to modify the actual kustomize files. @@ -41,7 +41,7 @@ func installDriver(testParams *testParameters, stagingImage, deployOverlayName s "image", fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, testParams.stagingVersion)).CombinedOutput() if err != nil { - return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err) + return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err.Error()) } } @@ -54,7 +54,7 @@ func installDriver(testParams *testParameters, stagingImage, deployOverlayName s // Need to copy it to name the file "cloud-sa.json" out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput() if err != nil { - return fmt.Errorf("error copying service account key: %s, err: %v", out, err) + return fmt.Errorf("error copying service account key: %s, err: %v", out, err.Error()) } defer shredFile(tmpSaFile) @@ -99,14 +99,14 @@ func installDriver(testParams *testParameters, stagingImage, deployOverlayName s podsCmd := exec.Command("kubectl", "get", "pods", "-l", "app=gcp-compute-persistent-disk-csi-driver-win", "-o", "name", "-n", getDriverNamespace()) out, err = podsCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to get pd csi pods: %v", err) + return fmt.Errorf("failed to get pd csi pods: %v", err.Error()) } pods := strings.Fields(string(out)) for _, pod := range pods { logCmd := exec.Command("kubectl", "logs", pod, "-n", getDriverNamespace(), "--all-containers=true") out, err := logCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to get logs from pod %s: %v", pod, err) + return fmt.Errorf("failed to get logs from pod %s: %v", pod, err.Error()) } klog.Infof("kubectl logs pod: %s, output %s", pod, string(out)) } @@ -123,7 +123,7 @@ func deleteDriver(testParams *testParameters, deployOverlayName string) error { ) err := runCommand("Deleting driver", deleteCmd) if err != nil { - return fmt.Errorf("failed to delete driver: %v", err) + return fmt.Errorf("failed to delete driver: %v", err.Error()) } return nil } @@ -146,7 +146,7 @@ func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) err = runCommand("Building and Pushing GCP Container for Windows", cmd) if err != nil { - return fmt.Errorf("failed to run make command for windows: err: %v", err) + return fmt.Errorf("failed to run make command for windows: err: %v", err.Error()) } } else { cmd = exec.Command("make", "-C", pkgDir, "push-container", @@ -154,7 +154,7 @@ func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) err = runCommand("Pushing GCP Container for Linux", cmd) if err != nil { - return fmt.Errorf("failed to run make command for linux: err: %v", err) + return fmt.Errorf("failed to run make command for linux: err: %v", err.Error()) } } @@ -165,7 +165,7 @@ func deleteImage(stagingImage, stagingVersion string) error { cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet") err := runCommand("Deleting GCR Container", cmd) if err != nil { - return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err) + return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err.Error()) } return nil } @@ -180,7 +180,7 @@ func dumpDriverLogs() (context.CancelFunc, error) { if ok { client, err := getKubeClient() if err != nil { - return nil, fmt.Errorf("failed to get kubeclient: %v", err) + return nil, fmt.Errorf("failed to get kubeclient: %v", err.Error()) } out := podlogs.LogOutput{ StatusWriter: os.Stdout, @@ -188,7 +188,7 @@ func dumpDriverLogs() (context.CancelFunc, error) { } ctx, cancel := context.WithCancel(context.Background()) if err = podlogs.CopyAllLogs(ctx, client, getDriverNamespace(), out); err != nil { - return cancel, fmt.Errorf("failed to start pod logger: %v", err) + return cancel, fmt.Errorf("failed to start pod logger: %v", err.Error()) } return cancel, nil } diff --git a/test/k8s-integration/filter-junit.go b/test/k8s-integration/filter-junit.go index 83e39227..844df603 100644 --- a/test/k8s-integration/filter-junit.go +++ b/test/k8s-integration/filter-junit.go @@ -25,7 +25,7 @@ import ( "regexp" "strings" - "k8s.io/klog" + "k8s.io/klog/v2" ) /* @@ -78,7 +78,7 @@ func MergeJUnit(testFilter string, sourceDirectories []string, destination strin for _, dir := range sourceDirectories { files, err := ioutil.ReadDir(dir) if err != nil { - klog.Errorf("Failed to read juint directory %s: %v", dir, err) + klog.Errorf("Failed to read juint directory %s: %w", dir, err) mergeErrors = append(mergeErrors, err.Error()) continue } diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 444d2209..f1663a9d 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" apimachineryversion "k8s.io/apimachinery/pkg/util/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/klog" + "k8s.io/klog/v2" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" ) @@ -193,7 +193,7 @@ func main() { err := handle() if err != nil { - klog.Fatalf("Failed to run integration test: %v", err) + klog.Fatalf("Failed to run integration test: %w", err) } } @@ -223,18 +223,18 @@ func handle() error { oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() project := strings.TrimSpace(string(oldProject)) if err != nil { - return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err) + return fmt.Errorf("failed to get gcloud project: %s, err: %w", oldProject, err) } newproject, _ := testutils.SetupProwConfig(*boskosResourceType) err = setEnvProject(newproject) if err != nil { - return fmt.Errorf("failed to set project environment to %s: %v", newproject, err) + return fmt.Errorf("failed to set project environment to %s: %w", newproject, err) } defer func() { err = setEnvProject(string(oldProject)) if err != nil { - klog.Errorf("failed to set project environment to %s: %v", oldProject, err) + klog.Errorf("failed to set project environment to %s: %w", oldProject, err.Error()) } }() project = newproject @@ -244,7 +244,7 @@ func handle() error { if _, ok := os.LookupEnv("USER"); !ok { err = os.Setenv("USER", "prow") if err != nil { - return fmt.Errorf("failed to set user in prow to prow: %v", err) + return fmt.Errorf("failed to set user in prow to prow: %v", err.Error()) } } } @@ -254,13 +254,13 @@ func handle() error { klog.Infof("Building GCE PD CSI Driver") err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform) if err != nil { - return fmt.Errorf("failed pushing image: %v", err) + return fmt.Errorf("failed pushing image: %v", err.Error()) } defer func() { if *teardownCluster { err := deleteImage(*stagingImage, testParams.stagingVersion) if err != nil { - klog.Errorf("failed to delete image: %v", err) + klog.Errorf("failed to delete image: %w", err) } } }() @@ -276,11 +276,11 @@ func handle() error { testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *kubeVersion) if err != nil { - return fmt.Errorf("failed to download Kubernetes source: %v", err) + return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) } err = buildKubernetes(testParams.k8sSourceDir, "quick-release") if err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err) + return fmt.Errorf("failed to build Kubernetes: %v", err.Error()) } } else { testParams.k8sSourceDir = *localK8sDir @@ -292,20 +292,20 @@ func handle() error { testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *testVersion) if err != nil { - return fmt.Errorf("failed to download Kubernetes source: %v", err) + return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) } err = buildKubernetes(testParams.k8sSourceDir, "WHAT=test/e2e/e2e.test") if err != nil { - return fmt.Errorf("failed to build Kubernetes e2e: %v", err) + return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error()) } // kubetest relies on ginkgo and kubectl already built in the test k8s directory err = buildKubernetes(testParams.k8sSourceDir, "ginkgo") if err != nil { - return fmt.Errorf("failed to build gingko: %v", err) + return fmt.Errorf("failed to build gingko: %v", err.Error()) } err = buildKubernetes(testParams.k8sSourceDir, "kubectl") if err != nil { - return fmt.Errorf("failed to build kubectl: %v", err) + return fmt.Errorf("failed to build kubectl: %v", err.Error()) } } @@ -329,7 +329,7 @@ func handle() error { err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy) } if err != nil { - return fmt.Errorf("failed to cluster up: %v", err) + return fmt.Errorf("failed to cluster up: %w", err) } } @@ -340,12 +340,12 @@ func handle() error { case "gce": err := clusterDownGCE(testParams.k8sSourceDir) if err != nil { - klog.Errorf("failed to cluster down: %v", err) + klog.Errorf("failed to cluster down: %w", err) } case "gke": err := clusterDownGKE(*gceZone, *gceRegion) if err != nil { - klog.Errorf("failed to cluster down: %v", err) + klog.Errorf("failed to cluster down: %w", err) } default: klog.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy) @@ -361,14 +361,14 @@ func handle() error { nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=windows", "-o", "name") out, err := nodesCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to get windows nodes: %v", err) + return fmt.Errorf("failed to get windows nodes: %v", err.Error()) } nodes := strings.Fields(string(out)) for _, node := range nodes { taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os:NoSchedule-") out, err := taintCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to untaint windows node %s: error %v. output %s", node, err, string(out)) + return fmt.Errorf("failed to untaint windows node %s: error %v. output %s", node, err.Error(), string(out)) } klog.Infof("untaint windows nodes: %s, output %s", node, string(out)) } @@ -382,13 +382,13 @@ func handle() error { } out, err = exec.Command(filepath.Join(testParams.pkgDir, "test", "k8s-integration", "prepull-image.sh")).CombinedOutput() if err != nil { - return fmt.Errorf("failed to prepull images: %s, err: %v", out, err) + return fmt.Errorf("failed to prepull images: %s, err: %v", out, err.Error()) } out, err = exec.Command("kubectl", "describe", "pods", "-n", getDriverNamespace()).CombinedOutput() klog.Infof("describe pods \n %s", string(out)) if err != nil { - return fmt.Errorf("failed to describe pods: %v", err) + return fmt.Errorf("failed to describe pods: %v", err.Error()) } } @@ -399,19 +399,19 @@ func handle() error { if *teardownDriver { defer func() { if teardownErr := deleteDriver(testParams, *deployOverlayName); teardownErr != nil { - klog.Errorf("failed to delete driver: %v", teardownErr) + klog.Errorf("failed to delete driver: %w", teardownErr) } }() } if err != nil { - return fmt.Errorf("failed to install CSI Driver: %v", err) + return fmt.Errorf("failed to install CSI Driver: %w", err) } } // Dump all driver logs to the test artifacts cancel, err := dumpDriverLogs() if err != nil { - return fmt.Errorf("failed to start driver logging: %v", err) + return fmt.Errorf("failed to start driver logging: %w", err) } defer func() { if cancel != nil { @@ -430,7 +430,7 @@ func handle() error { nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=linux", "-o", "name") out, err := nodesCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to get linux nodes: %v", err) + return fmt.Errorf("failed to get linux nodes: %v", err.Error()) } nodes := strings.Fields(string(out)) testParams.allowedNotReadyNodes = len(nodes) @@ -438,7 +438,7 @@ func handle() error { taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os=linux:NoSchedule", "--overwrite=true") out, err := taintCmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to untaint windows node %s, error %v, output %s", node, err, string(out)) + return fmt.Errorf("failed to untaint windows node %s, error %v, output %s", node, err.Error(), string(out)) } klog.Infof("taint linux nodes: %s, output %s", node, string(out)) } @@ -448,7 +448,7 @@ func handle() error { case "gke": testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, *useKubeTest2) if err != nil { - return fmt.Errorf("failed to build GKE kubetest args: %v", err) + return fmt.Errorf("failed to build GKE kubetest args: %v", err.Error()) } case "gce": if *useKubeTest2 { @@ -638,13 +638,21 @@ func generateGKETestSkip(testParams *testParameters) string { skipString = skipString + "|volumes.should.store.data|provisioning.should.provision.storage.with.snapshot.data.source" } + // Starting in 1.24, the storage framework has a new test case: + // https://github.com/kubernetes/kubernetes/commit/4a076578451aa27e8ac60beec1fd3f23918c5331, + // which breaks the node skew tests when the node version + // is less than 1.24. + if nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.24.0")) { + skipString = skipString + "|provisioning.should.mount.multiple.PV.pointing.to.the.same.storage.on.the.same.node" + } + return skipString } func setEnvProject(project string) error { out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput() if err != nil { - return fmt.Errorf("failed to set gcloud project to %s: %s, err: %v", project, out, err) + return fmt.Errorf("failed to set gcloud project to %s: %s, err: %v", project, out, err.Error()) } err = os.Setenv("PROJECT", project) @@ -671,13 +679,13 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix if !*useKubeTest2 && len(testParams.k8sSourceDir) > 0 { err := os.Chdir(testParams.k8sSourceDir) if err != nil { - return fmt.Errorf("Failed to chdir to k8sSourceDir %s: %w", testParams.k8sSourceDir, err) + return fmt.Errorf("failed to chdir to k8sSourceDir %s: %v", testParams.k8sSourceDir, err.Error()) } } kubeconfig, err := getKubeConfig() if err != nil { - return fmt.Errorf("Failed to get kubeconfig: %w", err) + return fmt.Errorf("failed to get kubeconfig: %v", err.Error()) } os.Setenv("KUBECONFIG", kubeconfig) @@ -687,7 +695,7 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix if len(reportPrefix) > 0 { kubetestDumpDir = filepath.Join(artifactsDir, reportPrefix) if err := os.MkdirAll(kubetestDumpDir, 0755); err != nil { - return fmt.Errorf("failed to make dump dir %s: %w", kubetestDumpDir, err) + return fmt.Errorf("failed to make dump dir %s: %v", kubetestDumpDir, err.Error()) } } else { kubetestDumpDir = artifactsDir @@ -759,7 +767,7 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix klog.Infof("Copying kubernetes binaries to path=%s to run the tests", kubernetesTestBinariesPath) err := copyKubernetesTestBinaries(testParams.k8sSourceDir, kubernetesTestBinariesPath) if err != nil { - return fmt.Errorf("Failed to copy the kubernetes test binaries, err=%v", err) + return fmt.Errorf("failed to copy the kubernetes test binaries, err=%v", err.Error()) } kubeTest2Args = append(kubeTest2Args, "--use-built-binaries") } else { @@ -789,7 +797,7 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...)) } if err != nil { - return fmt.Errorf("failed to run tests on e2e cluster: %v", err) + return fmt.Errorf("failed to run tests on e2e cluster: %v", err.Error()) } return nil @@ -813,10 +821,10 @@ func copyKubernetesTestBinaries(kuberoot string, outroot string) error { if _, err := os.Stat(source); err == nil { klog.Infof("copying %s to %s", source, dest) if err := CopyFile(source, dest); err != nil { - return fmt.Errorf("failed to copy %s to %s: %v", source, dest, err) + return fmt.Errorf("failed to copy %s to %s: %v", source, dest, err.Error()) } } else { - return fmt.Errorf("could not find %s: %v", source, err) + return fmt.Errorf("could not find %s: %v", source, err.Error()) } } return nil diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go index 201baa34..e9bcf6ea 100644 --- a/test/k8s-integration/utils.go +++ b/test/k8s-integration/utils.go @@ -5,7 +5,7 @@ import ( "os" "os/exec" - "k8s.io/klog" + "k8s.io/klog/v2" ) func runCommand(action string, cmd *exec.Cmd) error { @@ -32,7 +32,7 @@ func runCommand(action string, cmd *exec.Cmd) error { func generateUniqueTmpDir() string { dir, err := ioutil.TempDir("", "gcp-pd-driver-tmp") if err != nil { - klog.Fatalf("Error creating temp dir: %v", err) + klog.Fatalf("Error creating temp dir: %w", err) } return dir } @@ -40,7 +40,7 @@ func generateUniqueTmpDir() string { func removeDir(dir string) { err := os.RemoveAll(dir) if err != nil { - klog.Fatalf("Error removing temp dir: %v", err) + klog.Fatalf("Error removing temp dir: %w", err) } } diff --git a/test/k8s-integration/version.go b/test/k8s-integration/version.go index f2fd5d14..0cbca8ee 100644 --- a/test/k8s-integration/version.go +++ b/test/k8s-integration/version.go @@ -7,17 +7,17 @@ import ( "strconv" "strings" - "k8s.io/klog" + "k8s.io/klog/v2" ) var ( versionNum = `(0|[1-9][0-9]*)` internalPatchVersion = `(\-gke\.[0-9]+)` - versionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + internalPatchVersion + "?$") - minorVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `$`) - alphaVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + `-alpha.*`) - gkeExtraVersionRegex = regexp.MustCompile(`^(?:gke)\.(0|[1-9][0-9]*)$`) + versionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + internalPatchVersion + "?$") + minorVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `$`) + prereleaseVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + `-(?:alpha|beta).*`) + gkeExtraVersionRegex = regexp.MustCompile(`^(?:gke)\.(0|[1-9][0-9]*)$`) ) type version struct { @@ -26,7 +26,7 @@ type version struct { func (v *version) String() string { if v.version[3] == -2 { - return fmt.Sprintf("%d.%d.%d-alpha", v.version[0], v.version[1], v.version[2]) + return fmt.Sprintf("%d.%d.%d-prerelease", v.version[0], v.version[1], v.version[2]) } else if v.version[3] != -1 { return fmt.Sprintf("%d.%d.%d-gke.%d", v.version[0], v.version[1], v.version[2], v.version[3]) } @@ -85,8 +85,8 @@ func parseVersion(vs string) (*version, error) { case versionRegex.MatchString(vs): submatches = versionRegex.FindStringSubmatch(vs) lastIndex = 4 - case alphaVersionRegex.MatchString(vs): - submatches = alphaVersionRegex.FindStringSubmatch(vs) + case prereleaseVersionRegex.MatchString(vs): + submatches = prereleaseVersionRegex.FindStringSubmatch(vs) v.version[3] = -2 lastIndex = 4 case minorVersionRegex.MatchString(vs): @@ -106,7 +106,7 @@ func parseVersion(vs string) (*version, error) { } } - if minorVersionRegex.MatchString(vs) || alphaVersionRegex.MatchString(vs) { + if minorVersionRegex.MatchString(vs) || prereleaseVersionRegex.MatchString(vs) { return &v, nil } diff --git a/test/k8s-integration/version_test.go b/test/k8s-integration/version_test.go index 5147d0ad..2f1d3dcd 100644 --- a/test/k8s-integration/version_test.go +++ b/test/k8s-integration/version_test.go @@ -78,6 +78,12 @@ func TestParseVersion(t *testing.T) { version: [4]int{1, 26, 0, -2}, }, }, + { + version: "v1.26.0-beta.0.25+4b1b42624e06a9", + expectedV: version{ + version: [4]int{1, 26, 0, -2}, + }, + }, // Negative test cases { version: "1", @@ -112,11 +118,11 @@ func TestParseVersion(t *testing.T) { expectErr: true, }, { - version: "1.18.0-beta.x", + version: "1.18.0-zeta.x", expectErr: true, }, { - version: "1.18.0-beta.alpha.1", + version: "1.18.0-zeta.alpha.1", expectErr: true, }, { diff --git a/test/remote/archiver.go b/test/remote/archiver.go index 6bdaf644..aacd0036 100644 --- a/test/remote/archiver.go +++ b/test/remote/archiver.go @@ -23,32 +23,32 @@ import ( "os/exec" "path/filepath" - "k8s.io/klog" + "k8s.io/klog/v2" ) func CreateDriverArchive(archiveName, architecture, pkgPath, binPath string) (string, error) { klog.V(2).Infof("Building archive...") tarDir, err := ioutil.TempDir("", "driver-temp-archive") if err != nil { - return "", fmt.Errorf("failed to create temporary directory %v", err) + return "", fmt.Errorf("failed to create temporary directory %v", err.Error()) } defer os.RemoveAll(tarDir) // Call the suite function to setup the test package. err = setupBinaries(architecture, tarDir, pkgPath, binPath) if err != nil { - return "", fmt.Errorf("failed to setup test package %q: %v", tarDir, err) + return "", fmt.Errorf("failed to setup test package %q: %v", tarDir, err.Error()) } // Build the tar out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tarDir, ".").CombinedOutput() if err != nil { - return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) + return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err.Error(), out) } dir, err := os.Getwd() if err != nil { - return "", fmt.Errorf("failed to get working directory %v", err) + return "", fmt.Errorf("failed to get working directory %v", err.Error()) } return filepath.Join(dir, archiveName), nil } @@ -57,16 +57,16 @@ func setupBinaries(architecture, tarDir, pkgPath, binPath string) error { klog.V(4).Infof("Making binaries and copying to temp dir...") out, err := exec.Command("make", "-C", pkgPath, "GOARCH="+architecture).CombinedOutput() if err != nil { - return fmt.Errorf("Failed to make at %s: %v: %v", pkgPath, string(out), err) + return fmt.Errorf("Failed to make at %s: %v: %v", pkgPath, string(out), err.Error()) } // Copy binaries if _, err := os.Stat(binPath); err != nil { - return fmt.Errorf("failed to locate test binary %s: %v", binPath, err) + return fmt.Errorf("failed to locate test binary %s: %v", binPath, err.Error()) } out, err = exec.Command("cp", binPath, tarDir).CombinedOutput() if err != nil { - return fmt.Errorf("failed to copy %q: %v Output: %q", binPath, err, out) + return fmt.Errorf("failed to copy %q: %v Output: %q", binPath, err.Error(), out) } return nil diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go index 8bfccee0..748c0918 100644 --- a/test/remote/client-wrappers.go +++ b/test/remote/client-wrappers.go @@ -21,7 +21,7 @@ import ( csipb "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "k8s.io/apimachinery/pkg/util/wait" @@ -85,7 +85,7 @@ func (c *CsiClient) AssertCSIConnection() error { return true, nil }) if err != nil || conn == nil { - return fmt.Errorf("Failed to get client connection: %v", err) + return fmt.Errorf("Failed to get client connection: %v", err.Error()) } c.conn = conn c.idClient = csipb.NewIdentityClient(conn) diff --git a/test/remote/instance.go b/test/remote/instance.go index c5eefe96..c716dcef 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -28,11 +28,12 @@ import ( "golang.org/x/oauth2/google" computealpha "google.golang.org/api/compute/v0.alpha" + computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" + "k8s.io/klog/v2" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" ) @@ -71,12 +72,11 @@ func (i *InstanceInfo) GetNodeID() string { func CreateInstanceInfo(project, instanceArchitecture, instanceZone, name, machineType string, cs *compute.Service) (*InstanceInfo, error) { return &InstanceInfo{ - project: project, - architecture: instanceArchitecture, - zone: instanceZone, - name: name, - machineType: machineType, - + project: project, + architecture: instanceArchitecture, + zone: instanceZone, + name: name, + machineType: machineType, computeService: cs, }, nil } @@ -91,7 +91,7 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro err = i.createDefaultFirewallRule() if err != nil { - return fmt.Errorf("Failed to create firewall rule: %v", err) + return fmt.Errorf("Failed to create firewall rule: %v", err.Error()) } newInst := &compute.Instance{ @@ -143,9 +143,9 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro return err } - then := time.Now() + start := time.Now() err := wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(then)) + klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(start)) if curInst, _ = i.computeService.Instances.Get(i.project, i.zone, i.name).Do(); curInst != nil { return false, nil } @@ -161,7 +161,7 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro op, err := i.computeService.Instances.Insert(i.project, i.zone, newInst).Do() klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.project, i.zone) if err != nil { - ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err) + ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err.Error()) if op != nil { ret = fmt.Sprintf("%s. op error: %v", ret, op.Error) } @@ -173,13 +173,13 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro klog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", newInst.Name) } - then := time.Now() + start := time.Now() err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(then)) + klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(start)) instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do() if err != nil { - klog.Errorf("Failed to get instance %v: %v", i.name, err) + klog.Errorf("Failed to get instance %v: %w", i.name, err) return false, nil } @@ -194,7 +194,7 @@ func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) erro } if sshOut, err := i.SSHCheckAlive(); err != nil { - err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err) + err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err.Error()) klog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) return false, nil } @@ -219,10 +219,37 @@ func (i *InstanceInfo) DeleteInstance() { if isGCEError(err, "notFound") { return } - klog.Errorf("Error deleting instance %q: %v", i.name, err) + klog.Errorf("Error deleting instance %q: %w", i.name, err) } } +func (i *InstanceInfo) DetachDisk(diskName string) error { + klog.V(4).Infof("Detaching disk %q", diskName) + op, err := i.computeService.Instances.DetachDisk(i.project, i.zone, i.name, diskName).Do() + if err != nil { + if isGCEError(err, "notFound") { + return nil + } + klog.Errorf("Error deleting disk %q: %w", diskName, err) + } + + start := time.Now() + if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { + klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.name, time.Since(start)) + + op, err = i.computeService.ZoneOperations.Get(i.project, i.zone, op.Name).Do() + if err != nil { + return true, fmt.Errorf("Failed to get operation %q, err: %v", op.Name, err) + } + return op.Status == "DONE", nil + }); err != nil { + return err + } + + klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.name, op.Error) + return nil +} + func getexternalIP(instance *compute.Instance) string { for i := range instance.NetworkInterfaces { ni := instance.NetworkInterfaces[i] @@ -262,7 +289,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error { klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) return nil } - return fmt.Errorf("Failed to insert required default SSH firewall Rule %v: %v", defaultFirewallRule, err) + return fmt.Errorf("Failed to insert required default SSH firewall Rule %v: %v", defaultFirewallRule, err.Error()) } } else { klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) @@ -330,6 +357,36 @@ func GetComputeAlphaClient() (*computealpha.Service, error) { return nil, err } +func GetComputeBetaClient() (*computebeta.Service, error) { + const retries = 10 + const backoff = time.Second * 6 + + klog.V(4).Infof("Getting compute client...") + + // Setup the gce client for provisioning instances + // Getting credentials on gce jenkins is flaky, so try a couple times + var err error + var cs *computebeta.Service + for i := 0; i < retries; i++ { + if i > 0 { + time.Sleep(backoff) + } + + var client *http.Client + client, err = google.DefaultClient(context.Background(), computebeta.ComputeScope) + if err != nil { + continue + } + + cs, err = computebeta.New(client) + if err != nil { + continue + } + return cs, nil + } + return nil, err +} + func generateMetadataWithPublicKey(pubKeyFile string) (*compute.Metadata, error) { publicKeyByte, err := ioutil.ReadFile(pubKeyFile) if err != nil { diff --git a/test/remote/runner.go b/test/remote/runner.go index 1d006c29..2fb8d8c9 100644 --- a/test/remote/runner.go +++ b/test/remote/runner.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "k8s.io/klog" + "k8s.io/klog/v2" ) func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) { @@ -34,13 +34,13 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s // Do not sudo here, so that we can use scp to copy test archive to the directdory. if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output) + return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err.Error(), output) } // Copy the archive to the staging directory if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output) + return -1, fmt.Errorf("failed to copy test archive: %v, output: %q", err.Error(), output) } // Extract the archive @@ -54,7 +54,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s // we want the extracted files to be owned by the current user. if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output) + return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err.Error(), output) } klog.V(4).Infof("Starting driver on %q", i.name) @@ -62,7 +62,7 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s output, err := i.SSH(driverRunCmd) if err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed start driver, got output: %v, error: %v", output, err) + return -1, fmt.Errorf("failed start driver, got output: %v, error: %v", output, err.Error()) } // Get the driver PID @@ -77,12 +77,12 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s driverPIDString, err := i.SSHNoSudo("sh", "-c", driverPIDCmd) if err != nil { // Exit failure with the error - return -1, fmt.Errorf("failed to get PID of driver, got output: %v, error: %v", output, err) + return -1, fmt.Errorf("failed to get PID of driver, got output: %v, error: %v", output, err.Error()) } driverPID, err := strconv.Atoi(strings.Fields(driverPIDString)[1]) if err != nil { - return -1, fmt.Errorf("failed to convert driver PID from string %s to int: %v", driverPIDString, err) + return -1, fmt.Errorf("failed to convert driver PID from string %s to int: %v", driverPIDString, err.Error()) } return driverPID, nil diff --git a/test/remote/setup-teardown.go b/test/remote/setup-teardown.go index d2a3a588..b20893c0 100644 --- a/test/remote/setup-teardown.go +++ b/test/remote/setup-teardown.go @@ -22,7 +22,7 @@ import ( compute "google.golang.org/api/compute/v1" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog" + "k8s.io/klog/v2" ) // TestContext holds the CSI Client handle to a remotely connected Driver @@ -93,13 +93,13 @@ func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*Tes // Create an SSH tunnel from port to port sshPID, err := instance.CreateSSHTunnel(config.Port, config.Port) if err != nil { - return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", sshPID, err) + return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", sshPID, err.Error()) } client := CreateCSIClient(fmt.Sprintf("localhost:%s", config.Port)) err = client.AssertCSIConnection() if err != nil { - return nil, fmt.Errorf("asserting csi connection failed with: %v", err) + return nil, fmt.Errorf("asserting csi connection failed with: %v", err.Error()) } return &TestContext{ @@ -118,22 +118,22 @@ func TeardownDriverAndClient(context *TestContext) error { // Close the client connection err := context.Client.CloseConn() if err != nil { - return fmt.Errorf("failed to close CSI Client connection: %v", err) + return fmt.Errorf("failed to close CSI Client connection: %v", err.Error()) } // Close the SSH tunnel proc, err := os.FindProcess(context.proc.sshTunnel) if err != nil { - return fmt.Errorf("unable to efind process for ssh tunnel %v: %v", context.proc.sshTunnel, err) + return fmt.Errorf("unable to efind process for ssh tunnel %v: %v", context.proc.sshTunnel, err.Error()) } if err = proc.Kill(); err != nil { - return fmt.Errorf("failed to kill ssh tunnel process %v: %v", context.proc.sshTunnel, err) + return fmt.Errorf("failed to kill ssh tunnel process %v: %v", context.proc.sshTunnel, err.Error()) } // Kill the driver process on remote cmd := fmt.Sprintf("kill %v", context.proc.remoteDriver) output, err := context.Instance.SSH(cmd) if err != nil { - return fmt.Errorf("failed to kill driver on remote instance, got output %s: %v", output, err) + return fmt.Errorf("failed to kill driver on remote instance, got output %s: %v", output, err.Error()) } return nil diff --git a/test/remote/ssh.go b/test/remote/ssh.go index 4a09acc1..521b2519 100644 --- a/test/remote/ssh.go +++ b/test/remote/ssh.go @@ -23,7 +23,7 @@ import ( "os/user" "strings" - "k8s.io/klog" + "k8s.io/klog/v2" ) var ( @@ -34,7 +34,7 @@ var ( func init() { usr, err := user.Current() if err != nil { - klog.Fatal(err) + klog.Fatal(err.Error()) } sshDefaultKey = fmt.Sprintf("%s/.ssh/google_compute_engine", usr.HomeDir) @@ -106,7 +106,7 @@ func runSSHCommand(cmd string, args ...string) (string, error) { output, err := exec.Command(cmd, args...).CombinedOutput() if err != nil { - return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err) + return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err.Error()) } return string(output), nil } diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go index 902d470b..a59cfb11 100644 --- a/test/sanity/sanity_test.go +++ b/test/sanity/sanity_test.go @@ -40,6 +40,7 @@ import ( "path" "strings" "testing" + "time" "github.com/google/uuid" "google.golang.org/grpc" @@ -47,6 +48,7 @@ import ( sanity "github.com/kubernetes-csi/csi-test/v4/pkg/sanity" compute "google.golang.org/api/compute/v1" common "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata" driver "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver" @@ -75,19 +77,19 @@ func TestSanity(t *testing.T) { cloudProvider, err := gce.CreateFakeCloudProvider(project, zone, nil) if err != nil { - t.Fatalf("Failed to get cloud provider: %v", err) + t.Fatalf("Failed to get cloud provider: %v", err.Error()) } mounter := mountmanager.NewFakeSafeMounter() - deviceUtils := mountmanager.NewFakeDeviceUtils() + deviceUtils := deviceutils.NewFakeDeviceUtils() - //Initialize GCE Driver + // Initialize GCE Driver identityServer := driver.NewIdentityServer(gceDriver) - controllerServer := driver.NewControllerServer(gceDriver, cloudProvider) + controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute) nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}, func(s string) (string, error) { return s, nil }) err = gceDriver.SetupGCEDriver(driverName, vendorVersion, extraLabels, identityServer, controllerServer, nodeServer) if err != nil { - t.Fatalf("Failed to initialize GCE CSI Driver: %v", err) + t.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) } instance := &compute.Instance{ @@ -98,18 +100,18 @@ func TestSanity(t *testing.T) { err = os.MkdirAll(tmpDir, 0755) if err != nil { - t.Fatalf("Failed to create sanity temp working dir %s: %v", tmpDir, err) + t.Fatalf("Failed to create sanity temp working dir %s: %v", tmpDir, err.Error()) } defer func() { // Clean up tmp dir if err = os.RemoveAll(tmpDir); err != nil { - t.Fatalf("Failed to clean up sanity temp working dir %s: %v", tmpDir, err) + t.Fatalf("Failed to clean up sanity temp working dir %s: %v", tmpDir, err.Error()) } }() go func() { - gceDriver.Run(endpoint) + gceDriver.Run(endpoint, 10000) }() // TODO(#818): Fix failing tests and remove test skip flag.