From 3493bd4cc44cf25f08b7cba341908cca060d4b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= Date: Tue, 18 Jun 2024 13:14:48 +0200 Subject: [PATCH] Rebase to upstream v1.14.0 and prepare release v1.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update chart * Bump Constellation dependency * Remove e2e Go tests, as they were unused and created dependency problems * Remove dependabot config Signed-off-by: Daniel Weiße --- CHANGELOG/CHANGELOG-1.10.md | 242 +++ CHANGELOG/CHANGELOG-1.11.md | 257 +++ CHANGELOG/CHANGELOG-1.12.md | 291 +++ CHANGELOG/CHANGELOG-1.13.md | 371 ++++ CHANGELOG/CHANGELOG-1.7.md | 115 ++ CHANGELOG/CHANGELOG-1.8.md | 182 ++ CHANGELOG/CHANGELOG-1.9.md | 262 +++ Dockerfile | 39 +- Dockerfile.Windows | 2 +- Dockerfile.debug | 2 +- OWNERS | 4 + README.md | 3 +- charts/Chart.yaml | 4 +- charts/values.yaml | 12 +- cmd/gce-pd-csi-driver/main.go | 141 +- .../prow-stable-sidecar-rc-master/image.yaml | 2 +- .../images/stable-master/image.yaml | 12 +- docs/release/overlays.md | 2 +- .../demo-sc-with-resource-tags.yaml | 4 +- .../volumesnapshotclass-resource-tags.yaml | 7 +- go.mod | 128 +- go.sum | 1706 ++--------------- hack/generate-patch-release-notes.sh | 97 + pkg/common/constants.go | 8 + pkg/common/errors.go | 58 + pkg/common/parameters.go | 86 +- pkg/common/parameters_test.go | 226 ++- pkg/common/utils.go | 323 +++- pkg/common/utils_test.go | 633 ++++++ pkg/deviceutils/device-utils.go | 40 +- pkg/deviceutils/device-utils_linux.go | 23 +- pkg/deviceutils/device-utils_test.go | 68 + pkg/deviceutils/device-utils_windows.go | 8 + pkg/deviceutils/fake-device-utils.go | 11 +- pkg/gce-cloud-provider/compute/cloud-disk.go | 35 +- .../compute/cloud-disk_test.go | 122 +- pkg/gce-cloud-provider/compute/fake-gce.go | 81 +- pkg/gce-cloud-provider/compute/gce-compute.go | 469 ++++- .../compute/gce-compute_test.go | 2 +- pkg/gce-cloud-provider/compute/gce.go | 163 +- pkg/gce-cloud-provider/compute/gce_test.go | 74 + pkg/gce-pd-csi-driver/controller.go | 793 ++++++-- pkg/gce-pd-csi-driver/controller_test.go | 1574 +++++++++++++-- pkg/gce-pd-csi-driver/gce-pd-driver.go | 26 +- pkg/gce-pd-csi-driver/gce-pd-driver_test.go | 8 +- pkg/gce-pd-csi-driver/identity.go | 2 +- pkg/gce-pd-csi-driver/identity_test.go | 6 +- pkg/gce-pd-csi-driver/node.go | 90 +- pkg/gce-pd-csi-driver/node_test.go | 205 +- pkg/gce-pd-csi-driver/server.go | 17 +- pkg/gce-pd-csi-driver/trace.go | 56 + pkg/gce-pd-csi-driver/utils.go | 70 + pkg/gce-pd-csi-driver/utils_linux.go | 21 + pkg/gce-pd-csi-driver/utils_test.go | 432 +++++ pkg/gce-pd-csi-driver/utils_windows.go | 5 + pkg/metrics/metrics.go | 36 +- pkg/metrics/metrics_test.go | 142 +- pkg/mount-manager/statter_linux.go | 24 +- release-tools/SIDECAR_RELEASE_PROCESS.md | 8 +- release-tools/filter-junit.go | 5 +- test/e2e/tests/multi_zone_e2e_test.go | 410 ---- test/e2e/tests/resize_e2e_test.go | 354 ---- test/e2e/tests/setup_e2e_test.go | 157 -- test/e2e/tests/single_zone_e2e_test.go | 1519 --------------- test/e2e/utils/utils.go | 315 --- test/k8s-integration/cluster.go | 493 ----- test/k8s-integration/cluster_test.go | 71 - .../config/pd-volumesnapshotclass.yaml | 6 - test/k8s-integration/config/sc-balanced.yaml | 8 - test/k8s-integration/config/sc-extreme.yaml | 11 - test/k8s-integration/config/sc-regional.yaml | 9 - .../config/sc-standard-no-labels.yaml | 8 - test/k8s-integration/config/sc-standard.yaml | 10 - test/k8s-integration/config/sc-windows.yaml | 9 - test/k8s-integration/config/sc-xfs.yaml | 9 - .../config/test-config-template.in | 31 - test/k8s-integration/driver-config.go | 163 -- test/k8s-integration/driver.go | 220 --- test/k8s-integration/filter-junit.go | 143 -- test/k8s-integration/fs.go | 71 - test/k8s-integration/main.go | 892 --------- test/k8s-integration/podlogs/podlogs.go | 262 --- test/k8s-integration/prepull-image.sh | 43 - test/k8s-integration/prepull.yaml | 41 - test/k8s-integration/utils.go | 104 - test/k8s-integration/version.go | 170 -- test/k8s-integration/version_test.go | 344 ---- test/remote/archiver.go | 73 - test/remote/client-wrappers.go | 311 --- test/remote/instance.go | 429 ----- test/remote/runner.go | 93 - test/remote/setup-teardown.go | 140 -- test/remote/ssh.go | 112 -- test/run-e2e-local.sh | 7 +- test/run-k8s-integration-ci.sh | 9 + test/run-k8s-integration.sh | 4 +- test/sanity/sanity_test.go | 14 +- 97 files changed, 7720 insertions(+), 9180 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-1.12.md create mode 100644 CHANGELOG/CHANGELOG-1.13.md rename test/k8s-integration/config/sc-ssd.yaml => examples/kubernetes/demo-sc-with-resource-tags.yaml (60%) rename test/k8s-integration/config/image-volumesnapshotclass.yaml => examples/kubernetes/snapshot/volumesnapshotclass-resource-tags.yaml (56%) create mode 100755 hack/generate-patch-release-notes.sh create mode 100644 pkg/common/errors.go create mode 100644 pkg/gce-pd-csi-driver/trace.go delete mode 100644 test/e2e/tests/multi_zone_e2e_test.go delete mode 100644 test/e2e/tests/resize_e2e_test.go delete mode 100644 test/e2e/tests/setup_e2e_test.go delete mode 100644 test/e2e/tests/single_zone_e2e_test.go delete mode 100644 test/e2e/utils/utils.go delete mode 100644 test/k8s-integration/cluster.go delete mode 100644 test/k8s-integration/cluster_test.go delete mode 100644 test/k8s-integration/config/pd-volumesnapshotclass.yaml delete mode 100644 test/k8s-integration/config/sc-balanced.yaml delete mode 100644 test/k8s-integration/config/sc-extreme.yaml delete mode 100644 test/k8s-integration/config/sc-regional.yaml delete mode 100644 test/k8s-integration/config/sc-standard-no-labels.yaml delete mode 100644 test/k8s-integration/config/sc-standard.yaml delete mode 100644 test/k8s-integration/config/sc-windows.yaml delete mode 100644 test/k8s-integration/config/sc-xfs.yaml delete mode 100644 test/k8s-integration/config/test-config-template.in delete mode 100644 test/k8s-integration/driver-config.go delete mode 100644 test/k8s-integration/driver.go delete mode 100644 test/k8s-integration/filter-junit.go delete mode 100644 test/k8s-integration/fs.go delete mode 100644 test/k8s-integration/main.go delete mode 100644 test/k8s-integration/podlogs/podlogs.go delete mode 100755 test/k8s-integration/prepull-image.sh delete mode 100644 test/k8s-integration/prepull.yaml delete mode 100644 test/k8s-integration/utils.go delete mode 100644 test/k8s-integration/version.go delete mode 100644 test/k8s-integration/version_test.go delete mode 100644 test/remote/archiver.go delete mode 100644 test/remote/client-wrappers.go delete mode 100644 test/remote/instance.go delete mode 100644 test/remote/runner.go delete mode 100644 test/remote/setup-teardown.go delete mode 100644 test/remote/ssh.go diff --git a/CHANGELOG/CHANGELOG-1.10.md b/CHANGELOG/CHANGELOG-1.10.md index f8c8107b..36c1ac18 100644 --- a/CHANGELOG/CHANGELOG-1.10.md +++ b/CHANGELOG/CHANGELOG-1.10.md @@ -1,3 +1,245 @@ +# v1.10.14 - Changelog since v1.10.13 + +## Changes by Kind + +### Bug + +- Change GetDisk error reporting to temporary in CreateVolume codepath ([#1602])https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1602), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +# v1.10.13 - Changelog since v1.10.12 + +## Changes by Kind + +### Uncategorized + +- Bump golang.org/x/crypto from v0.14.0 to v0.17.0 to fix CVE-2023-48795 ([#1551](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1551), [@sunnylovestiramisu](https://github.com/sunnylovestiramisu)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- golang.org/x/crypto: v0.14.0 → v0.17.0 +- golang.org/x/sys: v0.13.0 → v0.15.0 +- golang.org/x/term: v0.13.0 → v0.15.0 +- golang.org/x/text: v0.13.0 → v0.14.0 + +### Removed +_Nothing has changed._ + + +# v1.10.12 - Changelog since v1.10.11 + +## Changes by Kind + +### Uncategorized + +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1517](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1517), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1545](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1545), [@pwschuurman](https://github.com/pwschuurman)) +- Update golang builder to 1.20.12 ([#1541](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1541), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.10.11 - Changelog since v1.10.10 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1506](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1506), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.53.0 to v1.56.3 to fix CVE-2023-44487. ([#1494](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1494), [@uriel-guzman](https://github.com/uriel-guzman)) + +## Dependencies + +### Added +- github.com/google/go-pkcs11: [v0.2.0](https://github.com/google/go-pkcs11/tree/v0.2.0) +- github.com/google/s2a-go: [v0.1.4](https://github.com/google/s2a-go/tree/v0.1.4) +- google.golang.org/genproto/googleapis/api: ccb25ca +- google.golang.org/genproto/googleapis/bytestream: 659f7aa +- google.golang.org/genproto/googleapis/rpc: 659f7aa + +### Changed +- cloud.google.com/go/accessapproval: v1.5.0 → v1.7.1 +- cloud.google.com/go/accesscontextmanager: v1.4.0 → v1.8.1 +- cloud.google.com/go/aiplatform: v1.27.0 → v1.45.0 +- cloud.google.com/go/analytics: v0.12.0 → v0.21.2 +- cloud.google.com/go/apigateway: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeconnect: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeregistry: v0.4.0 → v0.7.1 +- cloud.google.com/go/appengine: v1.5.0 → v1.8.1 +- cloud.google.com/go/area120: v0.6.0 → v0.8.1 +- cloud.google.com/go/artifactregistry: v1.9.0 → v1.14.1 +- cloud.google.com/go/asset: v1.10.0 → v1.14.1 +- cloud.google.com/go/assuredworkloads: v1.9.0 → v1.11.1 +- cloud.google.com/go/automl: v1.8.0 → v1.13.1 +- cloud.google.com/go/baremetalsolution: v0.4.0 → v0.5.0 +- cloud.google.com/go/batch: v0.4.0 → v0.7.0 +- cloud.google.com/go/beyondcorp: v0.3.0 → v0.6.1 +- cloud.google.com/go/bigquery: v1.44.0 → v1.52.0 +- cloud.google.com/go/billing: v1.7.0 → v1.16.0 +- cloud.google.com/go/binaryauthorization: v1.4.0 → v1.6.1 +- cloud.google.com/go/certificatemanager: v1.4.0 → v1.7.1 +- cloud.google.com/go/channel: v1.9.0 → v1.16.0 +- cloud.google.com/go/cloudbuild: v1.4.0 → v1.10.1 +- cloud.google.com/go/clouddms: v1.4.0 → v1.6.1 +- cloud.google.com/go/cloudtasks: v1.8.0 → v1.11.1 +- cloud.google.com/go/compute: v1.18.0 → v1.20.1 +- cloud.google.com/go/contactcenterinsights: v1.4.0 → v1.9.1 +- cloud.google.com/go/container: v1.7.0 → v1.22.1 +- cloud.google.com/go/containeranalysis: v0.6.0 → v0.10.1 +- cloud.google.com/go/datacatalog: v1.8.0 → v1.14.1 +- cloud.google.com/go/dataflow: v0.7.0 → v0.9.1 +- cloud.google.com/go/dataform: v0.5.0 → v0.8.1 +- cloud.google.com/go/datafusion: v1.5.0 → v1.7.1 +- cloud.google.com/go/datalabeling: v0.6.0 → v0.8.1 +- cloud.google.com/go/dataplex: v1.4.0 → v1.8.1 +- cloud.google.com/go/dataproc: v1.8.0 → v1.12.0 +- cloud.google.com/go/dataqna: v0.6.0 → v0.8.1 +- cloud.google.com/go/datastore: v1.10.0 → v1.12.0 +- cloud.google.com/go/datastream: v1.5.0 → v1.9.1 +- cloud.google.com/go/deploy: v1.5.0 → v1.11.0 +- cloud.google.com/go/dialogflow: v1.29.0 → v1.38.0 +- cloud.google.com/go/dlp: v1.7.0 → v1.10.1 +- cloud.google.com/go/documentai: v1.10.0 → v1.20.0 +- cloud.google.com/go/domains: v0.7.0 → v0.9.1 +- cloud.google.com/go/edgecontainer: v0.2.0 → v1.1.1 +- cloud.google.com/go/essentialcontacts: v1.4.0 → v1.6.2 +- cloud.google.com/go/eventarc: v1.8.0 → v1.12.1 +- cloud.google.com/go/filestore: v1.4.0 → v1.7.1 +- cloud.google.com/go/firestore: v1.9.0 → v1.11.0 +- cloud.google.com/go/functions: v1.9.0 → v1.15.1 +- cloud.google.com/go/gaming: v1.8.0 → v1.10.1 +- cloud.google.com/go/gkebackup: v0.3.0 → v0.4.0 +- cloud.google.com/go/gkeconnect: v0.6.0 → v0.8.1 +- cloud.google.com/go/gkehub: v0.10.0 → v0.14.1 +- cloud.google.com/go/gkemulticloud: v0.4.0 → v0.6.1 +- cloud.google.com/go/gsuiteaddons: v1.4.0 → v1.6.1 +- cloud.google.com/go/iam: v0.11.0 → v1.1.0 +- cloud.google.com/go/iap: v1.5.0 → v1.8.1 +- cloud.google.com/go/ids: v1.2.0 → v1.4.1 +- cloud.google.com/go/iot: v1.4.0 → v1.7.1 +- cloud.google.com/go/kms: v1.6.0 → v1.12.1 +- cloud.google.com/go/language: v1.8.0 → v1.10.1 +- cloud.google.com/go/lifesciences: v0.6.0 → v0.9.1 +- cloud.google.com/go/logging: v1.6.1 → v1.7.0 +- cloud.google.com/go/longrunning: v0.3.0 → v0.5.1 +- cloud.google.com/go/managedidentities: v1.4.0 → v1.6.1 +- cloud.google.com/go/maps: v0.1.0 → v0.7.0 +- cloud.google.com/go/mediatranslation: v0.6.0 → v0.8.1 +- cloud.google.com/go/memcache: v1.7.0 → v1.10.1 +- cloud.google.com/go/metastore: v1.8.0 → v1.11.1 +- cloud.google.com/go/monitoring: v1.8.0 → v1.15.1 +- cloud.google.com/go/networkconnectivity: v1.7.0 → v1.12.1 +- cloud.google.com/go/networkmanagement: v1.5.0 → v1.8.0 +- cloud.google.com/go/networksecurity: v0.6.0 → v0.9.1 +- cloud.google.com/go/notebooks: v1.5.0 → v1.9.1 +- cloud.google.com/go/optimization: v1.2.0 → v1.4.1 +- cloud.google.com/go/orchestration: v1.4.0 → v1.8.1 +- cloud.google.com/go/orgpolicy: v1.5.0 → v1.11.1 +- cloud.google.com/go/osconfig: v1.10.0 → v1.12.1 +- cloud.google.com/go/oslogin: v1.7.0 → v1.10.1 +- cloud.google.com/go/phishingprotection: v0.6.0 → v0.8.1 +- cloud.google.com/go/policytroubleshooter: v1.4.0 → v1.7.1 +- cloud.google.com/go/privatecatalog: v0.6.0 → v0.9.1 +- cloud.google.com/go/pubsub: v1.27.1 → v1.32.0 +- cloud.google.com/go/pubsublite: v1.5.0 → v1.8.1 +- cloud.google.com/go/recaptchaenterprise/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/recommendationengine: v0.6.0 → v0.8.1 +- cloud.google.com/go/recommender: v1.8.0 → v1.10.1 +- cloud.google.com/go/redis: v1.10.0 → v1.13.1 +- cloud.google.com/go/resourcemanager: v1.4.0 → v1.9.1 +- cloud.google.com/go/resourcesettings: v1.4.0 → v1.6.1 +- cloud.google.com/go/retail: v1.11.0 → v1.14.1 +- cloud.google.com/go/run: v0.3.0 → v0.9.0 +- cloud.google.com/go/scheduler: v1.7.0 → v1.10.1 +- cloud.google.com/go/secretmanager: v1.9.0 → v1.11.1 +- cloud.google.com/go/security: v1.10.0 → v1.15.1 +- cloud.google.com/go/securitycenter: v1.16.0 → v1.23.0 +- cloud.google.com/go/servicedirectory: v1.7.0 → v1.10.1 +- cloud.google.com/go/shell: v1.4.0 → v1.7.1 +- cloud.google.com/go/spanner: v1.41.0 → v1.47.0 +- cloud.google.com/go/speech: v1.9.0 → v1.17.1 +- cloud.google.com/go/storagetransfer: v1.6.0 → v1.10.0 +- cloud.google.com/go/talent: v1.4.0 → v1.6.2 +- cloud.google.com/go/texttospeech: v1.5.0 → v1.7.1 +- cloud.google.com/go/tpu: v1.4.0 → v1.6.1 +- cloud.google.com/go/trace: v1.4.0 → v1.10.1 +- cloud.google.com/go/translate: v1.4.0 → v1.8.1 +- cloud.google.com/go/video: v1.9.0 → v1.17.1 +- cloud.google.com/go/videointelligence: v1.9.0 → v1.11.1 +- cloud.google.com/go/vision/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/vmmigration: v1.3.0 → v1.7.1 +- cloud.google.com/go/vmwareengine: v0.1.0 → v0.4.1 +- cloud.google.com/go/vpcaccess: v1.5.0 → v1.7.1 +- cloud.google.com/go/webrisk: v1.7.0 → v1.9.1 +- cloud.google.com/go/websecurityscanner: v1.4.0 → v1.6.1 +- cloud.google.com/go/workflows: v1.9.0 → v1.11.1 +- cloud.google.com/go: v0.107.0 → v0.110.4 +- github.com/cncf/xds/go: [06c439d → e9ce688](https://github.com/cncf/xds/go/compare/06c439d...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.10.3 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.10.3...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.9.1 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.9.1...v0.10.1) +- github.com/golang/glog: [v1.0.0 → v1.1.0](https://github.com/golang/glog/compare/v1.0.0...v1.1.0) +- github.com/golang/protobuf: [v1.5.2 → v1.5.3](https://github.com/golang/protobuf/compare/v1.5.2...v1.5.3) +- github.com/googleapis/enterprise-certificate-proxy: [v0.2.3 → v0.2.5](https://github.com/googleapis/enterprise-certificate-proxy/compare/v0.2.3...v0.2.5) +- github.com/googleapis/gax-go/v2: [v2.7.0 → v2.12.0](https://github.com/googleapis/gax-go/v2/compare/v2.7.0...v2.12.0) +- github.com/yuin/goldmark: [v1.4.1 → v1.4.13](https://github.com/yuin/goldmark/compare/v1.4.1...v1.4.13) +- golang.org/x/oauth2: v0.5.0 → v0.10.0 +- golang.org/x/sync: v0.1.0 → v0.3.0 +- google.golang.org/api: v0.111.0 → v0.134.0 +- google.golang.org/genproto: 637eb22 → ccb25ca +- google.golang.org/grpc: v1.53.0 → v1.56.3 +- google.golang.org/protobuf: v1.28.1 → v1.31.0 + +### Removed +- cloud.google.com/go/apikeys: v0.4.0 +- cloud.google.com/go/servicecontrol: v1.5.0 +- cloud.google.com/go/servicemanagement: v1.5.0 +- cloud.google.com/go/serviceusage: v1.4.0 + +# v1.10.10 - Changelog since v1.10.8 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Changed + +- golang.org/x/crypto: v0.11.0 → v0.14.0 +- golang.org/x/net: v0.12.0 → v0.17.0 +- golang.org/x/sys: v0.10.0 → v0.13.0 +- golang.org/x/term: v0.10.0 → v0.13.0 +- golang.org/x/text: v0.11.0 → v0.13.0 + +# v1.10.8 - Changelog since v1.10.7 + +## Changes by Kind + +### Bug or Regression + +- bump go version to 1.20.8 ([#1394](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1394), [@tyuchn](https://github.com/tyuchn)) +- Remove ARG BUILDPLATFORM from Dockerfile ([#1385](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1385), [@tyuchn](https://github.com/tyuchn)) +- Update test/run-e2e.sh to match PROW configuration ([#1360](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1360), [@pwschuurman](https://github.com/pwschuurman)) +- Always call LoggedError for errors returned from CloudProvider methods ([#1381](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1381), [@amacaskill](https://github.com/amacaskill)) + # v1.10.7 - Changelog since v1.10.6 ## Changes by Kind diff --git a/CHANGELOG/CHANGELOG-1.11.md b/CHANGELOG/CHANGELOG-1.11.md index b88812f5..5f3985fa 100644 --- a/CHANGELOG/CHANGELOG-1.11.md +++ b/CHANGELOG/CHANGELOG-1.11.md @@ -1,3 +1,260 @@ +# v1.11.9 - Changelog since v1.11.8 + +## Changes by Kind + +### Bug + +- Change GetDisk error reporting to temporary in CreateVolume codepath ([#1601])https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1601), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +# v1.11.8 - Changelog since v1.11.7 + +## Changes by Kind + +### Uncategorized + +- Bump golang.org/x/crypto from v0.14.0 to v0.17.0 to fix CVE-2023-48795 ([#1555](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1555), [@sunnylovestiramisu](https://github.com/sunnylovestiramisu)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- golang.org/x/crypto: v0.14.0 → v0.17.0 +- golang.org/x/sys: v0.13.0 → v0.15.0 +- golang.org/x/term: v0.13.0 → v0.15.0 +- golang.org/x/text: v0.13.0 → v0.14.0 + +### Removed +_Nothing has changed._ + + +# v1.11.7 - Changelog since v1.11.6 + +## Changes by Kind + +### Uncategorized + +- Update golang builder to 1.20.12 ([#1540](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1540), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.11.6 - Changelog since v1.11.5 + +## Changes by Kind + +### Uncategorized + +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1516](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1516), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.11.5 - Changelog since v1.11.4 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1507](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1507), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.56.2 to v1.56.3 to fix CVE-2023-44487. ([#1491](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1491), [@uriel-guzman](https://github.com/uriel-guzman)) + +### Uncategorized + +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1488](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1488), [@pwschuurman](https://github.com/pwschuurman)) +- The benign error when DisableDevice is not effective is logged as a warning. ([#1469](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1469), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- google.golang.org/grpc: v1.56.2 → v1.56.3 + +### Removed +_Nothing has changed._ +pigateway: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeconnect: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeregistry: v0.4.0 → v0.7.1 +- cloud.google.com/go/appengine: v1.5.0 → v1.8.1 +- cloud.google.com/go/area120: v0.6.0 → v0.8.1 +- cloud.google.com/go/artifactregistry: v1.9.0 → v1.14.1 +- cloud.google.com/go/asset: v1.10.0 → v1.14.1 +- cloud.google.com/go/assuredworkloads: v1.9.0 → v1.11.1 +- cloud.google.com/go/automl: v1.8.0 → v1.13.1 +- cloud.google.com/go/baremetalsolution: v0.4.0 → v0.5.0 +- cloud.google.com/go/batch: v0.4.0 → v0.7.0 +- cloud.google.com/go/beyondcorp: v0.3.0 → v0.6.1 +- cloud.google.com/go/bigquery: v1.44.0 → v1.52.0 +- cloud.google.com/go/billing: v1.7.0 → v1.16.0 +- cloud.google.com/go/binaryauthorization: v1.4.0 → v1.6.1 +- cloud.google.com/go/certificatemanager: v1.4.0 → v1.7.1 +- cloud.google.com/go/channel: v1.9.0 → v1.16.0 +- cloud.google.com/go/cloudbuild: v1.4.0 → v1.10.1 +- cloud.google.com/go/clouddms: v1.4.0 → v1.6.1 +- cloud.google.com/go/cloudtasks: v1.8.0 → v1.11.1 +- cloud.google.com/go/compute: v1.18.0 → v1.20.1 +- cloud.google.com/go/contactcenterinsights: v1.4.0 → v1.9.1 +- cloud.google.com/go/container: v1.7.0 → v1.22.1 +- cloud.google.com/go/containeranalysis: v0.6.0 → v0.10.1 +- cloud.google.com/go/datacatalog: v1.8.0 → v1.14.1 +- cloud.google.com/go/dataflow: v0.7.0 → v0.9.1 +- cloud.google.com/go/dataform: v0.5.0 → v0.8.1 +- cloud.google.com/go/datafusion: v1.5.0 → v1.7.1 +- cloud.google.com/go/datalabeling: v0.6.0 → v0.8.1 +- cloud.google.com/go/dataplex: v1.4.0 → v1.8.1 +- cloud.google.com/go/dataproc: v1.8.0 → v1.12.0 +- cloud.google.com/go/dataqna: v0.6.0 → v0.8.1 +- cloud.google.com/go/datastore: v1.10.0 → v1.12.0 +- cloud.google.com/go/datastream: v1.5.0 → v1.9.1 +- cloud.google.com/go/deploy: v1.5.0 → v1.11.0 +- cloud.google.com/go/dialogflow: v1.29.0 → v1.38.0 +- cloud.google.com/go/dlp: v1.7.0 → v1.10.1 +- cloud.google.com/go/documentai: v1.10.0 → v1.20.0 +- cloud.google.com/go/domains: v0.7.0 → v0.9.1 +- cloud.google.com/go/edgecontainer: v0.2.0 → v1.1.1 +- cloud.google.com/go/essentialcontacts: v1.4.0 → v1.6.2 +- cloud.google.com/go/eventarc: v1.8.0 → v1.12.1 +- cloud.google.com/go/filestore: v1.4.0 → v1.7.1 +- cloud.google.com/go/firestore: v1.9.0 → v1.11.0 +- cloud.google.com/go/functions: v1.9.0 → v1.15.1 +- cloud.google.com/go/gaming: v1.8.0 → v1.10.1 +- cloud.google.com/go/gkebackup: v0.3.0 → v0.4.0 +- cloud.google.com/go/gkeconnect: v0.6.0 → v0.8.1 +- cloud.google.com/go/gkehub: v0.10.0 → v0.14.1 +- cloud.google.com/go/gkemulticloud: v0.4.0 → v0.6.1 +- cloud.google.com/go/gsuiteaddons: v1.4.0 → v1.6.1 +- cloud.google.com/go/iam: v0.11.0 → v1.1.0 +- cloud.google.com/go/iap: v1.5.0 → v1.8.1 +- cloud.google.com/go/ids: v1.2.0 → v1.4.1 +- cloud.google.com/go/iot: v1.4.0 → v1.7.1 +- cloud.google.com/go/kms: v1.6.0 → v1.12.1 +- cloud.google.com/go/language: v1.8.0 → v1.10.1 +- cloud.google.com/go/lifesciences: v0.6.0 → v0.9.1 +- cloud.google.com/go/logging: v1.6.1 → v1.7.0 +- cloud.google.com/go/longrunning: v0.3.0 → v0.5.1 +- cloud.google.com/go/managedidentities: v1.4.0 → v1.6.1 +- cloud.google.com/go/maps: v0.1.0 → v0.7.0 +- cloud.google.com/go/mediatranslation: v0.6.0 → v0.8.1 +- cloud.google.com/go/memcache: v1.7.0 → v1.10.1 +- cloud.google.com/go/metastore: v1.8.0 → v1.11.1 +- cloud.google.com/go/monitoring: v1.8.0 → v1.15.1 +- cloud.google.com/go/networkconnectivity: v1.7.0 → v1.12.1 +- cloud.google.com/go/networkmanagement: v1.5.0 → v1.8.0 +- cloud.google.com/go/networksecurity: v0.6.0 → v0.9.1 +- cloud.google.com/go/notebooks: v1.5.0 → v1.9.1 +- cloud.google.com/go/optimization: v1.2.0 → v1.4.1 +- cloud.google.com/go/orchestration: v1.4.0 → v1.8.1 +- cloud.google.com/go/orgpolicy: v1.5.0 → v1.11.1 +- cloud.google.com/go/osconfig: v1.10.0 → v1.12.1 +- cloud.google.com/go/oslogin: v1.7.0 → v1.10.1 +- cloud.google.com/go/phishingprotection: v0.6.0 → v0.8.1 +- cloud.google.com/go/policytroubleshooter: v1.4.0 → v1.7.1 +- cloud.google.com/go/privatecatalog: v0.6.0 → v0.9.1 +- cloud.google.com/go/pubsub: v1.27.1 → v1.32.0 +- cloud.google.com/go/pubsublite: v1.5.0 → v1.8.1 +- cloud.google.com/go/recaptchaenterprise/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/recommendationengine: v0.6.0 → v0.8.1 +- cloud.google.com/go/recommender: v1.8.0 → v1.10.1 +- cloud.google.com/go/redis: v1.10.0 → v1.13.1 +- cloud.google.com/go/resourcemanager: v1.4.0 → v1.9.1 +- cloud.google.com/go/resourcesettings: v1.4.0 → v1.6.1 +- cloud.google.com/go/retail: v1.11.0 → v1.14.1 +- cloud.google.com/go/run: v0.3.0 → v0.9.0 +- cloud.google.com/go/scheduler: v1.7.0 → v1.10.1 +- cloud.google.com/go/secretmanager: v1.9.0 → v1.11.1 +- cloud.google.com/go/security: v1.10.0 → v1.15.1 +- cloud.google.com/go/securitycenter: v1.16.0 → v1.23.0 +- cloud.google.com/go/servicedirectory: v1.7.0 → v1.10.1 +- cloud.google.com/go/shell: v1.4.0 → v1.7.1 +- cloud.google.com/go/spanner: v1.41.0 → v1.47.0 +- cloud.google.com/go/speech: v1.9.0 → v1.17.1 +- cloud.google.com/go/storagetransfer: v1.6.0 → v1.10.0 +- cloud.google.com/go/talent: v1.4.0 → v1.6.2 +- cloud.google.com/go/texttospeech: v1.5.0 → v1.7.1 +- cloud.google.com/go/tpu: v1.4.0 → v1.6.1 +- cloud.google.com/go/trace: v1.4.0 → v1.10.1 +- cloud.google.com/go/translate: v1.4.0 → v1.8.1 +- cloud.google.com/go/video: v1.9.0 → v1.17.1 +- cloud.google.com/go/videointelligence: v1.9.0 → v1.11.1 +- cloud.google.com/go/vision/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/vmmigration: v1.3.0 → v1.7.1 +- cloud.google.com/go/vmwareengine: v0.1.0 → v0.4.1 +- cloud.google.com/go/vpcaccess: v1.5.0 → v1.7.1 +- cloud.google.com/go/webrisk: v1.7.0 → v1.9.1 +- cloud.google.com/go/websecurityscanner: v1.4.0 → v1.6.1 +- cloud.google.com/go/workflows: v1.9.0 → v1.11.1 +- cloud.google.com/go: v0.107.0 → v0.110.4 +- github.com/cncf/xds/go: [06c439d → e9ce688](https://github.com/cncf/xds/go/compare/06c439d...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.10.3 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.10.3...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.9.1 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.9.1...v0.10.1) +- github.com/golang/glog: [v1.0.0 → v1.1.0](https://github.com/golang/glog/compare/v1.0.0...v1.1.0) +- github.com/golang/protobuf: [v1.5.2 → v1.5.3](https://github.com/golang/protobuf/compare/v1.5.2...v1.5.3) +- github.com/googleapis/enterprise-certificate-proxy: [v0.2.3 → v0.2.5](https://github.com/googleapis/enterprise-certificate-proxy/compare/v0.2.3...v0.2.5) +- github.com/googleapis/gax-go/v2: [v2.7.0 → v2.12.0](https://github.com/googleapis/gax-go/v2/compare/v2.7.0...v2.12.0) +- github.com/yuin/goldmark: [v1.4.1 → v1.4.13](https://github.com/yuin/goldmark/compare/v1.4.1...v1.4.13) +- golang.org/x/oauth2: v0.5.0 → v0.10.0 +- golang.org/x/sync: v0.1.0 → v0.3.0 +- google.golang.org/api: v0.111.0 → v0.134.0 +- google.golang.org/genproto: 637eb22 → ccb25ca +- google.golang.org/grpc: v1.53.0 → v1.56.3 +- google.golang.org/protobuf: v1.28.1 → v1.31.0 + +### Removed +- cloud.google.com/go/apikeys: v0.4.0 +- cloud.google.com/go/servicecontrol: v1.5.0 +- cloud.google.com/go/servicemanagement: v1.5.0 +- cloud.google.com/go/serviceusage: v1.4.0 + +# v1.11.4 - Changelog since v1.11.2 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Changed + +- golang.org/x/crypto: v0.11.0 → v0.14.0 +- golang.org/x/net: v0.12.0 → v0.17.0 +- golang.org/x/sys: v0.10.0 → v0.13.0 +- golang.org/x/term: v0.10.0 → v0.13.0 +- golang.org/x/text: v0.11.0 → v0.13.0 + +# v1.11.2 - Changelog since v1.11.1 +## Changes by Kind + +### Bug or Regression + +- bump go version to 1.20.8 ([#1394](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1394), [@tyuchn](https://github.com/tyuchn)) +- Remove ARG BUILDPLATFORM from Dockerfile ([#1385](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1385), [@tyuchn](https://github.com/tyuchn)) +- Update test/run-e2e.sh to match PROW configuration ([#1360](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1360), [@pwschuurman](https://github.com/pwschuurman)) + # v1.11.1 - Changelog since v1.11.0 ## Changes by Kind diff --git a/CHANGELOG/CHANGELOG-1.12.md b/CHANGELOG/CHANGELOG-1.12.md new file mode 100644 index 00000000..c478d481 --- /dev/null +++ b/CHANGELOG/CHANGELOG-1.12.md @@ -0,0 +1,291 @@ +# v1.12.8 - Changelog since v1.12.7 + +## Changes by Kind + +### Bug + +- Change GetDisk error reporting to temporary in CreateVolume codepath ([#1600])https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1600), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +# v1.12.7 - Changelog since v1.12.6 + +## Changes by Kind + +### Uncategorized + +- Filter user misconfigured multiattach errors. ([#1560](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1560), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.12.6 - Changelog since v1.12.5 + +## Changes by Kind + +### Uncategorized + +- Bump golang.org/x/crypto from v0.15.0 to v0.17.0 to fix CVE-2023-48795 ([#1550](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1550), [@sunnylovestiramisu](https://github.com/sunnylovestiramisu)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- golang.org/x/crypto: v0.14.0 → v0.17.0 +- golang.org/x/sys: v0.13.0 → v0.15.0 +- golang.org/x/term: v0.13.0 → v0.15.0 +- golang.org/x/text: v0.13.0 → v0.14.0 + +### Removed +_Nothing has changed._ + + +# v1.12.5 - Changelog since v1.12.4 + +## Changes by Kind + +### Other (Cleanup or Flake) + +- Update golang builder to 1.20.12 ([#1536](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1536), [@msau42](https://github.com/msau42)) + +### Uncategorized + +- Add --fallback-requisite-zones flag to allow disk provisioning to fallback to a default set of zones when there are an insufficient number of zones available in a passed in requisite topology in CreateVolume. ([#1542](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1542), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1515](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1515), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.12.4 - Changelog since v1.12.3 + +## Changes by Kind + +### Uncategorized + +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1515](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1515), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.12.3 - Changelog since v1.12.2 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1502](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1502), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.56.2 to v1.56.3 to fix CVE-2023-44487. ([#1492](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1492), [@uriel-guzman](https://github.com/uriel-guzman)) + +### Uncategorized + +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1487](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1487), [@pwschuurman](https://github.com/pwschuurman)) +- The benign error when DisableDevice is not effective is logged as a warning. ([#1468](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1468), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Update go version to 1.20.10 ([#1460](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1460), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- google.golang.org/grpc: v1.56.2 → v1.56.3 + +### Removed +_Nothing has changed._ +e.com/go/area120: v0.6.0 → v0.8.1 +- cloud.google.com/go/artifactregistry: v1.9.0 → v1.14.1 +- cloud.google.com/go/asset: v1.10.0 → v1.14.1 +- cloud.google.com/go/assuredworkloads: v1.9.0 → v1.11.1 +- cloud.google.com/go/automl: v1.8.0 → v1.13.1 +- cloud.google.com/go/baremetalsolution: v0.4.0 → v0.5.0 +- cloud.google.com/go/batch: v0.4.0 → v0.7.0 +- cloud.google.com/go/beyondcorp: v0.3.0 → v0.6.1 +- cloud.google.com/go/bigquery: v1.44.0 → v1.52.0 +- cloud.google.com/go/billing: v1.7.0 → v1.16.0 +- cloud.google.com/go/binaryauthorization: v1.4.0 → v1.6.1 +- cloud.google.com/go/certificatemanager: v1.4.0 → v1.7.1 +- cloud.google.com/go/channel: v1.9.0 → v1.16.0 +- cloud.google.com/go/cloudbuild: v1.4.0 → v1.10.1 +- cloud.google.com/go/clouddms: v1.4.0 → v1.6.1 +- cloud.google.com/go/cloudtasks: v1.8.0 → v1.11.1 +- cloud.google.com/go/compute: v1.18.0 → v1.20.1 +- cloud.google.com/go/contactcenterinsights: v1.4.0 → v1.9.1 +- cloud.google.com/go/container: v1.7.0 → v1.22.1 +- cloud.google.com/go/containeranalysis: v0.6.0 → v0.10.1 +- cloud.google.com/go/datacatalog: v1.8.0 → v1.14.1 +- cloud.google.com/go/dataflow: v0.7.0 → v0.9.1 +- cloud.google.com/go/dataform: v0.5.0 → v0.8.1 +- cloud.google.com/go/datafusion: v1.5.0 → v1.7.1 +- cloud.google.com/go/datalabeling: v0.6.0 → v0.8.1 +- cloud.google.com/go/dataplex: v1.4.0 → v1.8.1 +- cloud.google.com/go/dataproc: v1.8.0 → v1.12.0 +- cloud.google.com/go/dataqna: v0.6.0 → v0.8.1 +- cloud.google.com/go/datastore: v1.10.0 → v1.12.0 +- cloud.google.com/go/datastream: v1.5.0 → v1.9.1 +- cloud.google.com/go/deploy: v1.5.0 → v1.11.0 +- cloud.google.com/go/dialogflow: v1.29.0 → v1.38.0 +- cloud.google.com/go/dlp: v1.7.0 → v1.10.1 +- cloud.google.com/go/documentai: v1.10.0 → v1.20.0 +- cloud.google.com/go/domains: v0.7.0 → v0.9.1 +- cloud.google.com/go/edgecontainer: v0.2.0 → v1.1.1 +- cloud.google.com/go/essentialcontacts: v1.4.0 → v1.6.2 +- cloud.google.com/go/eventarc: v1.8.0 → v1.12.1 +- cloud.google.com/go/filestore: v1.4.0 → v1.7.1 +- cloud.google.com/go/firestore: v1.9.0 → v1.11.0 +- cloud.google.com/go/functions: v1.9.0 → v1.15.1 +- cloud.google.com/go/gaming: v1.8.0 → v1.10.1 +- cloud.google.com/go/gkebackup: v0.3.0 → v0.4.0 +- cloud.google.com/go/gkeconnect: v0.6.0 → v0.8.1 +- cloud.google.com/go/gkehub: v0.10.0 → v0.14.1 +- cloud.google.com/go/gkemulticloud: v0.4.0 → v0.6.1 +- cloud.google.com/go/gsuiteaddons: v1.4.0 → v1.6.1 +- cloud.google.com/go/iam: v0.11.0 → v1.1.0 +- cloud.google.com/go/iap: v1.5.0 → v1.8.1 +- cloud.google.com/go/ids: v1.2.0 → v1.4.1 +- cloud.google.com/go/iot: v1.4.0 → v1.7.1 +- cloud.google.com/go/kms: v1.6.0 → v1.12.1 +- cloud.google.com/go/language: v1.8.0 → v1.10.1 +- cloud.google.com/go/lifesciences: v0.6.0 → v0.9.1 +- cloud.google.com/go/logging: v1.6.1 → v1.7.0 +- cloud.google.com/go/longrunning: v0.3.0 → v0.5.1 +- cloud.google.com/go/managedidentities: v1.4.0 → v1.6.1 +- cloud.google.com/go/maps: v0.1.0 → v0.7.0 +- cloud.google.com/go/mediatranslation: v0.6.0 → v0.8.1 +- cloud.google.com/go/memcache: v1.7.0 → v1.10.1 +- cloud.google.com/go/metastore: v1.8.0 → v1.11.1 +- cloud.google.com/go/monitoring: v1.8.0 → v1.15.1 +- cloud.google.com/go/networkconnectivity: v1.7.0 → v1.12.1 +- cloud.google.com/go/networkmanagement: v1.5.0 → v1.8.0 +- cloud.google.com/go/networksecurity: v0.6.0 → v0.9.1 +- cloud.google.com/go/notebooks: v1.5.0 → v1.9.1 +- cloud.google.com/go/optimization: v1.2.0 → v1.4.1 +- cloud.google.com/go/orchestration: v1.4.0 → v1.8.1 +- cloud.google.com/go/orgpolicy: v1.5.0 → v1.11.1 +- cloud.google.com/go/osconfig: v1.10.0 → v1.12.1 +- cloud.google.com/go/oslogin: v1.7.0 → v1.10.1 +- cloud.google.com/go/phishingprotection: v0.6.0 → v0.8.1 +- cloud.google.com/go/policytroubleshooter: v1.4.0 → v1.7.1 +- cloud.google.com/go/privatecatalog: v0.6.0 → v0.9.1 +- cloud.google.com/go/pubsub: v1.27.1 → v1.32.0 +- cloud.google.com/go/pubsublite: v1.5.0 → v1.8.1 +- cloud.google.com/go/recaptchaenterprise/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/recommendationengine: v0.6.0 → v0.8.1 +- cloud.google.com/go/recommender: v1.8.0 → v1.10.1 +- cloud.google.com/go/redis: v1.10.0 → v1.13.1 +- cloud.google.com/go/resourcemanager: v1.4.0 → v1.9.1 +- cloud.google.com/go/resourcesettings: v1.4.0 → v1.6.1 +- cloud.google.com/go/retail: v1.11.0 → v1.14.1 +- cloud.google.com/go/run: v0.3.0 → v0.9.0 +- cloud.google.com/go/scheduler: v1.7.0 → v1.10.1 +- cloud.google.com/go/secretmanager: v1.9.0 → v1.11.1 +- cloud.google.com/go/security: v1.10.0 → v1.15.1 +- cloud.google.com/go/securitycenter: v1.16.0 → v1.23.0 +- cloud.google.com/go/servicedirectory: v1.7.0 → v1.10.1 +- cloud.google.com/go/shell: v1.4.0 → v1.7.1 +- cloud.google.com/go/spanner: v1.41.0 → v1.47.0 +- cloud.google.com/go/speech: v1.9.0 → v1.17.1 +- cloud.google.com/go/storagetransfer: v1.6.0 → v1.10.0 +- cloud.google.com/go/talent: v1.4.0 → v1.6.2 +- cloud.google.com/go/texttospeech: v1.5.0 → v1.7.1 +- cloud.google.com/go/tpu: v1.4.0 → v1.6.1 +- cloud.google.com/go/trace: v1.4.0 → v1.10.1 +- cloud.google.com/go/translate: v1.4.0 → v1.8.1 +- cloud.google.com/go/video: v1.9.0 → v1.17.1 +- cloud.google.com/go/videointelligence: v1.9.0 → v1.11.1 +- cloud.google.com/go/vision/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/vmmigration: v1.3.0 → v1.7.1 +- cloud.google.com/go/vmwareengine: v0.1.0 → v0.4.1 +- cloud.google.com/go/vpcaccess: v1.5.0 → v1.7.1 +- cloud.google.com/go/webrisk: v1.7.0 → v1.9.1 +- cloud.google.com/go/websecurityscanner: v1.4.0 → v1.6.1 +- cloud.google.com/go/workflows: v1.9.0 → v1.11.1 +- cloud.google.com/go: v0.107.0 → v0.110.4 +- github.com/cncf/xds/go: [06c439d → e9ce688](https://github.com/cncf/xds/go/compare/06c439d...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.10.3 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.10.3...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.9.1 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.9.1...v0.10.1) +- github.com/golang/glog: [v1.0.0 → v1.1.0](https://github.com/golang/glog/compare/v1.0.0...v1.1.0) +- github.com/golang/protobuf: [v1.5.2 → v1.5.3](https://github.com/golang/protobuf/compare/v1.5.2...v1.5.3) +- github.com/googleapis/enterprise-certificate-proxy: [v0.2.3 → v0.2.5](https://github.com/googleapis/enterprise-certificate-proxy/compare/v0.2.3...v0.2.5) +- github.com/googleapis/gax-go/v2: [v2.7.0 → v2.12.0](https://github.com/googleapis/gax-go/v2/compare/v2.7.0...v2.12.0) +- github.com/yuin/goldmark: [v1.4.1 → v1.4.13](https://github.com/yuin/goldmark/compare/v1.4.1...v1.4.13) +- golang.org/x/oauth2: v0.5.0 → v0.10.0 +- golang.org/x/sync: v0.1.0 → v0.3.0 +- google.golang.org/api: v0.111.0 → v0.134.0 +- google.golang.org/genproto: 637eb22 → ccb25ca +- google.golang.org/grpc: v1.53.0 → v1.56.3 +- google.golang.org/protobuf: v1.28.1 → v1.31.0 + +### Removed +- cloud.google.com/go/apikeys: v0.4.0 +- cloud.google.com/go/servicecontrol: v1.5.0 +- cloud.google.com/go/servicemanagement: v1.5.0 +- cloud.google.com/go/serviceusage: v1.4.0 + +# v1.12.2 - Changelog since v1.12.0 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Changed + +- golang.org/x/crypto: v0.11.0 → v0.14.0 +- golang.org/x/net: v0.12.0 → v0.17.0 +- golang.org/x/sys: v0.10.0 → v0.13.0 +- golang.org/x/term: v0.10.0 → v0.13.0 +- golang.org/x/text: v0.11.0 → v0.13.0 + +# v1.12.0 - Changelog since v1.11.1 + +## Changes by Kind + +### Uncategorized + +- Build and publish arm64 images ([#1369](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1369), [@upodroid](https://github.com/upodroid)) + +## Dependencies + +### Added + +_Nothing has changed._ + +### Changed + +_Nothing has changed._ + +### Removed + +_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.13.md b/CHANGELOG/CHANGELOG-1.13.md new file mode 100644 index 00000000..8d6a80b0 --- /dev/null +++ b/CHANGELOG/CHANGELOG-1.13.md @@ -0,0 +1,371 @@ +# v1.13.4 - Changelog since v1.13.3 + +## Changes by Kind + +### Uncategorized + +- Fixes a regression in the NVMe device search path ([#1634](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1634), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.13.3 - Changelog since v1.13.2 + +## Changes by Kind + +### Uncategorized + +- Flag --use-instance-api-to-poll-attachment-disk-types uses instances.get API when polling for disk attachment in ControllerPublish for passed in disk types ([#1630](https://github.com/kube +rnetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1630), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Support for read_ahead_kb mount flag to change block device readahead ([#1631](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1631), [@k8s-infra-cherrypick- +robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.13.2 - Changelog since v1.13.1 + +## Changes by Kind + +### Bug + +- Fix error when no compute endpoint is passed ([#1622](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1622), [@Sneha-at](https://github.com/Sneha-at)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + +# v1.13.1 - Changelog since v1.13.0 + +## Changes by Kind + +### Feature + +- Add support for multi-zone volumeHandle ([#1616](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1616), [@pwschuurman](https://github.com/pwschuurman)) +- Update driver to support staging compute ([#1614](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1614), [@Sneha-at](https://github.com/Sneha-at)) + +### Other (Cleanup or Flake) + +- Map UNSUPPORTED_OPERATION GCE operation error codes to InvalidArgument ([#1610](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1610), [@amacaskill](https://github.com/amacaskill)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.13.0 - Changelog since v1.12.6 + +## Changes by Kind + +### Feature + +- Added enable-otel-tracing flag to enable opentelemetry tracing for self-managed drivers. The flag is disabled by default ([#1403](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1403), [@Fricounet](https://github.com/Fricounet)) +- Adds validation and support for creating volumes in storage pools. ([#1526](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1526), [@amacaskill](https://github.com/amacaskill)) + +### Bug or Regression + +- Add --fallback-requisite-zones flag to allow disk provisioning to fallback to a default set of zones when there are an insufficient number of zones available in a passed in requisite topology in CreateVolume. ([#1532](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1532), [@pwschuurman](https://github.com/pwschuurman)) +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) +- Filter user misconfigured multiattach errors. ([#1559](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1559), [@mattcary](https://github.com/mattcary)) +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1514](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1514), [@pwschuurman](https://github.com/pwschuurman)) +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1485](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1485), [@pwschuurman](https://github.com/pwschuurman)) + +### Other (Cleanup or Flake) + +- The benign error when DisableDevice is not effective is logged as a warning. ([#1467](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1467), [@mattcary](https://github.com/mattcary)) +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +### Uncategorized + +- Adds enable_storage_pools label to the OperationErrorMetric so we can track CSI operation error rates when storage pools is used. ([#1534](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1534), [@amacaskill](https://github.com/amacaskill)) + +## Dependencies + +### Added +- cloud.google.com/go/apikeys: v0.6.0 +- cloud.google.com/go/dataproc/v2: v2.3.0 +- cloud.google.com/go/grafeas: v0.2.0 +- cloud.google.com/go/recaptchaenterprise: v1.3.1 +- cloud.google.com/go/servicecontrol: v1.11.1 +- cloud.google.com/go/servicemanagement: v1.8.0 +- cloud.google.com/go/serviceusage: v1.6.0 +- cloud.google.com/go/vision: v1.2.0 +- gioui.org: 57750fc +- git.sr.ht/~sbinet/gg: v0.3.1 +- github.com/JohnCGriffin/overflow: [46fa312](https://github.com/JohnCGriffin/overflow/tree/46fa312) +- github.com/ajstarks/deck/generate: [c3f852c](https://github.com/ajstarks/deck/generate/tree/c3f852c) +- github.com/ajstarks/deck: [30c9fc6](https://github.com/ajstarks/deck/tree/30c9fc6) +- github.com/ajstarks/svgo: [1546f12](https://github.com/ajstarks/svgo/tree/1546f12) +- github.com/apache/arrow/go/v10: [v10.0.1](https://github.com/apache/arrow/go/v10/tree/v10.0.1) +- github.com/apache/arrow/go/v11: [v11.0.0](https://github.com/apache/arrow/go/v11/tree/v11.0.0) +- github.com/boombuler/barcode: [v1.0.1](https://github.com/boombuler/barcode/tree/v1.0.1) +- github.com/buger/jsonparser: [v1.1.1](https://github.com/buger/jsonparser/tree/v1.1.1) +- github.com/cenkalti/backoff/v4: [v4.2.1](https://github.com/cenkalti/backoff/v4/tree/v4.2.1) +- github.com/flowstack/go-jsonschema: [v0.1.1](https://github.com/flowstack/go-jsonschema/tree/v0.1.1) +- github.com/fogleman/gg: [v1.3.0](https://github.com/fogleman/gg/tree/v1.3.0) +- github.com/go-fonts/dejavu: [v0.1.0](https://github.com/go-fonts/dejavu/tree/v0.1.0) +- github.com/go-fonts/latin-modern: [v0.2.0](https://github.com/go-fonts/latin-modern/tree/v0.2.0) +- github.com/go-fonts/liberation: [v0.2.0](https://github.com/go-fonts/liberation/tree/v0.2.0) +- github.com/go-fonts/stix: [v0.1.0](https://github.com/go-fonts/stix/tree/v0.1.0) +- github.com/go-latex/latex: [c0d11ff](https://github.com/go-latex/latex/tree/c0d11ff) +- github.com/go-logr/stdr: [v1.2.2](https://github.com/go-logr/stdr/tree/v1.2.2) +- github.com/go-pdf/fpdf: [v0.6.0](https://github.com/go-pdf/fpdf/tree/v0.6.0) +- github.com/goccy/go-json: [v0.9.11](https://github.com/goccy/go-json/tree/v0.9.11) +- github.com/golang/freetype: [e2365df](https://github.com/golang/freetype/tree/e2365df) +- github.com/google/flatbuffers: [v2.0.8+incompatible](https://github.com/google/flatbuffers/tree/v2.0.8) +- github.com/google/gnostic-models: [c7be7c7](https://github.com/google/gnostic-models/tree/c7be7c7) +- github.com/googleapis/go-type-adapters: [v1.0.0](https://github.com/googleapis/go-type-adapters/tree/v1.0.0) +- github.com/googleapis/google-cloud-go-testing: [bcd43fb](https://github.com/googleapis/google-cloud-go-testing/tree/bcd43fb) +- github.com/grpc-ecosystem/grpc-gateway/v2: [v2.16.0](https://github.com/grpc-ecosystem/grpc-gateway/v2/tree/v2.16.0) +- github.com/iancoleman/strcase: [v0.2.0](https://github.com/iancoleman/strcase/tree/v0.2.0) +- github.com/jung-kurt/gofpdf: [24315ac](https://github.com/jung-kurt/gofpdf/tree/24315ac) +- github.com/klauspost/asmfmt: [v1.3.2](https://github.com/klauspost/asmfmt/tree/v1.3.2) +- github.com/klauspost/cpuid/v2: [v2.0.9](https://github.com/klauspost/cpuid/v2/tree/v2.0.9) +- github.com/lyft/protoc-gen-star: [v0.6.1](https://github.com/lyft/protoc-gen-star/tree/v0.6.1) +- github.com/minio/asm2plan9s: [cdd7644](https://github.com/minio/asm2plan9s/tree/cdd7644) +- github.com/minio/c2goasm: [36a3d3b](https://github.com/minio/c2goasm/tree/36a3d3b) +- github.com/phpdave11/gofpdf: [v1.4.2](https://github.com/phpdave11/gofpdf/tree/v1.4.2) +- github.com/phpdave11/gofpdi: [v1.0.13](https://github.com/phpdave11/gofpdi/tree/v1.0.13) +- github.com/pierrec/lz4/v4: [v4.1.15](https://github.com/pierrec/lz4/v4/tree/v4.1.15) +- github.com/pkg/diff: [20ebb0f](https://github.com/pkg/diff/tree/20ebb0f) +- github.com/ruudk/golang-pdf417: [a7e3863](https://github.com/ruudk/golang-pdf417/tree/a7e3863) +- github.com/zeebo/assert: [v1.3.0](https://github.com/zeebo/assert/tree/v1.3.0) +- github.com/zeebo/xxh3: [v1.0.2](https://github.com/zeebo/xxh3/tree/v1.0.2) +- go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc: v1.21.0 +- go.opentelemetry.io/otel/exporters/otlp/otlptrace: v1.21.0 +- gonum.org/v1/plot: v0.10.1 +- google.golang.org/grpc/cmd/protoc-gen-go-grpc: v1.1.0 +- lukechampine.com/uint128: v1.2.0 +- modernc.org/cc/v3: v3.36.3 +- modernc.org/ccgo/v3: v3.16.9 +- modernc.org/ccorpus: v1.11.6 +- modernc.org/httpfs: v1.0.6 +- modernc.org/libc: v1.17.1 +- modernc.org/memory: v1.2.1 +- modernc.org/opt: v0.1.3 +- modernc.org/sqlite: v1.18.1 +- modernc.org/tcl: v1.13.1 +- modernc.org/token: v1.0.0 +- modernc.org/z: v1.5.1 +- rsc.io/pdf: v0.1.1 + +### Changed +- cloud.google.com/go/accessapproval: v1.7.1 → v1.7.4 +- cloud.google.com/go/accesscontextmanager: v1.8.1 → v1.8.4 +- cloud.google.com/go/aiplatform: v1.45.0 → v1.57.0 +- cloud.google.com/go/analytics: v0.21.2 → v0.21.6 +- cloud.google.com/go/apigateway: v1.6.1 → v1.6.4 +- cloud.google.com/go/apigeeconnect: v1.6.1 → v1.6.4 +- cloud.google.com/go/apigeeregistry: v0.7.1 → v0.8.2 +- cloud.google.com/go/appengine: v1.8.1 → v1.8.4 +- cloud.google.com/go/area120: v0.8.1 → v0.8.4 +- cloud.google.com/go/artifactregistry: v1.14.1 → v1.14.6 +- cloud.google.com/go/asset: v1.14.1 → v1.15.3 +- cloud.google.com/go/assuredworkloads: v1.11.1 → v1.11.4 +- cloud.google.com/go/automl: v1.13.1 → v1.13.4 +- cloud.google.com/go/baremetalsolution: v0.5.0 → v1.2.3 +- cloud.google.com/go/batch: v0.7.0 → v1.7.0 +- cloud.google.com/go/beyondcorp: v0.6.1 → v1.0.3 +- cloud.google.com/go/bigquery: v1.52.0 → v1.57.1 +- cloud.google.com/go/billing: v1.16.0 → v1.18.0 +- cloud.google.com/go/binaryauthorization: v1.6.1 → v1.8.0 +- cloud.google.com/go/certificatemanager: v1.7.1 → v1.7.4 +- cloud.google.com/go/channel: v1.16.0 → v1.17.3 +- cloud.google.com/go/cloudbuild: v1.10.1 → v1.15.0 +- cloud.google.com/go/clouddms: v1.6.1 → v1.7.3 +- cloud.google.com/go/cloudtasks: v1.11.1 → v1.12.4 +- cloud.google.com/go/compute: v1.20.1 → v1.23.3 +- cloud.google.com/go/contactcenterinsights: v1.9.1 → v1.12.1 +- cloud.google.com/go/container: v1.22.1 → v1.29.0 +- cloud.google.com/go/containeranalysis: v0.10.1 → v0.11.3 +- cloud.google.com/go/datacatalog: v1.14.1 → v1.19.0 +- cloud.google.com/go/dataflow: v0.9.1 → v0.9.4 +- cloud.google.com/go/dataform: v0.8.1 → v0.9.1 +- cloud.google.com/go/datafusion: v1.7.1 → v1.7.4 +- cloud.google.com/go/datalabeling: v0.8.1 → v0.8.4 +- cloud.google.com/go/dataplex: v1.8.1 → v1.13.0 +- cloud.google.com/go/dataqna: v0.8.1 → v0.8.4 +- cloud.google.com/go/datastore: v1.12.0 → v1.15.0 +- cloud.google.com/go/datastream: v1.9.1 → v1.10.3 +- cloud.google.com/go/deploy: v1.11.0 → v1.16.0 +- cloud.google.com/go/dialogflow: v1.38.0 → v1.47.0 +- cloud.google.com/go/dlp: v1.10.1 → v1.11.1 +- cloud.google.com/go/documentai: v1.20.0 → v1.23.6 +- cloud.google.com/go/domains: v0.9.1 → v0.9.4 +- cloud.google.com/go/edgecontainer: v1.1.1 → v1.1.4 +- cloud.google.com/go/essentialcontacts: v1.6.2 → v1.6.5 +- cloud.google.com/go/eventarc: v1.12.1 → v1.13.3 +- cloud.google.com/go/filestore: v1.7.1 → v1.8.0 +- cloud.google.com/go/firestore: v1.11.0 → v1.14.0 +- cloud.google.com/go/functions: v1.15.1 → v1.15.4 +- cloud.google.com/go/gaming: v1.10.1 → v1.9.0 +- cloud.google.com/go/gkebackup: v0.4.0 → v1.3.4 +- cloud.google.com/go/gkeconnect: v0.8.1 → v0.8.4 +- cloud.google.com/go/gkehub: v0.14.1 → v0.14.4 +- cloud.google.com/go/gkemulticloud: v0.6.1 → v1.0.3 +- cloud.google.com/go/gsuiteaddons: v1.6.1 → v1.6.4 +- cloud.google.com/go/iam: v1.1.0 → v1.1.5 +- cloud.google.com/go/iap: v1.8.1 → v1.9.3 +- cloud.google.com/go/ids: v1.4.1 → v1.4.4 +- cloud.google.com/go/iot: v1.7.1 → v1.7.4 +- cloud.google.com/go/kms: v1.12.1 → v1.15.5 +- cloud.google.com/go/language: v1.10.1 → v1.12.2 +- cloud.google.com/go/lifesciences: v0.9.1 → v0.9.4 +- cloud.google.com/go/logging: v1.7.0 → v1.8.1 +- cloud.google.com/go/longrunning: v0.5.1 → v0.5.4 +- cloud.google.com/go/managedidentities: v1.6.1 → v1.6.4 +- cloud.google.com/go/maps: v0.7.0 → v1.6.2 +- cloud.google.com/go/mediatranslation: v0.8.1 → v0.8.4 +- cloud.google.com/go/memcache: v1.10.1 → v1.10.4 +- cloud.google.com/go/metastore: v1.11.1 → v1.13.3 +- cloud.google.com/go/monitoring: v1.15.1 → v1.16.3 +- cloud.google.com/go/networkconnectivity: v1.12.1 → v1.14.3 +- cloud.google.com/go/networkmanagement: v1.8.0 → v1.9.3 +- cloud.google.com/go/networksecurity: v0.9.1 → v0.9.4 +- cloud.google.com/go/notebooks: v1.9.1 → v1.11.2 +- cloud.google.com/go/optimization: v1.4.1 → v1.6.2 +- cloud.google.com/go/orchestration: v1.8.1 → v1.8.4 +- cloud.google.com/go/orgpolicy: v1.11.1 → v1.11.4 +- cloud.google.com/go/osconfig: v1.12.1 → v1.12.4 +- cloud.google.com/go/oslogin: v1.10.1 → v1.12.2 +- cloud.google.com/go/phishingprotection: v0.8.1 → v0.8.4 +- cloud.google.com/go/policytroubleshooter: v1.7.1 → v1.10.2 +- cloud.google.com/go/privatecatalog: v0.9.1 → v0.9.4 +- cloud.google.com/go/pubsub: v1.32.0 → v1.33.0 +- cloud.google.com/go/recaptchaenterprise/v2: v2.7.2 → v2.9.0 +- cloud.google.com/go/recommendationengine: v0.8.1 → v0.8.4 +- cloud.google.com/go/recommender: v1.10.1 → v1.11.3 +- cloud.google.com/go/redis: v1.13.1 → v1.14.1 +- cloud.google.com/go/resourcemanager: v1.9.1 → v1.9.4 +- cloud.google.com/go/resourcesettings: v1.6.1 → v1.6.4 +- cloud.google.com/go/retail: v1.14.1 → v1.14.4 +- cloud.google.com/go/run: v0.9.0 → v1.3.3 +- cloud.google.com/go/scheduler: v1.10.1 → v1.10.5 +- cloud.google.com/go/secretmanager: v1.11.1 → v1.11.4 +- cloud.google.com/go/security: v1.15.1 → v1.15.4 +- cloud.google.com/go/securitycenter: v1.23.0 → v1.24.3 +- cloud.google.com/go/servicedirectory: v1.10.1 → v1.11.3 +- cloud.google.com/go/shell: v1.7.1 → v1.7.4 +- cloud.google.com/go/spanner: v1.47.0 → v1.53.1 +- cloud.google.com/go/speech: v1.17.1 → v1.21.0 +- cloud.google.com/go/storage: v1.12.0 → v1.29.0 +- cloud.google.com/go/storagetransfer: v1.10.0 → v1.10.3 +- cloud.google.com/go/talent: v1.6.2 → v1.6.5 +- cloud.google.com/go/texttospeech: v1.7.1 → v1.7.4 +- cloud.google.com/go/tpu: v1.6.1 → v1.6.4 +- cloud.google.com/go/trace: v1.10.1 → v1.10.4 +- cloud.google.com/go/translate: v1.8.1 → v1.9.3 +- cloud.google.com/go/video: v1.17.1 → v1.20.3 +- cloud.google.com/go/videointelligence: v1.11.1 → v1.11.4 +- cloud.google.com/go/vision/v2: v2.7.2 → v2.7.5 +- cloud.google.com/go/vmmigration: v1.7.1 → v1.7.4 +- cloud.google.com/go/vmwareengine: v0.4.1 → v1.0.3 +- cloud.google.com/go/vpcaccess: v1.7.1 → v1.7.4 +- cloud.google.com/go/webrisk: v1.9.1 → v1.9.4 +- cloud.google.com/go/websecurityscanner: v1.6.1 → v1.6.4 +- cloud.google.com/go/workflows: v1.11.1 → v1.12.3 +- cloud.google.com/go: v0.110.4 → v0.111.0 +- github.com/Microsoft/go-winio: [v0.4.17 → v0.6.1](https://github.com/Microsoft/go-winio/compare/v0.4.17...v0.6.1) +- github.com/andybalholm/brotli: [5f990b6 → v1.0.4](https://github.com/andybalholm/brotli/compare/5f990b6...v1.0.4) +- github.com/apache/thrift: [v0.12.0 → v0.16.0](https://github.com/apache/thrift/compare/v0.12.0...v0.16.0) +- github.com/envoyproxy/go-control-plane: [9239064 → v0.11.1](https://github.com/envoyproxy/go-control-plane/compare/9239064...v0.11.1) +- github.com/envoyproxy/protoc-gen-validate: [v0.10.1 → v1.0.2](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.10.1...v1.0.2) +- github.com/felixge/httpsnoop: [v1.0.1 → v1.0.4](https://github.com/felixge/httpsnoop/compare/v1.0.1...v1.0.4) +- github.com/go-logr/logr: [v1.2.3 → v1.3.0](https://github.com/go-logr/logr/compare/v1.2.3...v1.3.0) +- github.com/go-openapi/jsonpointer: [v0.19.5 → v0.20.0](https://github.com/go-openapi/jsonpointer/compare/v0.19.5...v0.20.0) +- github.com/go-openapi/swag: [v0.21.1 → v0.22.4](https://github.com/go-openapi/swag/compare/v0.21.1...v0.22.4) +- github.com/go-task/slim-sprig: [348f09d → 52ccab3](https://github.com/go-task/slim-sprig/compare/348f09d...52ccab3) +- github.com/golang/glog: [v1.1.0 → v1.1.2](https://github.com/golang/glog/compare/v1.1.0...v1.1.2) +- github.com/golang/snappy: [v0.0.1 → v0.0.4](https://github.com/golang/snappy/compare/v0.0.1...v0.0.4) +- github.com/google/gnostic: [v0.5.7-v3refs → v0.7.0](https://github.com/google/gnostic/compare/v0.5.7-v3refs...v0.7.0) +- github.com/google/go-cmp: [v0.5.9 → v0.6.0](https://github.com/google/go-cmp/compare/v0.5.9...v0.6.0) +- github.com/google/go-pkcs11: [v0.2.0 → c6f7932](https://github.com/google/go-pkcs11/compare/v0.2.0...c6f7932) +- github.com/google/martian/v3: [v3.1.0 → v3.3.2](https://github.com/google/martian/v3/compare/v3.1.0...v3.3.2) +- github.com/google/pprof: [94a9f03 → 4bb14d4](https://github.com/google/pprof/compare/94a9f03...4bb14d4) +- github.com/google/s2a-go: [v0.1.4 → v0.1.7](https://github.com/google/s2a-go/compare/v0.1.4...v0.1.7) +- github.com/google/uuid: [v1.3.0 → v1.5.0](https://github.com/google/uuid/compare/v1.3.0...v1.5.0) +- github.com/googleapis/enterprise-certificate-proxy: [v0.2.5 → v0.3.2](https://github.com/googleapis/enterprise-certificate-proxy/compare/v0.2.5...v0.3.2) +- github.com/klauspost/compress: [v1.13.6 → v1.15.9](https://github.com/klauspost/compress/compare/v1.13.6...v1.15.9) +- github.com/kr/pretty: [v0.3.0 → v0.3.1](https://github.com/kr/pretty/compare/v0.3.0...v0.3.1) +- github.com/kubernetes-csi/csi-proxy/client: [v1.1.1 → v1.1.3](https://github.com/kubernetes-csi/csi-proxy/client/compare/v1.1.1...v1.1.3) +- github.com/mattn/go-isatty: [v0.0.12 → v0.0.16](https://github.com/mattn/go-isatty/compare/v0.0.12...v0.0.16) +- github.com/onsi/ginkgo/v2: [v2.7.1 → v2.14.0](https://github.com/onsi/ginkgo/v2/compare/v2.7.1...v2.14.0) +- github.com/onsi/gomega: [v1.25.0 → v1.30.0](https://github.com/onsi/gomega/compare/v1.25.0...v1.30.0) +- github.com/pkg/sftp: [v1.10.1 → v1.13.1](https://github.com/pkg/sftp/compare/v1.10.1...v1.13.1) +- github.com/remyoudompheng/bigfft: [52369c6 → eec4a21](https://github.com/remyoudompheng/bigfft/compare/52369c6...eec4a21) +- github.com/rogpeppe/go-internal: [v1.9.0 → v1.10.0](https://github.com/rogpeppe/go-internal/compare/v1.9.0...v1.10.0) +- github.com/sirupsen/logrus: [v1.8.1 → v1.9.0](https://github.com/sirupsen/logrus/compare/v1.8.1...v1.9.0) +- github.com/spf13/afero: [v1.6.0 → v1.9.2](https://github.com/spf13/afero/compare/v1.6.0...v1.9.2) +- github.com/stretchr/testify: [v1.8.1 → v1.8.4](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.4) +- github.com/xeipuuv/gojsonschema: [v1.1.0 → v1.2.0](https://github.com/xeipuuv/gojsonschema/compare/v1.1.0...v1.2.0) +- go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc: v0.20.0 → v0.46.1 +- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp: v0.20.0 → v0.46.1 +- go.opentelemetry.io/otel/metric: v0.20.0 → v1.21.0 +- go.opentelemetry.io/otel/sdk: v0.20.0 → v1.21.0 +- go.opentelemetry.io/otel/trace: v0.20.0 → v1.21.0 +- go.opentelemetry.io/otel: v0.20.0 → v1.21.0 +- go.opentelemetry.io/proto/otlp: v0.7.0 → v1.0.0 +- go.uber.org/goleak: v1.1.10 → v1.3.0 +- golang.org/x/crypto: v0.11.0 → v0.18.0 +- golang.org/x/exp: 6cc2880 → 334a238 +- golang.org/x/image: cff245a → 723b81c +- golang.org/x/mod: v0.8.0 → v0.14.0 +- golang.org/x/net: v0.12.0 → v0.20.0 +- golang.org/x/oauth2: v0.10.0 → v0.16.0 +- golang.org/x/sync: v0.3.0 → v0.6.0 +- golang.org/x/sys: v0.10.0 → v0.16.0 +- golang.org/x/term: v0.10.0 → v0.16.0 +- golang.org/x/text: v0.11.0 → v0.14.0 +- golang.org/x/time: 90d013b → v0.5.0 +- golang.org/x/tools: v0.6.0 → v0.16.1 +- golang.org/x/xerrors: 5ec99f8 → 04be3eb +- gonum.org/v1/gonum: 3d26580 → v0.11.0 +- google.golang.org/api: v0.134.0 → v0.156.0 +- google.golang.org/appengine: v1.6.7 → v1.6.8 +- google.golang.org/genproto/googleapis/api: ccb25ca → 995d672 +- google.golang.org/genproto/googleapis/bytestream: 659f7aa → 50ed04b +- google.golang.org/genproto/googleapis/rpc: 659f7aa → 50ed04b +- google.golang.org/genproto: ccb25ca → 995d672 +- google.golang.org/grpc: v1.56.2 → v1.60.1 +- google.golang.org/protobuf: v1.31.0 → v1.32.0 +- honnef.co/go/tools: v0.0.1-2020.1.4 → v0.1.3 +- k8s.io/klog/v2: v2.90.1 → v2.100.1 +- k8s.io/mount-utils: v0.27.0-alpha.3 → v0.29.0-alpha.2 +- k8s.io/utils: a36077c → 3b25d92 +- modernc.org/mathutil: v1.0.0 → v1.5.0 +- modernc.org/strutil: v1.0.0 → v1.1.3 +- sigs.k8s.io/structured-merge-diff/v4: v4.2.1 → v4.4.1 +- sigs.k8s.io/yaml: v1.3.0 → v1.4.0 + +### Removed +_Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.7.md b/CHANGELOG/CHANGELOG-1.7.md index dba7f026..dc7e49b0 100644 --- a/CHANGELOG/CHANGELOG-1.7.md +++ b/CHANGELOG/CHANGELOG-1.7.md @@ -1,3 +1,112 @@ +# v1.7.20 - Changelog since v1.7.19 + +## Changes by Kind + +### Uncategorized + +- Bump golang.org/x/crypto from v0.14.0 to v0.17.0 to fix CVE-2023-48795 ([#1554](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1554), [@sunnylovestiramisu](https://github.com/sunnylovestiramisu)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- golang.org/x/crypto: v0.14.0 → v0.17.0 +- golang.org/x/sys: v0.13.0 → v0.15.0 +- golang.org/x/term: v0.13.0 → v0.15.0 +- golang.org/x/text: v0.13.0 → v0.14.0 + +### Removed +_Nothing has changed._ + + +# v1.7.19 - Changelog since v1.7.18 + +## Changes by Kind + +### Uncategorized + +- Update golang builder to 1.20.12 ([#1539](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1539), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.7.18 - Changelog since v1.7.17 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1503](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1503), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.55.1 to v1.56.3 to fix CVE-2023-44487. ([#1497](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1497), [@uriel-guzman](https://github.com/uriel-guzman)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- github.com/cncf/xds/go: [32f1caf → e9ce688](https://github.com/cncf/xds/go/compare/32f1caf...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.11.0 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.11.0...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.10.0 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.10.0...v0.10.1) +- google.golang.org/grpc: v1.55.1 → v1.56.3 + +### Removed +_Nothing has changed._ + +# v1.7.17 - Changelog since v1.7.15 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Added + +_Nothing has changed._ + +### Changed + +- golang.org/x/crypto: v0.9.0 → v0.14.0 +- golang.org/x/net: v0.10.0 → v0.17.0 +- golang.org/x/sys: v0.8.0 → v0.13.0 +- golang.org/x/term: v0.8.0 → v0.13.0 +- golang.org/x/text: v0.9.0 → v0.13.0 + +# v1.7.15 - Changelog since v1.7.13 + +## Changes by Kind + +### Bug or Regression + +- bump go version to 1.20.8 ([#1394](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1394), [@tyuchn](https://github.com/tyuchn)) +- Remove ARG BUILDPLATFORM from Dockerfile ([#1385](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1385), [@tyuchn](https://github.com/tyuchn)) + +# v1.7.13 - Changelog since v1.7.12 + +## Changes by Kind + +### Bug or Regression + +- Upgrade google.golang.org/grpc from v1.55.0 -> v1.55.1 to address https://github.com/grpc/grpc-go/issues/6373 ([#1373](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1373), [@pwschuurman](https://github.com/pwschuurman)) + # v1.7.12 - Changelog since v1.7.11 ## Changes by Kind @@ -41,6 +150,7 @@ ## Dependencies ### Added + - bitbucket.org/creachadair/stringset: v0.0.9 - cloud.google.com/go/accessapproval: v1.6.0 - cloud.google.com/go/accesscontextmanager: v1.7.0 @@ -218,6 +328,7 @@ - sigs.k8s.io/json: 9f7c6b3 ### Changed + - cloud.google.com/go/bigquery: v1.8.0 → v1.50.0 - cloud.google.com/go/datastore: v1.1.0 → v1.11.0 - cloud.google.com/go/firestore: v1.1.0 → v1.9.0 @@ -338,6 +449,7 @@ - sigs.k8s.io/yaml: v1.2.0 → v1.3.0 ### Removed + - contrib.go.opencensus.io/exporter/stackdriver: v0.12.8 - git.apache.org/thrift.git: 2566ecd - github.com/aws/aws-k8s-tester: [b411acf](https://github.com/aws/aws-k8s-tester/tree/b411acf) @@ -474,10 +586,13 @@ _Nothing has changed._ ## Dependencies ### Added + _Nothing has changed._ ### Changed + - github.com/prometheus/client_golang: [v1.11.0 → v1.11.1](https://github.com/prometheus/client_golang/compare/v1.11.0...v1.11.1) ### Removed + _Nothing has changed._ diff --git a/CHANGELOG/CHANGELOG-1.8.md b/CHANGELOG/CHANGELOG-1.8.md index 1308dda5..a9ab8c8c 100644 --- a/CHANGELOG/CHANGELOG-1.8.md +++ b/CHANGELOG/CHANGELOG-1.8.md @@ -1,3 +1,182 @@ +# v1.8.20 - Changelog since v1.8.19 + +## Changes by Kind + +### Bug + +- Change GetDisk error reporting to temporary in CreateVolume codepath ([#1604])https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1604), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +# v1.8.19 - Changelog since v1.8.18 + +## Changes by Kind + +### Uncategorized + +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1519](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1519), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1581](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1581), [@pwschuurman](https://github.com/pwschuurman)) +- Update golang builder to 1.20.12 ([#1538](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1538), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.8.18 - Changelog since v1.8.17 + +## Changes by Kind + +### Uncategorized + +- Update golang builder to 1.20.12 ([#1538](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1538), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.8.17 - Changelog since v1.8.16 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1504](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1504), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.55.1 to v1.56.3 to fix CVE-2023-44487. ([#1496](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1496), [@uriel-guzman](https://github.com/uriel-guzman)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- cloud.google.com/go/accesscontextmanager: v1.6.0 → v1.7.0 +- cloud.google.com/go/aiplatform: v1.35.0 → v1.37.0 +- cloud.google.com/go/analytics: v0.18.0 → v0.19.0 +- cloud.google.com/go/apigeeregistry: v0.5.0 → v0.6.0 +- cloud.google.com/go/apikeys: v0.5.0 → v0.6.0 +- cloud.google.com/go/appengine: v1.6.0 → v1.7.1 +- cloud.google.com/go/artifactregistry: v1.11.2 → v1.13.0 +- cloud.google.com/go/asset: v1.11.1 → v1.13.0 +- cloud.google.com/go/beyondcorp: v0.4.0 → v0.5.0 +- cloud.google.com/go/bigquery: v1.48.0 → v1.50.0 +- cloud.google.com/go/billing: v1.12.0 → v1.13.0 +- cloud.google.com/go/channel: v1.11.0 → v1.12.0 +- cloud.google.com/go/cloudbuild: v1.7.0 → v1.9.0 +- cloud.google.com/go/cloudtasks: v1.9.0 → v1.10.0 +- cloud.google.com/go/compute: v1.18.0 → v1.19.1 +- cloud.google.com/go/container: v1.13.1 → v1.15.0 +- cloud.google.com/go/containeranalysis: v0.7.0 → v0.9.0 +- cloud.google.com/go/datacatalog: v1.12.0 → v1.13.0 +- cloud.google.com/go/dataform: v0.6.0 → v0.7.0 +- cloud.google.com/go/dataplex: v1.5.2 → v1.6.0 +- cloud.google.com/go/datastore: v1.10.0 → v1.11.0 +- cloud.google.com/go/datastream: v1.6.0 → v1.7.0 +- cloud.google.com/go/deploy: v1.6.0 → v1.8.0 +- cloud.google.com/go/dialogflow: v1.31.0 → v1.32.0 +- cloud.google.com/go/documentai: v1.16.0 → v1.18.0 +- cloud.google.com/go/edgecontainer: v0.3.0 → v1.0.0 +- cloud.google.com/go/eventarc: v1.10.0 → v1.11.0 +- cloud.google.com/go/filestore: v1.5.0 → v1.6.0 +- cloud.google.com/go/functions: v1.10.0 → v1.13.0 +- cloud.google.com/go/gkehub: v0.11.0 → v0.12.0 +- cloud.google.com/go/iam: v0.12.0 → v0.13.0 +- cloud.google.com/go/iap: v1.6.0 → v1.7.1 +- cloud.google.com/go/iot: v1.5.0 → v1.6.0 +- cloud.google.com/go/kms: v1.9.0 → v1.10.1 +- cloud.google.com/go/maps: v0.6.0 → v0.7.0 +- cloud.google.com/go/monitoring: v1.12.0 → v1.13.0 +- cloud.google.com/go/networkconnectivity: v1.10.0 → v1.11.0 +- cloud.google.com/go/networksecurity: v0.7.0 → v0.8.0 +- cloud.google.com/go/notebooks: v1.7.0 → v1.8.0 +- cloud.google.com/go/policytroubleshooter: v1.5.0 → v1.6.0 +- cloud.google.com/go/privatecatalog: v0.7.0 → v0.8.0 +- cloud.google.com/go/pubsub: v1.28.0 → v1.30.0 +- cloud.google.com/go/pubsublite: v1.6.0 → v1.7.0 +- cloud.google.com/go/recaptchaenterprise/v2: v2.6.0 → v2.7.0 +- cloud.google.com/go/resourcemanager: v1.5.0 → v1.7.0 +- cloud.google.com/go/run: v0.8.0 → v0.9.0 +- cloud.google.com/go/scheduler: v1.8.0 → v1.9.0 +- cloud.google.com/go/security: v1.12.0 → v1.13.0 +- cloud.google.com/go/securitycenter: v1.18.1 → v1.19.0 +- cloud.google.com/go/servicecontrol: v1.11.0 → v1.11.1 +- cloud.google.com/go/servicedirectory: v1.8.0 → v1.9.0 +- cloud.google.com/go/servicemanagement: v1.6.0 → v1.8.0 +- cloud.google.com/go/serviceusage: v1.5.0 → v1.6.0 +- cloud.google.com/go/spanner: v1.44.0 → v1.45.0 +- cloud.google.com/go/speech: v1.14.1 → v1.15.0 +- cloud.google.com/go/storagetransfer: v1.7.0 → v1.8.0 +- cloud.google.com/go/trace: v1.8.0 → v1.9.0 +- cloud.google.com/go/translate: v1.6.0 → v1.7.0 +- cloud.google.com/go/video: v1.13.0 → v1.15.0 +- cloud.google.com/go/vision/v2: v2.6.0 → v2.7.0 +- cloud.google.com/go/vmmigration: v1.5.0 → v1.6.0 +- cloud.google.com/go/vmwareengine: v0.2.2 → v0.3.0 +- github.com/cncf/xds/go: [32f1caf → e9ce688](https://github.com/cncf/xds/go/compare/32f1caf...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.11.0 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.11.0...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.10.0 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.10.0...v0.10.1) +- github.com/googleapis/gax-go/v2: [v2.7.0 → v2.7.1](https://github.com/googleapis/gax-go/v2/compare/v2.7.0...v2.7.1) +- golang.org/x/oauth2: v0.6.0 → v0.7.0 +- google.golang.org/api: v0.110.0 → v0.114.0 +- google.golang.org/genproto: 7f2fa6f → daa745c +- google.golang.org/grpc: v1.55.1 → v1.56.3 + +### Removed +_Nothing has changed._ + +# v1.8.16 - Changelog since v1.8.14 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Changed + +- golang.org/x/crypto: 8634188 → v0.14.0 +- golang.org/x/net: v0.8.0 → v0.17.0 +- golang.org/x/sys: v0.6.0 → v0.13.0 +- golang.org/x/term: v0.6.0 → v0.13.0 +- golang.org/x/text: v0.8.0 → v0.13.0 + +# v1.8.14 - Changelog since v1.8.11 + +## Changes by Kind + +### Bug or Regression + +- bump go version to 1.20.8 ([#1394](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1394), [@tyuchn](https://github.com/tyuchn)) +- Remove ARG BUILDPLATFORM from Dockerfile ([#1385](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1385), [@tyuchn](https://github.com/tyuchn)) +- Filter out user errors from GetDisk error returned from ControllerPublishVolume ([#1380](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1380), [@amacaskill](https://github.com/amacaskill)) + +# v1.8.11 - Changelog since v1.8.10 + +## Changes by Kind + +### Bug or Regression + +- Upgrade google.golang.org/grpc from v1.55.0 -> v1.55.1 to address https://github.com/grpc/grpc-go/issues/6373 ([#1371](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1371), [@pwschuurman](https://github.com/pwschuurman)) + # v1.8.10 - Changelog since v1.8.9 ## Changes by Kind @@ -22,6 +201,7 @@ ## Dependencies ### Added + - cloud.google.com/go/accessapproval: v1.6.0 - cloud.google.com/go/accesscontextmanager: v1.6.0 - cloud.google.com/go/aiplatform: v1.35.0 @@ -137,6 +317,7 @@ - cloud.google.com/go/workflows: v1.10.0 ### Changed + - cloud.google.com/go/bigquery: v1.8.0 → v1.48.0 - cloud.google.com/go/compute: v1.7.0 → v1.18.0 - cloud.google.com/go/datastore: v1.1.0 → v1.10.0 @@ -176,6 +357,7 @@ - google.golang.org/protobuf: v1.28.0 → v1.30.0 ### Removed + - github.com/googleapis/go-type-adapters: [v1.0.0](https://github.com/googleapis/go-type-adapters/tree/v1.0.0) # v1.8.8 - Changelog since v1.8.7 diff --git a/CHANGELOG/CHANGELOG-1.9.md b/CHANGELOG/CHANGELOG-1.9.md index b8b266de..c78dc87e 100644 --- a/CHANGELOG/CHANGELOG-1.9.md +++ b/CHANGELOG/CHANGELOG-1.9.md @@ -1,3 +1,265 @@ +# v1.9.17 - Changelog since v1.9.16 + +## Changes by Kind + +### Bug + +- Change GetDisk error reporting to temporary in CreateVolume codepath ([#1603])https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1603), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + + +# v1.9.16 - Changelog since v1.9.15 + +## Changes by Kind + +### Uncategorized + +- Reduce log spam when identifying NVMe devices located in `/dev` ([#1580](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1580), [@pwschuurman](https://github.com/pwschuurman)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.9.15 - Changelog since v1.9.14 + +## Changes by Kind + +### Uncategorized + +- Bump golang.org/x/crypto from v0.14.0 to v0.17.0 to fix CVE-2023-48795 ([#1552](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1552), [@sunnylovestiramisu](https://github.com/sunnylovestiramisu)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- golang.org/x/crypto: v0.14.0 → v0.17.0 +- golang.org/x/sys: v0.13.0 → v0.15.0 +- golang.org/x/term: v0.13.0 → v0.15.0 +- golang.org/x/text: v0.13.0 → v0.14.0 + +### Removed +_Nothing has changed._ + + +# v1.9.14 - Changelog since v1.9.13 + +## Changes by Kind + +### Uncategorized + +- Properly wrap error from GCE Images.Get() API call, to fix a potential nil-ptr dereference ([#1518](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1518), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) +- Update golang builder to 1.20.12 ([#1537](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1537), [@k8s-infra-cherrypick-robot](https://github.com/k8s-infra-cherrypick-robot)) + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + +# v1.9.13 - Changelog since v1.9.12 + +## Changes by Kind + +### Bug or Regression + +- Bump Golang Builder version to 1.20.11 ([#1505](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1505), [@uriel-guzman](https://github.com/uriel-guzman)) +- Bump google.golang.org/grpc from v1.53.0 to v1.56.3 to fix CVE-2023-44487. ([#1495](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1495), [@uriel-guzman](https://github.com/uriel-guzman)) + +## Dependencies + +### Added +- github.com/google/go-pkcs11: [v0.2.0](https://github.com/google/go-pkcs11/tree/v0.2.0) +- github.com/google/s2a-go: [v0.1.4](https://github.com/google/s2a-go/tree/v0.1.4) +- google.golang.org/genproto/googleapis/api: ccb25ca +- google.golang.org/genproto/googleapis/bytestream: 659f7aa +- google.golang.org/genproto/googleapis/rpc: 659f7aa + +### Changed +- cloud.google.com/go/accessapproval: v1.5.0 → v1.7.1 +- cloud.google.com/go/accesscontextmanager: v1.4.0 → v1.8.1 +- cloud.google.com/go/aiplatform: v1.27.0 → v1.45.0 +- cloud.google.com/go/analytics: v0.12.0 → v0.21.2 +- cloud.google.com/go/apigateway: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeconnect: v1.4.0 → v1.6.1 +- cloud.google.com/go/apigeeregistry: v0.4.0 → v0.7.1 +- cloud.google.com/go/appengine: v1.5.0 → v1.8.1 +- cloud.google.com/go/area120: v0.6.0 → v0.8.1 +- cloud.google.com/go/artifactregistry: v1.9.0 → v1.14.1 +- cloud.google.com/go/asset: v1.10.0 → v1.14.1 +- cloud.google.com/go/assuredworkloads: v1.9.0 → v1.11.1 +- cloud.google.com/go/automl: v1.8.0 → v1.13.1 +- cloud.google.com/go/baremetalsolution: v0.4.0 → v0.5.0 +- cloud.google.com/go/batch: v0.4.0 → v0.7.0 +- cloud.google.com/go/beyondcorp: v0.3.0 → v0.6.1 +- cloud.google.com/go/bigquery: v1.44.0 → v1.52.0 +- cloud.google.com/go/billing: v1.7.0 → v1.16.0 +- cloud.google.com/go/binaryauthorization: v1.4.0 → v1.6.1 +- cloud.google.com/go/certificatemanager: v1.4.0 → v1.7.1 +- cloud.google.com/go/channel: v1.9.0 → v1.16.0 +- cloud.google.com/go/cloudbuild: v1.4.0 → v1.10.1 +- cloud.google.com/go/clouddms: v1.4.0 → v1.6.1 +- cloud.google.com/go/cloudtasks: v1.8.0 → v1.11.1 +- cloud.google.com/go/compute: v1.18.0 → v1.20.1 +- cloud.google.com/go/contactcenterinsights: v1.4.0 → v1.9.1 +- cloud.google.com/go/container: v1.7.0 → v1.22.1 +- cloud.google.com/go/containeranalysis: v0.6.0 → v0.10.1 +- cloud.google.com/go/datacatalog: v1.8.0 → v1.14.1 +- cloud.google.com/go/dataflow: v0.7.0 → v0.9.1 +- cloud.google.com/go/dataform: v0.5.0 → v0.8.1 +- cloud.google.com/go/datafusion: v1.5.0 → v1.7.1 +- cloud.google.com/go/datalabeling: v0.6.0 → v0.8.1 +- cloud.google.com/go/dataplex: v1.4.0 → v1.8.1 +- cloud.google.com/go/dataproc: v1.8.0 → v1.12.0 +- cloud.google.com/go/dataqna: v0.6.0 → v0.8.1 +- cloud.google.com/go/datastore: v1.10.0 → v1.12.0 +- cloud.google.com/go/datastream: v1.5.0 → v1.9.1 +- cloud.google.com/go/deploy: v1.5.0 → v1.11.0 +- cloud.google.com/go/dialogflow: v1.29.0 → v1.38.0 +- cloud.google.com/go/dlp: v1.7.0 → v1.10.1 +- cloud.google.com/go/documentai: v1.10.0 → v1.20.0 +- cloud.google.com/go/domains: v0.7.0 → v0.9.1 +- cloud.google.com/go/edgecontainer: v0.2.0 → v1.1.1 +- cloud.google.com/go/essentialcontacts: v1.4.0 → v1.6.2 +- cloud.google.com/go/eventarc: v1.8.0 → v1.12.1 +- cloud.google.com/go/filestore: v1.4.0 → v1.7.1 +- cloud.google.com/go/firestore: v1.9.0 → v1.11.0 +- cloud.google.com/go/functions: v1.9.0 → v1.15.1 +- cloud.google.com/go/gaming: v1.8.0 → v1.10.1 +- cloud.google.com/go/gkebackup: v0.3.0 → v0.4.0 +- cloud.google.com/go/gkeconnect: v0.6.0 → v0.8.1 +- cloud.google.com/go/gkehub: v0.10.0 → v0.14.1 +- cloud.google.com/go/gkemulticloud: v0.4.0 → v0.6.1 +- cloud.google.com/go/gsuiteaddons: v1.4.0 → v1.6.1 +- cloud.google.com/go/iam: v0.11.0 → v1.1.0 +- cloud.google.com/go/iap: v1.5.0 → v1.8.1 +- cloud.google.com/go/ids: v1.2.0 → v1.4.1 +- cloud.google.com/go/iot: v1.4.0 → v1.7.1 +- cloud.google.com/go/kms: v1.6.0 → v1.12.1 +- cloud.google.com/go/language: v1.8.0 → v1.10.1 +- cloud.google.com/go/lifesciences: v0.6.0 → v0.9.1 +- cloud.google.com/go/logging: v1.6.1 → v1.7.0 +- cloud.google.com/go/longrunning: v0.3.0 → v0.5.1 +- cloud.google.com/go/managedidentities: v1.4.0 → v1.6.1 +- cloud.google.com/go/maps: v0.1.0 → v0.7.0 +- cloud.google.com/go/mediatranslation: v0.6.0 → v0.8.1 +- cloud.google.com/go/memcache: v1.7.0 → v1.10.1 +- cloud.google.com/go/metastore: v1.8.0 → v1.11.1 +- cloud.google.com/go/monitoring: v1.8.0 → v1.15.1 +- cloud.google.com/go/networkconnectivity: v1.7.0 → v1.12.1 +- cloud.google.com/go/networkmanagement: v1.5.0 → v1.8.0 +- cloud.google.com/go/networksecurity: v0.6.0 → v0.9.1 +- cloud.google.com/go/notebooks: v1.5.0 → v1.9.1 +- cloud.google.com/go/optimization: v1.2.0 → v1.4.1 +- cloud.google.com/go/orchestration: v1.4.0 → v1.8.1 +- cloud.google.com/go/orgpolicy: v1.5.0 → v1.11.1 +- cloud.google.com/go/osconfig: v1.10.0 → v1.12.1 +- cloud.google.com/go/oslogin: v1.7.0 → v1.10.1 +- cloud.google.com/go/phishingprotection: v0.6.0 → v0.8.1 +- cloud.google.com/go/policytroubleshooter: v1.4.0 → v1.7.1 +- cloud.google.com/go/privatecatalog: v0.6.0 → v0.9.1 +- cloud.google.com/go/pubsub: v1.27.1 → v1.32.0 +- cloud.google.com/go/pubsublite: v1.5.0 → v1.8.1 +- cloud.google.com/go/recaptchaenterprise/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/recommendationengine: v0.6.0 → v0.8.1 +- cloud.google.com/go/recommender: v1.8.0 → v1.10.1 +- cloud.google.com/go/redis: v1.10.0 → v1.13.1 +- cloud.google.com/go/resourcemanager: v1.4.0 → v1.9.1 +- cloud.google.com/go/resourcesettings: v1.4.0 → v1.6.1 +- cloud.google.com/go/retail: v1.11.0 → v1.14.1 +- cloud.google.com/go/run: v0.3.0 → v0.9.0 +- cloud.google.com/go/scheduler: v1.7.0 → v1.10.1 +- cloud.google.com/go/secretmanager: v1.9.0 → v1.11.1 +- cloud.google.com/go/security: v1.10.0 → v1.15.1 +- cloud.google.com/go/securitycenter: v1.16.0 → v1.23.0 +- cloud.google.com/go/servicedirectory: v1.7.0 → v1.10.1 +- cloud.google.com/go/shell: v1.4.0 → v1.7.1 +- cloud.google.com/go/spanner: v1.41.0 → v1.47.0 +- cloud.google.com/go/speech: v1.9.0 → v1.17.1 +- cloud.google.com/go/storagetransfer: v1.6.0 → v1.10.0 +- cloud.google.com/go/talent: v1.4.0 → v1.6.2 +- cloud.google.com/go/texttospeech: v1.5.0 → v1.7.1 +- cloud.google.com/go/tpu: v1.4.0 → v1.6.1 +- cloud.google.com/go/trace: v1.4.0 → v1.10.1 +- cloud.google.com/go/translate: v1.4.0 → v1.8.1 +- cloud.google.com/go/video: v1.9.0 → v1.17.1 +- cloud.google.com/go/videointelligence: v1.9.0 → v1.11.1 +- cloud.google.com/go/vision/v2: v2.5.0 → v2.7.2 +- cloud.google.com/go/vmmigration: v1.3.0 → v1.7.1 +- cloud.google.com/go/vmwareengine: v0.1.0 → v0.4.1 +- cloud.google.com/go/vpcaccess: v1.5.0 → v1.7.1 +- cloud.google.com/go/webrisk: v1.7.0 → v1.9.1 +- cloud.google.com/go/websecurityscanner: v1.4.0 → v1.6.1 +- cloud.google.com/go/workflows: v1.9.0 → v1.11.1 +- cloud.google.com/go: v0.107.0 → v0.110.4 +- github.com/cncf/xds/go: [06c439d → e9ce688](https://github.com/cncf/xds/go/compare/06c439d...e9ce688) +- github.com/envoyproxy/go-control-plane: [v0.10.3 → 9239064](https://github.com/envoyproxy/go-control-plane/compare/v0.10.3...9239064) +- github.com/envoyproxy/protoc-gen-validate: [v0.9.1 → v0.10.1](https://github.com/envoyproxy/protoc-gen-validate/compare/v0.9.1...v0.10.1) +- github.com/golang/glog: [v1.0.0 → v1.1.0](https://github.com/golang/glog/compare/v1.0.0...v1.1.0) +- github.com/golang/protobuf: [v1.5.2 → v1.5.3](https://github.com/golang/protobuf/compare/v1.5.2...v1.5.3) +- github.com/googleapis/enterprise-certificate-proxy: [v0.2.3 → v0.2.5](https://github.com/googleapis/enterprise-certificate-proxy/compare/v0.2.3...v0.2.5) +- github.com/googleapis/gax-go/v2: [v2.7.0 → v2.12.0](https://github.com/googleapis/gax-go/v2/compare/v2.7.0...v2.12.0) +- github.com/yuin/goldmark: [v1.4.1 → v1.4.13](https://github.com/yuin/goldmark/compare/v1.4.1...v1.4.13) +- golang.org/x/oauth2: v0.5.0 → v0.10.0 +- golang.org/x/sync: v0.1.0 → v0.3.0 +- google.golang.org/api: v0.111.0 → v0.134.0 +- google.golang.org/genproto: 637eb22 → ccb25ca +- google.golang.org/grpc: v1.53.0 → v1.56.3 +- google.golang.org/protobuf: v1.28.1 → v1.31.0 + +### Removed +- cloud.google.com/go/apikeys: v0.4.0 +- cloud.google.com/go/servicecontrol: v1.5.0 +- cloud.google.com/go/servicemanagement: v1.5.0 +- cloud.google.com/go/serviceusage: v1.4.0 + +# v1.9.12 - Changelog since v1.9.10 + +## Changes by Kind + +### Bug or Regression + +- CVE fixes: CVE-2023-39323 ([#1412](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1412), [@dannawang0221](https://github.com/dannawang0221)) + +### Other (Cleanup or Flake) + +- Update go version to 1.20.10 ([#1453](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1453), [@tyuchn](https://github.com/tyuchn)) + +## Dependencies + +### Changed + +- golang.org/x/crypto: v0.11.0 → v0.14.0 +- golang.org/x/net: v0.12.0 → v0.17.0 +- golang.org/x/sys: v0.10.0 → v0.13.0 +- golang.org/x/term: v0.10.0 → v0.13.0 +- golang.org/x/text: v0.11.0 → v0.13.0 + +# v1.9.10 - Changelog since v1.9.9 + +## Changes by Kind + +### Bug or Regression + +- bump go version to 1.20.8 ([#1394](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1394), [@tyuchn](https://github.com/tyuchn)) +- Remove ARG BUILDPLATFORM from Dockerfile ([#1385](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1385), [@tyuchn](https://github.com/tyuchn)) +- Update test/run-e2e.sh to match PROW configuration ([#1360](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1360), [@pwschuurman](https://github.com/pwschuurman)) +- Always call LoggedError for errors returned from CloudProvider methods ([#1381](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1381), [@amacaskill](https://github.com/amacaskill)) + # v1.9.9 - Changelog since v1.9.8 ## Changes by Kind diff --git a/Dockerfile b/Dockerfile index b110a163..62119dbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$BUILDPLATFORM golang:1.21.1-bullseye as builder +FROM --platform=$BUILDPLATFORM golang:1.22.4-bookworm as builder ARG STAGINGVERSION ARG TARGETPLATFORM @@ -41,13 +41,15 @@ RUN apt update --yes && apt install -y libcryptsetup-dev RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=$STAGINGVERSION make gce-pd-driver # Start from Kubernetes Debian base. -FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as debian + +FROM gke.gcr.io/debian-base:bookworm-v1.0.3-gke.0 as debian + # Install necessary dependencies # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash libcryptsetup-dev # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc. -FROM gcr.io/distroless/base-debian11 as distroless-base +FROM gcr.io/distroless/base-debian12 as distroless-base # The distroless amd64 image has a target triplet of x86_64 FROM distroless-base AS distroless-amd64 @@ -93,8 +95,7 @@ COPY --from=debian /bin/udevadm /bin/udevadm COPY --from=debian /sbin/dmsetup /sbin/dmsetup # Copy shared libraries into distroless base. -COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \ - /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ +COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \ @@ -104,6 +105,19 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libreadline.so.8 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libz.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/liburcu.so.8 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libcap.so.2 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libcrypto.so.3 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libdbus-1.so.3 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libjson-c.so.5 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme-mi.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme.so.1 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libpthread.so.0 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libcryptsetup.so.12 \ /lib/${LIB_DIR_PREFIX}-linux-gnu/libcryptsetup.so \ @@ -114,26 +128,19 @@ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libinih.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1.3.0 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.67 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.67 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.67 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.72 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.72 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.72 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libkmod.so.2 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmd.so.0 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0.10.1 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libstdc++.so.6 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libargon2.so.1 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libjson-c.so.5.1.0 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libjson-c.so.5 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/engines-1.1/afalg.so \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/engines-1.1/padlock.so \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcrypto.so.1.1 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libssl.so.1.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libgpg-error.so.0 \ /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/ # Copy NVME support required script and rules into distroless base. diff --git a/Dockerfile.Windows b/Dockerfile.Windows index e13d78e9..bc2570b0 100644 --- a/Dockerfile.Windows +++ b/Dockerfile.Windows @@ -13,7 +13,7 @@ # limitations under the License. ARG BASE_IMAGE -FROM --platform=$BUILDPLATFORM golang:1.20.8 AS builder +FROM --platform=$BUILDPLATFORM golang:1.22.4 AS builder ARG TARGETPLATFORM ARG STAGINGVERSION diff --git a/Dockerfile.debug b/Dockerfile.debug index f2de3527..dc8a2c13 100644 --- a/Dockerfile.debug +++ b/Dockerfile.debug @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.20.8 as builder +FROM golang:1.22.4 as builder WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver ADD . . diff --git a/OWNERS b/OWNERS index bf259fa4..fe7aae6e 100644 --- a/OWNERS +++ b/OWNERS @@ -6,6 +6,8 @@ reviewers: - saikat-royc - sunnylovestiramisu - pwschuurman + - tyuchn + - songjiaxun approvers: - amacaskill - leiyiz @@ -14,6 +16,8 @@ approvers: - saikat-royc - sunnylovestiramisu - pwschuurman + - tyuchn + - songjiaxun emeritus_reviewers: - davidz627 - jingxu97 diff --git a/README.md b/README.md index f0ac68ea..0c8f44f6 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ This driver allows a Constellation cluster to use [GCP Persistent Disks](https:/ | disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. | | labels | `key1=value1,key2=value2` | | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). | | provisioned-iops-on-create | string (int64 format). Values typically between 10,000 and 120,000 | | Indicates how many IOPS to provision for the disk. See the [Extreme persistent disk documentation](https://cloud.google.com/compute/docs/disks/extreme-persistent-disk) for details, including valid ranges for IOPS. | -| provisioned-throughput-on-create | string (int64 format). Values typically between 1 and 7,124 mb per second | | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation](TBD) for details, including valid ranges for throughput. | +| provisioned-throughput-on-create | string (int64 format). Values typically between 1 and 7,124 mb per second | | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk#create) for details, including valid ranges for throughput. | +| resource-tags | `//,//` | | Resource tags allow you to attach user-defined tags to each Compute Disk, Image and Snapshot. See [Tags overview](https://cloud.google.com/resource-manager/docs/tags/tags-overview), [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing). | ### Topology diff --git a/charts/Chart.yaml b/charts/Chart.yaml index ed800823..0380cc53 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -version: 1.3.0 -appVersion: "v1.3.0" +version: 1.4.0 +appVersion: "v1.4.0" description: GCP Compute Persistent Disk Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: gcp-compute-persistent-disk-csi-driver diff --git a/charts/values.yaml b/charts/values.yaml index e620dde5..2ac7e6b5 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -1,28 +1,28 @@ image: csiProvisioner: repo: registry.k8s.io/sig-storage/csi-provisioner - tag: v3.4.0@sha256:e468dddcd275163a042ab297b2d8c2aca50d5e148d2d22f3b6ba119e2f31fa79 + tag: v3.6.3@sha256:10624570c0aceb03f55f1eb07147b0c537e4676869cca2e9bd4bab113f810ac4 pullPolicy: IfNotPresent csiAttacher: repo: registry.k8s.io/sig-storage/csi-attacher - tag: v4.2.0@sha256:34cf9b32736c6624fc9787fb149ea6e0fbeb45415707ac2f6440ac960f1116e6 + tag: v4.4.3@sha256:d7325367ab72b2d469a5091d87b4fc01142d2d13d1a28b2defbbe3e6fdbc4611 pullPolicy: IfNotPresent csiResizer: repo: registry.k8s.io/sig-storage/csi-resizer - tag: v1.7.0@sha256:3a7bdf5d105783d05d0962fa06ca53032b01694556e633f27366201c2881e01d + tag: v1.9.3@sha256:3c116f543f0590aeff3299c8bb0683f250817d11a77d9e9071b15a0bffdabcd9 pullPolicy: IfNotPresent csiSnapshotter: repo: registry.k8s.io/sig-storage/csi-snapshotter - tag: v6.1.0@sha256:291334908ddf71a4661fd7f6d9d97274de8a5378a2b6fdfeb2ce73414a34f82f + tag: v6.3.3@sha256:f1bd6ee18c4021c1c94f29edfab89b49b6a4d1b800936c19dbef2d75f8202f2d pullPolicy: IfNotPresent csiNodeRegistrar: repo: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.7.0@sha256:4a4cae5118c4404e35d66059346b7fa0835d7e6319ff45ed73f4bba335cf5183 + tag: v2.9.3@sha256:0f64602ea791246712b51df334bbd701a0f31df9950a4cb9c28c059f367baa9e pullPolicy: IfNotPresent gcepdDriver: repo: ghcr.io/edgelesssys/constellation/gcp-csi-driver # CSI driver version is independent of Constellation releases - tag: v1.3.0@sha256:0ecb68f348ed6c287075db00f9c5ea731e7e2db9f2f7511b65391fb6856fe11a + tag: v1.4.0@sha256:53d608aa03dd07059bc04e1f8c64e2feb6fceff50fb0cbe276d31a8652a19bac pullPolicy: IfNotPresent csiController: diff --git a/cmd/gce-pd-csi-driver/main.go b/cmd/gce-pd-csi-driver/main.go index 01ae5636..f5714f2a 100644 --- a/cmd/gce-pd-csi-driver/main.go +++ b/cmd/gce-pd-csi-driver/main.go @@ -35,13 +35,18 @@ package main import ( "context" + "errors" "flag" + "fmt" "math/rand" + "net/url" "os" "runtime" + "strings" "time" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" "github.com/edgelesssys/constellation/v2/csi/cryptmapper" cryptKms "github.com/edgelesssys/constellation/v2/csi/kms" @@ -58,12 +63,12 @@ var ( constellationAddr = flag.String("kms-addr", "kms.kube-system:9000", "Address of the Constellation Coordinator's VPN API. Used to request keys (default: kms.kube-system:9000") cloudConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config") endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint") - computeEndpoint = flag.String("compute-endpoint", "", "If set, used as the endpoint for the GCE API.") runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)") runNodeService = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)") httpEndpoint = flag.String("http-endpoint", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.") metricsPath = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.") grpcLogCharCap = flag.Int("grpc-log-char-cap", 10000, "The maximum amount of characters logged for every grpc responses") + enableOtelTracing = flag.Bool("enable-otel-tracing", false, "If set, enable opentelemetry tracing for the driver. The tracing is disabled by default. Configure the exporter endpoint with OTEL_EXPORTER_OTLP_ENDPOINT and other env variables, see https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration.") errorBackoffInitialDurationMs = flag.Int("backoff-initial-duration-ms", 200, "The amount of ms for the initial duration of the backoff condition for controller publish/unpublish CSI operations. Default is 200.") errorBackoffMaxDurationMs = flag.Int("backoff-max-duration-ms", 300000, "The amount of ms for the max duration of the backoff condition for controller publish/unpublish CSI operations. Default is 300000 (5m).") @@ -86,6 +91,21 @@ var ( maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations") formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount") + fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk") + enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools") + + multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable") + multiZoneVolumeHandleEnableFlag = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled") + + computeEnvironment gce.Environment = gce.EnvironmentProduction + computeEndpoint *url.URL + allowedComputeEnvironment = []gce.Environment{gce.EnvironmentStaging, gce.EnvironmentProduction} + + useInstanceAPIOnWaitForAttachDiskTypesFlag = flag.String("use-instance-api-to-poll-attachment-disk-types", "", "Comma separated list of disk types that should use instances.get API when polling for disk attach during ControllerPublish") + useInstanceAPIForListVolumesPublishedNodesFlag = flag.Bool("use-instance-api-to-list-volumes-published-nodes", false, "Enables using the instances.list API to determine published_node_ids in ListVolumes. When false (default), the disks.list API is used") + instancesListFiltersFlag = flag.String("instances-list-filters", "", "Comma separated list of filters to use when calling the instances.list API. By default instances.list fetches all instances in a region") + + extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '//,...,//'. parent_id is the Organization or the Project ID or Project name where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details") version string ) @@ -100,6 +120,8 @@ func init() { // Use V(4) for general debug information logging // Use V(5) for GCE Cloud Provider Call informational logging // Use V(6) for extra repeated/polling information + stringEnumFlag(&computeEnvironment, "compute-environment", allowedComputeEnvironment, "Operating compute environment") + urlFlag(&computeEndpoint, "compute-endpoint", "Compute endpoint") klog.InitFlags(flag.CommandLine) flag.Set("logtostderr", "true") } @@ -107,6 +129,7 @@ func init() { func main() { flag.Parse() rand.Seed(time.Now().UnixNano()) + klog.Infof("Operating compute environment set to: %s and computeEndpoint is set to: %v", computeEnvironment, computeEndpoint) handle() os.Exit(0) } @@ -122,6 +145,22 @@ func handle() { } klog.V(2).Infof("Driver vendor version %v", version) + // Start tracing as soon as possible + if *enableOtelTracing { + exporter, err := driver.InitOtelTracing() + if err != nil { + klog.Fatalf("Failed to initialize otel tracing: %v", err.Error()) + } + // Exporter will flush traces on shutdown + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := exporter.Shutdown(ctx); err != nil { + klog.Errorf("Could not shutdown otel exporter: %v", err.Error()) + } + }() + } + if *runControllerService && *httpEndpoint != "" { mm := metrics.NewMetricsManager() mm.InitializeHttpHandler(*httpEndpoint, *metricsPath) @@ -140,6 +179,14 @@ func handle() { klog.Fatalf("Bad extra volume labels: %v", err.Error()) } + if len(*extraTagsStr) > 0 && !*runControllerService { + klog.Fatalf("Extra tags provided but not running controller") + } + extraTags, err := common.ConvertTagsStringToMap(*extraTagsStr) + if err != nil { + klog.Fatalf("Bad extra tags: %v", err.Error()) + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -149,16 +196,41 @@ func handle() { // Initialize identity server identityServer := driver.NewIdentityServer(gceDriver) + // Initialize requisite zones + fallbackRequisiteZones := parseCSVFlag(*fallbackRequisiteZonesFlag) + + // Initialize multi-zone disk types + multiZoneVolumeHandleDiskTypes := parseCSVFlag(*multiZoneVolumeHandleDiskTypesFlag) + multiZoneVolumeHandleConfig := driver.MultiZoneVolumeHandleConfig{ + Enable: *multiZoneVolumeHandleEnableFlag, + DiskTypes: multiZoneVolumeHandleDiskTypes, + } + + // Initialize waitForAttach config + useInstanceAPIOnWaitForAttachDiskTypes := parseCSVFlag(*useInstanceAPIOnWaitForAttachDiskTypesFlag) + waitForAttachConfig := gce.WaitForAttachConfig{ + UseInstancesAPIForDiskTypes: useInstanceAPIOnWaitForAttachDiskTypes, + } + + // Initialize listVolumes config + instancesListFilters := parseCSVFlag(*instancesListFiltersFlag) + listInstancesConfig := gce.ListInstancesConfig{ + Filters: instancesListFilters, + } + listVolumesConfig := driver.ListVolumesConfig{ + UseInstancesAPIForPublishedNodes: *useInstanceAPIForListVolumesPublishedNodesFlag, + } + // Initialize requirements for the controller service var controllerServer *driver.GCEControllerServer if *runControllerService { - cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint) + cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, computeEndpoint, computeEnvironment, waitForAttachConfig, listInstancesConfig) if err != nil { klog.Fatalf("Failed to get cloud provider: %v", err.Error()) } initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond - controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration) + controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig) } else if *cloudConfigFilePath != "" { klog.Warningf("controller service is disabled but cloud config given - it has no effect") } @@ -186,7 +258,7 @@ func handle() { } } - err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, identityServer, controllerServer, nodeServer) + err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer) if err != nil { klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) } @@ -203,5 +275,64 @@ func handle() { gce.WaitForOpBackoff.Steps = *waitForOpBackoffSteps gce.WaitForOpBackoff.Cap = *waitForOpBackoffCap - gceDriver.Run(*endpoint, *grpcLogCharCap) + gceDriver.Run(*endpoint, *grpcLogCharCap, *enableOtelTracing) +} + +func notEmpty(v string) bool { + return v != "" +} + +func parseCSVFlag(list string) []string { + return slices.Filter(nil, strings.Split(list, ","), notEmpty) +} + +type enumConverter[T any] interface { + convert(v string) (T, error) + eq(a, b T) bool +} + +type stringConverter[T ~string] struct{} + +func (s stringConverter[T]) convert(v string) (T, error) { + return T(v), nil +} + +func (s stringConverter[T]) eq(a, b T) bool { + return a == b +} + +func stringEnumFlag[T ~string](target *T, name string, allowed []T, usage string) { + enumFlag(target, name, stringConverter[T]{}, allowed, usage) +} + +func enumFlag[T any](target *T, name string, converter enumConverter[T], allowed []T, usage string) { + flag.Func(name, usage, func(flagValue string) error { + tValue, err := converter.convert(flagValue) + if err != nil { + return err + } + for _, allowedValue := range allowed { + if converter.eq(allowedValue, tValue) { + *target = tValue + return nil + } + } + errMsg := fmt.Sprintf(`must be one of %v`, allowedComputeEnvironment) + return errors.New(errMsg) + }) +} + +func urlFlag(target **url.URL, name string, usage string) { + flag.Func(name, usage, func(flagValue string) error { + if flagValue == "" { + return nil + } + computeURL, err := url.ParseRequestURI(flagValue) + if err == nil { + *target = computeURL + return nil + } + klog.Errorf("Error parsing endpoint compute endpoint %v", err) + return err + }) } diff --git a/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml b/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml index b1e6ab1c..b8ce6237 100644 --- a/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml +++ b/deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml @@ -48,6 +48,6 @@ metadata: imageTag: name: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.8.11-rc1" + newTag: "v1.13.3-rc1" --- diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml index c80af58f..56a0ac6a 100644 --- a/deploy/kubernetes/images/stable-master/image.yaml +++ b/deploy/kubernetes/images/stable-master/image.yaml @@ -4,7 +4,7 @@ metadata: name: imagetag-csi-provisioner imageTag: name: registry.k8s.io/sig-storage/csi-provisioner - newTag: "v3.4.0" + newTag: "v3.6.3" --- apiVersion: builtin @@ -13,7 +13,7 @@ metadata: name: imagetag-csi-attacher imageTag: name: registry.k8s.io/sig-storage/csi-attacher - newTag: "v4.2.0" + newTag: "v4.4.3" --- apiVersion: builtin @@ -22,7 +22,7 @@ metadata: name: imagetag-csi-resizer imageTag: name: registry.k8s.io/sig-storage/csi-resizer - newTag: "v1.7.0" + newTag: "v1.9.3" --- apiVersion: builtin @@ -31,7 +31,7 @@ metadata: name: imagetag-csi-snapshotter imageTag: name: registry.k8s.io/sig-storage/csi-snapshotter - newTag: "v6.1.0" + newTag: "v6.3.3" --- apiVersion: builtin @@ -40,7 +40,7 @@ metadata: name: imagetag-csi-node-registrar imageTag: name: registry.k8s.io/sig-storage/csi-node-driver-registrar - newTag: "v2.7.0" + newTag: "v2.9.3" --- apiVersion: builtin @@ -52,5 +52,5 @@ imageTag: # Don't change stable image without changing pdImagePlaceholder in # test/k8s-integration/main.go newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver - newTag: "v1.10.1" + newTag: "v1.13.2" --- diff --git a/docs/release/overlays.md b/docs/release/overlays.md index a4b997aa..b00a54ff 100644 --- a/docs/release/overlays.md +++ b/docs/release/overlays.md @@ -10,7 +10,7 @@ When it comes to creation of overlays there are no strict rules on how overlays * Consider a scenario where the existing `stable-master` overlay needs to be changed for a new kubernetes version. * Update the `deploy/kubernetes/images/prow-stable-sidecar-rc-master` directory with appropriate image versions. * Update the `deploy/kubernetes/images/prow-stable-sidecar-rc-master` directory. If there are no additional kustomize diff patches to add, no updates are necessary. - * At this stage, validate the changes made to the `prow-stable-sidecar-rc-master` overlay. Ensure the upstream testgrids like [this](https://k8s-testgrid.appspot.com/provider-gcp-compute-persistent-disk-csi-driver) are green, and verify with repository maintainers that downstream test grids (e.g GKE internal prow test grids) are green. Care must be taken to avoid directly making changes to `deploy/kubernetes/base` manifests at this stage, as they may impact existing overlays. + * At this stage, validate the changes made to the `prow-stable-sidecar-rc-master` overlay. Ensure the upstream testgrids like [this](https://testgrid.k8s.io/provider-gcp-compute-persistent-disk-csi-driver) are green, and verify with repository maintainers that downstream test grids (e.g GKE internal prow test grids) are green. Care must be taken to avoid directly making changes to `deploy/kubernetes/base` manifests at this stage, as they may impact existing overlays. A sample kustomize patch file would look like this: diff --git a/test/k8s-integration/config/sc-ssd.yaml b/examples/kubernetes/demo-sc-with-resource-tags.yaml similarity index 60% rename from test/k8s-integration/config/sc-ssd.yaml rename to examples/kubernetes/demo-sc-with-resource-tags.yaml index 566fcc12..f6a0b471 100644 --- a/test/k8s-integration/config/sc-ssd.yaml +++ b/examples/kubernetes/demo-sc-with-resource-tags.yaml @@ -1,8 +1,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: csi-gcepd-ssd + name: csi-gce-pd-with-resource-tags provisioner: pd.csi.storage.gke.io parameters: - type: pd-ssd + resource-tags: parent1/key1/value1,parent2/key2/value2 volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/image-volumesnapshotclass.yaml b/examples/kubernetes/snapshot/volumesnapshotclass-resource-tags.yaml similarity index 56% rename from test/k8s-integration/config/image-volumesnapshotclass.yaml rename to examples/kubernetes/snapshot/volumesnapshotclass-resource-tags.yaml index ee62f215..6995fee1 100644 --- a/test/k8s-integration/config/image-volumesnapshotclass.yaml +++ b/examples/kubernetes/snapshot/volumesnapshotclass-resource-tags.yaml @@ -1,9 +1,8 @@ apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: - name: csi-gce-image-snapshot-class + name: csi-gce-pd-snapshot-class-with-resource-tags +parameters: + resource-tags: parent1/key1/value1,parent2/key2/value2 driver: pd.csi.storage.gke.io deletionPolicy: Delete -parameters: - snapshot-type: images - image-family: integration-test diff --git a/go.mod b/go.mod index 0591bcca..93839b66 100644 --- a/go.mod +++ b/go.mod @@ -1,109 +1,103 @@ module sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -go 1.21 +go 1.22.0 -toolchain go1.21.1 +toolchain go1.22.2 -require github.com/edgelesssys/constellation/v2 v2.11.1-0.20231005092022-f69ae2612236 +require github.com/edgelesssys/constellation/v2 v2.16.4 replace github.com/martinjungblut/go-cryptsetup => github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c require ( - cloud.google.com/go/compute/metadata v0.2.3 - cloud.google.com/go/kms v1.12.1 + cloud.google.com/go/compute/metadata v0.3.0 + cloud.google.com/go/resourcemanager v1.9.7 github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0 github.com/container-storage-interface/spec v1.6.0 - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.3.1 - github.com/kubernetes-csi/csi-proxy/client v1.1.1 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 + github.com/googleapis/gax-go/v2 v2.12.4 + github.com/kubernetes-csi/csi-proxy/client v1.1.3 github.com/kubernetes-csi/csi-test/v4 v4.4.0 - github.com/onsi/ginkgo/v2 v2.9.1 - github.com/onsi/gomega v1.27.4 - golang.org/x/oauth2 v0.10.0 - golang.org/x/sys v0.12.0 - google.golang.org/api v0.134.0 - google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 - google.golang.org/grpc v1.56.2 - google.golang.org/protobuf v1.31.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 + go.opentelemetry.io/otel/sdk v1.24.0 + golang.org/x/oauth2 v0.21.0 + golang.org/x/sys v0.21.0 + golang.org/x/time v0.5.0 + google.golang.org/api v0.183.0 + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.1 gopkg.in/gcfg.v1 v1.2.3 - k8s.io/api v0.27.3 - k8s.io/apimachinery v0.27.3 + k8s.io/apimachinery v0.29.0 k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible k8s.io/cloud-provider v0.24.1 - k8s.io/component-base v0.27.3 - k8s.io/klog/v2 v2.100.1 - k8s.io/mount-utils v0.27.3 - k8s.io/utils v0.0.0-20230505201702-9f6742963106 - sigs.k8s.io/boskos v0.0.0-20220711194915-6cb8a6fb2dd1 + k8s.io/component-base v0.29.0 + k8s.io/klog/v2 v2.120.1 + k8s.io/mount-utils v0.30.1 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e ) require ( - cloud.google.com/go/compute v1.20.1 // indirect - cloud.google.com/go/iam v1.1.0 // indirect + cloud.google.com/go v0.114.0 // indirect + cloud.google.com/go/auth v0.5.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/iam v1.1.8 // indirect + cloud.google.com/go/longrunning v0.5.7 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect - github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/imdario/mergo v0.3.13 // indirect - github.com/josharian/intern v1.0.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/ginkgo/v2 v2.19.0 // indirect + github.com/onsi/gomega v1.33.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.47.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect - go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/term v0.12.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.13.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8 // indirect + k8s.io/api v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) replace k8s.io/client-go => k8s.io/client-go v0.24.1 diff --git a/go.sum b/go.sum index fe52f8d9..7f7a50ee 100644 --- a/go.sum +++ b/go.sum @@ -1,285 +1,99 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cloud.google.com/go v0.25.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= -cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= -cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.55.0/go.mod h1:ZHmoY+/lIMNkN2+fBmuTiqZ4inFhvQad8ft7MT8IV5Y= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.60.0/go.mod h1:yw2G51M9IfRboUH61Us8GqCeF1PzPblB823Mn2q2eAU= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= -cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= +cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= +cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= +cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= -cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= -cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= -cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= -cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= -cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.4.0/go.mod h1:LFrqilwgdw4X2cJS9ALgzYmMu+ULyrUN6IHV3CPK4TM= +cloud.google.com/go/resourcemanager v1.9.7 h1:SdvD0PaPX60+yeKoSe16mawFpM0EPuiPPihTIVlhRsY= +cloud.google.com/go/resourcemanager v1.9.7/go.mod h1:cQH6lJwESufxEu6KepsoNAsjrUtYYNXRwxm4QFE5g8A= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.9.0/go.mod h1:m+/etGaqZbylxaNT876QGXqEHp4PR2Rq5GMqICWb9bU= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.10.1-0.20200805182106-fcd132957b02/go.mod h1:bdhVveip9CJX75wUu7ALOTnCSKjv6PHRY0bCeBmePnw= -cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= -code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= -contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= -contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs= -contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= -contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= -contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= -contrib.go.opencensus.io/exporter/stackdriver v0.12.9-0.20191108183826-59d068f8d8ff/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= -contrib.go.opencensus.io/exporter/stackdriver v0.13.1/go.mod h1:z2tyTZtPmQ2HvWH4cOmVDgtY+1lomfKdbLnkJvZdc8c= -contrib.go.opencensus.io/exporter/zipkin v0.1.1/go.mod h1:GMvdSl3eJ2gapOaLKzTKE3qDgUkJ86k9k3yY2eqwkzc= -contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= -contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v19.1.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v21.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v42.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= -github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.1.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= -github.com/Azure/go-autorest/autorest v0.2.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.10.2/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.1.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= -github.com/Djarvur/go-err113 v0.1.0/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= -github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= -github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20191009163259-e802c2cb94ae/go.mod h1:mjwGPas4yKduTyubHvD1Atl9r1rUq8DfVy+gkVvZ+oo= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0 h1:a/2jEb6YZq88x+36TcyUu9widTOcGlndZc4L54r+Jys= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0/go.mod h1:SZTgFVlGGwQ/X9NQsExTrYerE+frOeQvIuHCTuAGODI= -github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.3/go.mod h1:f96W2HYy3tiBNV5zbbRc+NczwYHgG1PHXMQfoEWv680= -github.com/GoogleCloudPlatform/testgrid v0.0.7/go.mod h1:lmtHGBL0M/MLbu1tR9BWV7FGZ1FEFIdPqmJiHNCL7y8= -github.com/GoogleCloudPlatform/testgrid v0.0.68/go.mod h1:SIRhudHYGiAUqMwKorBp2Kb5yJKhMq/nEMzFpYlKHVk= -github.com/IBM-Cloud/power-go-client v1.1.4/go.mod h1:YcAHrWuTvckGQYPLLReJ9ijcO/tQuRxAp2kCZ7fnnVk= -github.com/IBM/go-sdk-core/v5 v5.6.3/go.mod h1:tt/B9rxLkRtglE7pvqLuYikgCXaZFL3btdruJaoUeek= -github.com/IBM/go-sdk-core/v5 v5.9.1/go.mod h1:axE2JrRq79gIJTjKPBwV6gWHswvVptBjbcvvCPIxARM= -github.com/IBM/go-sdk-core/v5 v5.9.3/go.mod h1:YlOwV9LeuclmT/qi/LAK2AsobbAP42veV0j68/rlZsE= -github.com/IBM/platform-services-go-sdk v0.19.4/go.mod h1:mKtwiSvf5s2nyaSvcG+GNphun5pmiABcnsPjXzkC0OE= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= -github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.0.3/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig/v3 v3.0.2/go.mod h1:oesJ8kPONMONaZgtiHNzUShJbksypC5kWczhZAf6+aU= -github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U= -github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= -github.com/andygrunwald/go-gerrit v0.0.0-20210709065208-9d38b0be0268/go.mod h1:aqcjwEnmLLSalFNYR0p2ttnEXOVVRctIzsUMHbEcruU= -github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= -github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs= -github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= -github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= -github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= -github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= -github.com/aws/aws-k8s-tester v0.9.3/go.mod h1:nsh1f7joi8ZI1lvR+Ron6kJM2QdCYPU/vFePghSSuTc= -github.com/aws/aws-k8s-tester v1.0.0/go.mod h1:NUNd9k43+h9O5tvwL+4N1Ctb//SapmeeFX1G0/2/0Qc= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.15.90/go.mod h1:es1KtYUFs7le0xQ3rOihkuoVD90z7D0fR2Qm4S00/gU= -github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.23.22/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.29.32/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/aws/aws-sdk-go v1.29.34/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/aws/aws-sdk-go v1.30.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.16/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.31.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.31.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go v1.37.22/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= -github.com/bazelbuild/buildtools v0.0.0-20200922170545-10384511ce98/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= -github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg= -github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= -github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bwmarrin/snowflake v0.0.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= -github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= -github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= -github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -288,126 +102,49 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= -github.com/clarketm/json v1.13.4/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZWMLGg3om8SswStBoY6/SHvcLM19MuZqwDtMtmzs= -github.com/cloudevents/sdk-go v1.0.0/go.mod h1:3TkmM0cFqkhCHOq5JzzRU/RxRkwzoS8TZ+G448qVTog= -github.com/cloudevents/sdk-go/v2 v2.0.0/go.mod h1:3CTrpB4+u7Iaj6fd7E2Xvm5IxMdRoaAhqaRVnOr2rCU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c h1:ToajP6trZoiqlZ3Z4uoG1P02/wtqSw1AcowOXOYjATk= github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c/go.mod h1:gZoZ0+POlM1ge/VUxWpMmZVNPzzMJ7l436CgkQ5+qzU= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/deislabs/oras v0.8.1/go.mod h1:Mx0rMSbBNaNfY9hjpccEnxkOqJL6KGjtxNHPLC4G4As= -github.com/denisenkom/go-mssqldb v0.0.0-20190111225525-2fea367d496d/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= -github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20190925022749-754388324470/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v0.0.0-20200210162036-a4bedce16568/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20191216044856-a8371794149d/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edgelesssys/constellation/v2 v2.11.1-0.20231005092022-f69ae2612236 h1:+elXrNwMmjDGukA/n3/utCBft3uNgGCzFIoEB6Fkvzw= -github.com/edgelesssys/constellation/v2 v2.11.1-0.20231005092022-f69ae2612236/go.mod h1:x+/AAni3uAegCLMWRDE0278Zi/eAFzBnRfxICiLP8LE= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/edgelesssys/constellation/v2 v2.16.4 h1:+TX3rYIAJVz3eVMDMDPkn0wfHGBCqEaRszEq//0CjEA= +github.com/edgelesssys/constellation/v2 v2.16.4/go.mod h1:vgzs/an1+Rup95BUCorTyg9oW1Gjse4iSM2/C+Gk324= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -415,277 +152,66 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= -github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.8.1/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= -github.com/fsouza/fake-gcs-server v1.19.4/go.mod h1:I0/88nHCASqJJ5M7zVF0zKODkYTcuXFW5J5yajsNJnE= -github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= -github.com/go-critic/go-critic v0.4.1/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= -github.com/go-critic/go-critic v0.4.3/go.mod h1:j4O3D4RoIwRqlZw5jJpx0BNfXWWbpcJoKu5cYSe4YmQ= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.46.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.55.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= -github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= -github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= -github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= -github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= -github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= -github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= -github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= -github.com/go-openapi/loads v0.21.0/go.mod h1:rHYve9nZrQ4CJhyeIIFJINGCg1tQpx2yJrrNo8sf1ws= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= -github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= -github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/runtime v0.23.0/go.mod h1:aQg+kaIQEn+A2CRSY1TxbM8+sT9g2V3aLc1FbIAnbbs= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= -github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= -github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= -github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= -github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= -github.com/go-openapi/strfmt v0.20.1/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= -github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= -github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= -github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= -github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= -github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= -github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -695,7 +221,6 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -713,39 +238,11 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.23.7/go.mod h1:g/38bxfhp4rI7zeWSxcdIeHTQGS58TCak8FYcyCmavQ= -github.com/golangci/golangci-lint v1.27.0/go.mod h1:+eZALfxIuthdrHPtfM7w/R3POJLjHDfJJw8XZl9xOng= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= -github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -759,221 +256,84 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE= -github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= -github.com/google/go-containerregistry v0.0.0-20200123184029-53ce695e4179/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= -github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2/go.mod h1:pD1UFYs7MCAx+ZLShBdttcaOSbyc8F9Na/9IZLNwJeA= -github.com/google/go-containerregistry v0.1.1/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0= -github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= -github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= -github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= -github.com/google/go-licenses v0.0.0-20200227160636-0fa8c766a591/go.mod h1:JWeTIGPLQ9gF618ZOdlUitd1gRR/l99WOkHOlmR/UVA= -github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= -github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea h1:VcIYpAGBae3Z6BVncE0OnTE/ZjlDXqtYhOZky88neLM= github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= -github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= -github.com/google/mako v0.0.0-20190821191249-122f8dcef9e3/go.mod h1:YzLcVlL+NqWnmUEPuhS1LxDDwGO9WNbVlEXaF4IH35g= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= -github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/goreleaser/goreleaser v0.136.0/go.mod h1:wiKrPUeSNh6Wu8nUHxZydSOVQ/OZvOaO7DTtFqie904= -github.com/goreleaser/nfpm v1.2.1/go.mod h1:TtWrABZozuLOttX2uDlYyECfQX7x5XYkVxhjYcR6G9w= -github.com/goreleaser/nfpm v1.3.0/go.mod h1:w0p7Kc9TAUgWMyrub63ex3M2Mgw88M4GZXoTq5UCb40= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= -github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= -github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= -github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jenkins-x/go-scm v1.5.65/go.mod h1:MgGRkJScE/rJ30J/bXYqduN5sDPZqZFITJopsnZmTOw= -github.com/jenkins-x/go-scm v1.5.79/go.mod h1:PCT338UhP/pQ0IeEeMEf/hoLTYKcH7qjGEKd7jPkeYg= -github.com/jenkins-x/go-scm v1.5.117/go.mod h1:PCT338UhP/pQ0IeEeMEf/hoLTYKcH7qjGEKd7jPkeYg= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= -github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= -github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= -github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= -github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -983,432 +343,162 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= -github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubernetes-csi/csi-proxy/client v1.1.1 h1:GagYqArF85E/pQuzldTEQ3WmrcNVU/bYPNG+Ha8tTC4= -github.com/kubernetes-csi/csi-proxy/client v1.1.1/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU= +github.com/kubernetes-csi/csi-proxy/client v1.1.3 h1:FdGU7NtxGhQX2wTfnuscmThG920hq0OaVVpuJW9t2k0= +github.com/kubernetes-csi/csi-proxy/client v1.1.3/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU= github.com/kubernetes-csi/csi-test/v4 v4.4.0 h1:r0mnAwDURI24Vw3a/LyA/ga11yD5ZGuU7+REO35Na9s= github.com/kubernetes-csi/csi-test/v4 v4.4.0/go.mod h1:t1RzseMZJKy313nezI/d7TolbbiKpUZM3SXQvXxOX0w= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.1-0.20191009090205-6c0755d89d1e/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= -github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= -github.com/mattn/go-zglob v0.0.2/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= -github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= -github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= -github.com/nats-io/go-nats v1.7.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/octago/sflags v0.2.0/go.mod h1:G0bjdxh4qPRycF74a2B8pU36iTp9QHGx0w0dFZXPt80= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= +github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= -github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8/go.mod h1:CGFX09Ci3pq9QZdj86B+VGIdNj4VyCo2iPOGS9esB/k= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/ryancurrah/gomodguard v1.0.4/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE= -github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I= -github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= -github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= -github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/githubv4 v0.0.0-20180925043049-51d7b505e2e9/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/graphql v0.0.0-20180924043259-e4a3a37e6d42/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= -github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= -github.com/sourcegraph/go-diff v0.5.3/go.mod h1:v9JDtjCE4HHHCZGId75rg8gkKKa98RVjBcBGsVmMmak= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= -github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1418,93 +508,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= -github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= -github.com/tektoncd/pipeline v0.8.0/go.mod h1:IZzJdiX9EqEMuUcgdnElozdYYRh0/ZRC+NKMLj1K3Yw= -github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= -github.com/tektoncd/pipeline v0.11.0/go.mod h1:hlkH32S92+/UODROH0dmxzyuMxfRFp/Nc3e29MewLn8= -github.com/tektoncd/pipeline v0.13.1-0.20200625065359-44f22a067b75/go.mod h1:R5AlT46x/F8n/pFJFjZ1U1q71GWtVXgG7RZkkoRL554= -github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= -github.com/tektoncd/plumbing v0.0.0-20200217163359-cd0db6e567d2/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= -github.com/tektoncd/plumbing v0.0.0-20200430135134-e53521e1d887/go.mod h1:cZPJIeTIoP7UPTxQyTQLs7VE1TiXJSNj0te+If4Q+jI= -github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= -github.com/tetafro/godot v0.3.7/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= -github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= -github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= -github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= -github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= -github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= -github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= -github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= -github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= -github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible/go.mod h1:Smz/ZWfhKRcyDDChZkG3CyTHdj87lHzio/HOCkbndXM= -github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= -github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= -github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= -github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= -github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= -github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= @@ -1515,25 +532,8 @@ go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46O go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1543,96 +543,61 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.9.2-0.20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= -go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1641,7 +606,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1656,56 +620,37 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1718,39 +663,28 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1760,16 +694,14 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1777,10 +709,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1788,73 +718,40 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190219203350-90b0e4468f99/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191119060738-e882bf8e40c2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1865,159 +762,92 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190807223507-b346f7fd45de/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112005509-a3f652f18032/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113232020-e2727e816f5a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191118222007-07fc4c7f2b98/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200102140908-9497f49d5709/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200115165105-de0b1760071a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204192400-7124308813f3/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200210192313-1ace956b0e17/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200214144324-88be01311a71/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200331202046-9d5940d49312/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200701000337-a32c0cb1d5b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2029,33 +859,16 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= -google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= @@ -2065,52 +878,31 @@ google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.26.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= -google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= -google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -2123,8 +915,6 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200326112834-f447254575fd/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -2132,23 +922,13 @@ google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200528110217-3d3490e7e671/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200701001935-0939c5918c31/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804151602-45615f50871c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2160,28 +940,16 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 h1:Au6te5hbKUV8pIYWHqOUZ1pva5qK/rwbIhoXEUB9Lu8= -google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 h1:Z8qdAF9GFsmcUuWQ5KVYIpP3PCKydn/YKORnghIalu4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= +google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -2191,7 +959,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= @@ -2201,10 +968,9 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2218,293 +984,93 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= -gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -helm.sh/helm/v3 v3.1.1/go.mod h1:WYsFJuMASa/4XUqLyv54s0U/f3mlAaRErGmyy4z921g= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.0.0-20180904230853-4e7be11eab3f/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= -k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= -k8s.io/api v0.16.13/go.mod h1:QWu8UWSTiuQZMMeYjwLs6ILu5O74qKSJ0c+4vrchDxs= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= -k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= -k8s.io/api v0.17.3/go.mod h1:YZ0OTkuw7ipbe305fMpIdf3GLXZKRigjtZaV5gzC2J0= -k8s.io/api v0.17.4/go.mod h1:5qxx6vjmwUVG2nHQTKGlLts8Tbok8PzHl4vHtVFuZCA= -k8s.io/api v0.17.6/go.mod h1:1jKVwkj0UZ4huak/yRt3MFfU5wc32+B41SkNN5HhyFg= -k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.24.1/go.mod h1:JhoOvNiLXKTPQ60zh2g0ewpA+bnEYf5q44Flhquh4vQ= -k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= -k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg= -k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= -k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= -k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= -k8s.io/apiextensions-apiserver v0.17.6/go.mod h1:Z3CHLP3Tha+Rbav7JR3S+ye427UaJkHBomK2c4XtZ3A= -k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= -k8s.io/apimachinery v0.0.0-20180904193909-def12e63c512/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= -k8s.io/apimachinery v0.0.0-20190816221834-a9f1d8a9c101/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BFw7lAZPCx7scubsTfosqHkhXCWJKw= -k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= -k8s.io/apimachinery v0.16.5-beta.1/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= -k8s.io/apimachinery v0.16.13/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= -k8s.io/apimachinery v0.17.4/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= -k8s.io/apimachinery v0.17.6/go.mod h1:Lg8zZ5iC/O8UjCqW6DNhcQG2m4TdjF9kwG3891OWbbA= -k8s.io/apimachinery v0.18.5/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= +k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= +k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= k8s.io/apimachinery v0.24.1/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= -k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= -k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= -k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= -k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= -k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= -k8s.io/apiserver v0.17.4/go.mod h1:5ZDQ6Xr5MNBxyi3iUZXS84QOhZl+W7Oq2us/29c0j9I= -k8s.io/apiserver v0.17.6/go.mod h1:sAYqm8hUDNA9aj/TzqwsJoExWrxprKv0tqs/z88qym0= -k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= +k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= +k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/apiserver v0.24.1/go.mod h1:dQWNMx15S8NqJMp0gpYfssyvhYnkilc1LpExd/dkLh0= -k8s.io/cli-runtime v0.17.2/go.mod h1:aa8t9ziyQdbkuizkNLAw3qe3srSyWh9zlSB7zTqRNPI= -k8s.io/cli-runtime v0.17.3/go.mod h1:X7idckYphH4SZflgNpOOViSxetiMj6xI0viMAjM81TA= k8s.io/client-go v0.24.1 h1:w1hNdI9PFrzu3OlovVeTnf4oHDt+FJLd9Ndluvnb42E= k8s.io/client-go v0.24.1/go.mod h1:f1kIDqcEYmwXS/vTbbhopMUbhKp2JhOeVTfxgaCIlF8= -k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= -k8s.io/cloud-provider v0.17.4/go.mod h1:XEjKDzfD+b9MTLXQFlDGkk6Ho8SGMpaU8Uugx/KNK9U= k8s.io/cloud-provider v0.24.1 h1:SaQNq2Ax+epdY9wFngwN9GWpOVnM72hUqr2qy20cOvg= k8s.io/cloud-provider v0.24.1/go.mod h1:h5m/KIiwiQ76hpUBsgrwm/rxteIfJG9kJQ/+/w1as2M= -k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= -k8s.io/code-generator v0.0.0-20190831074504-732c9ca86353/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/code-generator v0.16.4/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= -k8s.io/code-generator v0.17.1/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/code-generator v0.17.3/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ= -k8s.io/code-generator v0.17.6/go.mod h1:iiHz51+oTx+Z9D0vB3CH3O4HDDPWrvZyUgUYaIE9h9M= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= -k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= -k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= -k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= -k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= -k8s.io/component-base v0.17.4/go.mod h1:5BRqHMbbQPm2kKu35v3G+CpVq4K0RJKC7TRioF0I9lE= -k8s.io/component-base v0.17.6/go.mod h1:jgRLWl0B0rOzFNtxQ9E4BphPmDqoMafujdau6AdG2Xo= -k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= k8s.io/component-base v0.24.1/go.mod h1:DW5vQGYVCog8WYpNob3PMmmsY8A3L9QZNg4j/dV3s38= -k8s.io/component-base v0.27.3 h1:g078YmdcdTfrCE4fFobt7qmVXwS8J/3cI1XxRi/2+6k= -k8s.io/component-base v0.27.3/go.mod h1:JNiKYcGImpQ44iwSYs6dysxzR9SxIIgQalk4HaCNVUY= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= k8s.io/component-helpers v0.24.1/go.mod h1:q5Z1pWV/QfX9ThuNeywxasiwkLw9KsR4Q9TAOdb/Y3s= k8s.io/controller-manager v0.24.1/go.mod h1:g105ENexD6A2holEq7Bl6ae+69LJHiLnoEEm7wkE6sc= -k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= -k8s.io/csi-translation-lib v0.17.4/go.mod h1:CsxmjwxEI0tTNMzffIAcgR9lX4wOh6AKHdxQrT7L0oo= -k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190306031000-7a1b7fb0289f/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20191108084044-e500ee069b5c/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200205140755-e0e292d8aa12/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= -k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/kubectl v0.17.2/go.mod h1:y4rfLV0n6aPmvbRCqZQjvOp3ezxsFgpqL+zF5jH/lxk= -k8s.io/kubernetes v1.11.10/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/kubernetes v1.14.7/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= -k8s.io/legacy-cloud-providers v0.17.4/go.mod h1:FikRNoD64ECjkxO36gkDgJeiQWwyZTuBkhu+yxOc1Js= -k8s.io/metrics v0.17.2/go.mod h1:3TkNHET4ROd+NfzNxkjoVfQ0Ob4iZnaHmSEA4vYpwLw= -k8s.io/mount-utils v0.27.3 h1:oubkDKLTZUneW27wgyOmp8a1AAZj04vGmtq+YW8wdvY= -k8s.io/mount-utils v0.27.3/go.mod h1:vmcjYdi2Vg1VTWY7KkhvwJVY6WDHxb/QQhiQKkR8iNs= -k8s.io/test-infra v0.0.0-20181019233642-2e10a0bbe9b3/go.mod h1:2NzXB13Ji0nqpyublHeiPC4FZwU0TknfvyaaNfl/BTA= -k8s.io/test-infra v0.0.0-20191212060232-70b0b49fe247/go.mod h1:d8SKryJBXAwfCFVL4wieRez47J2NOOAb9d029sWLseQ= -k8s.io/test-infra v0.0.0-20200407001919-bc7f71ef65b8/go.mod h1:/WpJWcaDvuykB322WXP4kJbX8IpalOzuPxA62GpwkJk= -k8s.io/test-infra v0.0.0-20200514184223-ba32c8aae783/go.mod h1:bW6thaPZfL2hW7ecjx2WYwlP9KQLM47/xIJyttkVk5s= -k8s.io/test-infra v0.0.0-20200617221206-ea73eaeab7ff/go.mod h1:L3+cRvwftUq8IW1TrHji5m3msnc4uck/7LsE/GR/aZk= -k8s.io/test-infra v0.0.0-20200630233406-1dca6122872e/go.mod h1:L3+cRvwftUq8IW1TrHji5m3msnc4uck/7LsE/GR/aZk= -k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8 h1:pTVP23RROkJO3egAdVkF4kcAhY1ODTorTwhOcWk7c7s= -k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8/go.mod h1:RXgSaKbQA0upN4GGyH38yRkotDJr3myiKWkvdfB5yP4= -k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20190907131718-3d4f5b7dea0b/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/mount-utils v0.30.1 h1:4HEFqo2bzRjCHHXRu7yQh6tvpMnplwWaqhuU7oE3710= +k8s.io/mount-utils v0.30.1/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= -k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= -knative.dev/caching v0.0.0-20200116200605-67bca2c83dfa/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= -knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= -knative.dev/eventing-contrib v0.11.2/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= -knative.dev/pkg v0.0.0-20191101194912-56c2594e4f11/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= -knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= -knative.dev/pkg v0.0.0-20200207155214-fef852970f43/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= -knative.dev/pkg v0.0.0-20200428194351-90fc61bae7f7/go.mod h1:o+e8OVEJKIuvXPsGVPIautjXgs05xbos7G+QMRjuUps= -knative.dev/pkg v0.0.0-20200505191044-3da93ebb24c2/go.mod h1:Q6sL35DdGs8hIQZKdaCXJGgY8f90BmNBKSb8z6d/BTM= -knative.dev/pkg v0.0.0-20200515002500-16d7b963416f/go.mod h1:tMOHGbxtRz8zYFGEGpV/bpoTEM1o89MwYFC4YJXl3GY= -knative.dev/pkg v0.0.0-20200528142800-1c6815d7e4c9/go.mod h1:QgNZTxnwpB/oSpNcfnLVlw+WpEwwyKAvJlvR3hgeltA= -knative.dev/pkg v0.0.0-20200711004937-22502028e31a/go.mod h1:AqAJV6rYi8IGikDjJ/9ZQd9qKdkXVlesVnVjwx62YB8= -knative.dev/test-infra v0.0.0-20200407185800-1b88cb3b45a5/go.mod h1:xcdUkMJrLlBswIZqL5zCuBFOC22WIPMQoVX1L35i0vQ= -knative.dev/test-infra v0.0.0-20200505052144-5ea2f705bb55/go.mod h1:WqF1Azka+FxPZ20keR2zCNtiQA1MP9ZB4BH4HuI+SIU= -knative.dev/test-infra v0.0.0-20200513011557-d03429a76034/go.mod h1:aMif0KXL4g19YCYwsy4Ocjjz5xgPlseYV+B95Oo4JGE= -knative.dev/test-infra v0.0.0-20200519015156-82551620b0a9/go.mod h1:A5b2OAXTOeHT3hHhVQm3dmtbuWvIDP7qzgtqxA3/2pE= -knative.dev/test-infra v0.0.0-20200707183444-aed09e56ddc7/go.mod h1:RjYAhXnZqeHw9+B0zsbqSPlae0lCvjekO/nw5ZMpLCs= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc= -mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= -pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= -pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/boskos v0.0.0-20200526191642-45fc818e2d00/go.mod h1:L1ubP7d1CCMSQSjKiZv6dGbh7b4kfoG+dFPj8cfYDnI= -sigs.k8s.io/boskos v0.0.0-20200617235605-f289ba6555ba/go.mod h1:ZO5RV+VxJS9mb6DvZ1yAjywoyq/wQ8b0vDoZxcIA5kE= -sigs.k8s.io/boskos v0.0.0-20220711194915-6cb8a6fb2dd1 h1:KCcH/1chmSLMuUIKT0d0AT9rIapPkO1dXUJ7n128Org= -sigs.k8s.io/boskos v0.0.0-20220711194915-6cb8a6fb2dd1/go.mod h1:VELvEunXNBH+wKjZlM3C7dOIcqbr0twBPRaA4B0PtlU= -sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= -sigs.k8s.io/controller-runtime v0.5.0/go.mod h1:REiJzC7Y00U+2YkMbT8wxgrsX5USpXKGhb2sCtAXiT8= -sigs.k8s.io/controller-runtime v0.5.4/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A= -sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= -sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= -sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/generate-patch-release-notes.sh b/hack/generate-patch-release-notes.sh new file mode 100755 index 00000000..9abf0d8e --- /dev/null +++ b/hack/generate-patch-release-notes.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Usage: generate_patch_release_notes.sh +# +# Generates and creates commits for GCP CSI driver patch release notes. +# +# Required environment variables: +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# +# Prerequisites: +# - This script creates and deletes the origin/changelog branch in your git +# workspace. Make sure you are not using the branch for anything else. +# +# Instructions: +# 1. Update the versions in the $releases array +# 2. Set environment variables +# 3. Run the script +# 4. The script pushes the commits to your origin/changelog branch +# 5. Make any modifications as needed +# 6. Create a PR +# +# Caveats: +# - This script doesn't handle regenerating and updating existing branches. +# - The "--start-rev" option in the release-notes generator is inclusive, which +# causes the last commit from the last patch to show up in the release notes (if +# there was a release note). This needs to be manually modified until a solution is found. + +set -e +set -x + +repo="gcp-compute-persistent-disk-csi-driver" +releases=( + "1.12.5" + "1.11.7" + "1.10.12" + "1.9.14" + "1.8.18" + "1.7.19" +) + +function gen_patch_relnotes() { + rm out.md || true + rm -rf /tmp/k8s-repo || true + GITHUB_TOKEN=$CSI_RELEASE_TOKEN \ + release-notes --start-rev=$3 --end-rev=$2 --branch=$2 \ + --org=kubernetes-sigs --repo=$1 \ + --required-author="" --markdown-links --output out.md +} + +script_dir="$(dirname "$(readlink -f "$0")")" +pushd "$script_dir/../CHANGELOG" + +# Create branch +git fetch upstream +git checkout master +git rebase upstream/master + +branch="changelog" +if [ `git rev-parse --verify "$branch" 2>/dev/null` ]; then + git branch -D "$branch" +fi +git checkout -b $branch + +for version in "${releases[@]}"; do + # Parse minor and patch version + minorPatchPattern="(^[[:digit:]]+\.[[:digit:]]+)\.([[:digit:]]+)" + [[ "$version" =~ $minorPatchPattern ]] + minor="${BASH_REMATCH[1]}" + patch="${BASH_REMATCH[2]}" + + echo $repo $version $minor $patch + newVer="v$minor.$patch" + prevPatch="$(($patch-1))" + prevVer="v$minor.$prevPatch" + + # Generate release notes + gen_patch_relnotes $repo release-$minor $prevVer + cat > tmp.md <> tmp.md + echo >> tmp.md + echo >> tmp.md + + file="CHANGELOG-$minor.md" + cat $file >> tmp.md + mv tmp.md $file + + git add -u + git commit -m "Add changelog for $version" +done + +git push -f origin $branch + +popd diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 9ca91f40..9851b11d 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -24,4 +24,12 @@ const ( VolumeAttributePartition = "partition" UnspecifiedValue = "UNSPECIFIED" + + // Keyword indicating a 'multi-zone' volumeHandle. Replaces "zones" in the volumeHandle: + // eg: projects/{project}/zones/multi-zone/disks/{name} vs. + // projects/{project}/zones/{zone}/disks/{name} + MultiZoneValue = "multi-zone" + + // Label that is set on a disk when it is used by a 'multi-zone' VolumeHandle + MultiZoneLabel = "goog-gke-multi-zone" ) diff --git a/pkg/common/errors.go b/pkg/common/errors.go new file mode 100644 index 00000000..afd83e08 --- /dev/null +++ b/pkg/common/errors.go @@ -0,0 +1,58 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TemporaryError wraps an error with a temporary error code. +// It implements the error interface. Do not return TemporaryError +// directly from CSI Spec API calls, as CSI Spec API calls MUST +// return a standard gRPC status. If TemporaryErrors are returned from +// helper functions within a CSI Spec API method, make sure the outer CSI +// Spec API method returns a standard gRPC status. (e.g. LoggedError(tempErr) ) +type TemporaryError struct { + err error + code codes.Code +} + +// Unwrap extracts the original error. +func (te *TemporaryError) Unwrap() error { + return te.err +} + +// GRPCStatus extracts the underlying gRPC Status error. +// This method is necessary to fulfill the grpcstatus interface +// described in https://pkg.go.dev/google.golang.org/grpc/status#FromError. +// FromError is used in CodeForError to get existing error codes from status errors. +func (te *TemporaryError) GRPCStatus() *status.Status { + if te.err == nil { + return status.New(codes.OK, "") + } + return status.New(te.code, te.err.Error()) +} + +func NewTemporaryError(code codes.Code, err error) *TemporaryError { + return &TemporaryError{err: err, code: code} +} + +// Error returns a readable representation of the TemporaryError. +func (te *TemporaryError) Error() string { + return te.err.Error() +} diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 4c7e5e44..2a1fa4d1 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -31,6 +31,9 @@ const ( ParameterKeyProvisionedThroughputOnCreate = "provisioned-throughput-on-create" ParameterAvailabilityClass = "availability-class" ParameterKeyEnableConfidentialCompute = "enable-confidential-storage" + ParameterKeyStoragePools = "storage-pools" + ParameterKeyResourceTags = "resource-tags" + ParameterKeyEnableMultiZoneProvisioning = "enable-multi-zone-provisioning" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -94,6 +97,15 @@ type DiskParameters struct { EnableConfidentialCompute bool // Default: false ForceAttach bool + // Values: {[]string} + // Default: "" + StoragePools []StoragePool + // Values: {map[string]string} + // Default: "" + ResourceTags map[string]string + // Values: {bool} + // Default: false + MultiZoneProvisioning bool } // SnapshotParameters contains normalized and defaulted parameters for snapshots @@ -103,25 +115,44 @@ type SnapshotParameters struct { ImageFamily string Tags map[string]string Labels map[string]string + ResourceTags map[string]string +} + +type StoragePool struct { + Project string + Zone string + Name string + ResourceName string +} + +type ParameterProcessor struct { + DriverName string + EnableStoragePools bool + EnableMultiZone bool } // ExtractAndDefaultParameters will take the relevant parameters from a map and // put them into a well defined struct making sure to default unspecified fields. // extraVolumeLabels are added as labels; if there are also labels specified in // parameters, any matching extraVolumeLabels will be overridden. -func ExtractAndDefaultParameters(parameters map[string]string, driverName string, extraVolumeLabels map[string]string) (DiskParameters, error) { +func (pp *ParameterProcessor) ExtractAndDefaultParameters(parameters map[string]string, extraVolumeLabels map[string]string, extraTags map[string]string) (DiskParameters, error) { p := DiskParameters{ DiskType: "pd-standard", // Default ReplicationType: replicationTypeNone, // Default DiskEncryptionKMSKey: "", // Default Tags: make(map[string]string), // Default Labels: make(map[string]string), // Default + ResourceTags: make(map[string]string), // Default } for k, v := range extraVolumeLabels { p.Labels[k] = v } + for k, v := range extraTags { + p.ResourceTags[k] = v + } + for k, v := range parameters { if k == "csiProvisionerSecretName" || k == "csiProvisionerSecretNamespace" { // These are hardcoded secrets keys required to function but not needed by GCE PD @@ -183,29 +214,60 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string if paramEnableConfidentialCompute { // DiskEncryptionKmsKey is needed to enable confidentialStorage if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) { - return p, fmt.Errorf("Valid %v is required to enbale ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) + return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) } } p.EnableConfidentialCompute = paramEnableConfidentialCompute + case ParameterKeyStoragePools: + if !pp.EnableStoragePools { + return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools) + } + storagePools, err := ParseStoragePools(v) + if err != nil { + return p, fmt.Errorf("parameters contains invalid value for %s parameter %q: %w", ParameterKeyStoragePools, v, err) + } + p.StoragePools = storagePools + case ParameterKeyResourceTags: + if err := extractResourceTagsParameter(v, p.ResourceTags); err != nil { + return p, err + } + case ParameterKeyEnableMultiZoneProvisioning: + if !pp.EnableMultiZone { + return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyEnableMultiZoneProvisioning) + } + paramEnableMultiZoneProvisioning, err := ConvertStringToBool(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyEnableMultiZoneProvisioning, err) + } + p.MultiZoneProvisioning = paramEnableMultiZoneProvisioning + if paramEnableMultiZoneProvisioning { + p.Labels[MultiZoneLabel] = "true" + } default: return p, fmt.Errorf("parameters contains invalid option %q", k) } } if len(p.Tags) > 0 { - p.Tags[tagKeyCreatedBy] = driverName + p.Tags[tagKeyCreatedBy] = pp.DriverName } return p, nil } -func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string) (SnapshotParameters, error) { +func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string, extraTags map[string]string) (SnapshotParameters, error) { p := SnapshotParameters{ StorageLocations: []string{}, SnapshotType: DiskSnapshotType, Tags: make(map[string]string), // Default Labels: make(map[string]string), // Default + ResourceTags: make(map[string]string), // Default + } + + for k, v := range extraTags { + p.ResourceTags[k] = v } + for k, v := range parameters { switch strings.ToLower(k) { case ParameterKeyStorageLocations: @@ -237,6 +299,10 @@ func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverNam for labelKey, labelValue := range paramLabels { p.Labels[labelKey] = labelValue } + case ParameterKeyResourceTags: + if err := extractResourceTagsParameter(v, p.ResourceTags); err != nil { + return p, err + } default: return p, fmt.Errorf("parameters contains invalid option %q", k) } @@ -246,3 +312,15 @@ func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverNam } return p, nil } + +func extractResourceTagsParameter(tagsString string, resourceTags map[string]string) error { + paramResourceTags, err := ConvertTagsStringToMap(tagsString) + if err != nil { + return fmt.Errorf("parameters contain invalid %s parameter: %w", ParameterKeyResourceTags, err) + } + // Override any existing resource tags with those from this parameter. + for tagParentIDKey, tagValue := range paramResourceTags { + resourceTags[tagParentIDKey] = tagValue + } + return nil +} diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index acffe5a5..75f1c2a1 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -19,15 +19,20 @@ package common import ( "reflect" "testing" + + "github.com/google/go-cmp/cmp" ) func TestExtractAndDefaultParameters(t *testing.T) { tests := []struct { - name string - parameters map[string]string - labels map[string]string - expectParams DiskParameters - expectErr bool + name string + parameters map[string]string + labels map[string]string + enableStoragePools bool + enableMultiZone bool + extraTags map[string]string + expectParams DiskParameters + expectErr bool }{ { name: "defaults", @@ -39,11 +44,12 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { name: "specified empties", - parameters: map[string]string{ParameterKeyType: "", ParameterKeyReplicationType: "", ParameterKeyDiskEncryptionKmsKey: "", ParameterKeyLabels: ""}, + parameters: map[string]string{ParameterKeyType: "", ParameterKeyReplicationType: "", ParameterKeyDiskEncryptionKmsKey: "", ParameterKeyLabels: "", ParameterKeyResourceTags: ""}, labels: map[string]string{}, expectParams: DiskParameters{ DiskType: "pd-standard", @@ -51,6 +57,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { @@ -61,7 +68,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { }, { name: "values from parameters", - parameters: map[string]string{ParameterKeyType: "pd-ssd", ParameterKeyReplicationType: "regional-pd", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2"}, + parameters: map[string]string{ParameterKeyType: "pd-ssd", ParameterKeyReplicationType: "regional-pd", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyResourceTags: "parent1/key1/value1,parent2/key2/value2"}, labels: map[string]string{}, expectParams: DiskParameters{ DiskType: "pd-ssd", @@ -72,11 +79,15 @@ func TestExtractAndDefaultParameters(t *testing.T) { "key1": "value1", "key2": "value2", }, + ResourceTags: map[string]string{ + "parent1/key1": "value1", + "parent2/key2": "value2", + }, }, }, { name: "values from parameters, checking pd-extreme", - parameters: map[string]string{ParameterKeyType: "pd-extreme", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyProvisionedIOPSOnCreate: "10k"}, + parameters: map[string]string{ParameterKeyType: "pd-extreme", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyResourceTags: "parent1/key1/value1,parent2/key2/value2", ParameterKeyProvisionedIOPSOnCreate: "10k"}, labels: map[string]string{}, expectParams: DiskParameters{ DiskType: "pd-extreme", @@ -87,12 +98,16 @@ func TestExtractAndDefaultParameters(t *testing.T) { "key1": "value1", "key2": "value2", }, + ResourceTags: map[string]string{ + "parent1/key1": "value1", + "parent2/key2": "value2", + }, ProvisionedIOPSOnCreate: 10000, }, }, { name: "values from parameters, checking hyperdisk-throughput", - parameters: map[string]string{ParameterKeyType: "hyperdisk-throughput", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyProvisionedThroughputOnCreate: "1000Mi"}, + parameters: map[string]string{ParameterKeyType: "hyperdisk-throughput", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyResourceTags: "parent1/key1/value1,parent2/key2/value2", ParameterKeyProvisionedThroughputOnCreate: "1000Mi"}, labels: map[string]string{}, expectParams: DiskParameters{ DiskType: "hyperdisk-throughput", @@ -103,6 +118,10 @@ func TestExtractAndDefaultParameters(t *testing.T) { "key1": "value1", "key2": "value2", }, + ResourceTags: map[string]string{ + "parent1/key1": "value1", + "parent2/key2": "value2", + }, ProvisionedThroughputOnCreate: 1000, }, }, @@ -116,6 +135,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "foo/key", Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { @@ -128,6 +148,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "foo/key", Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { @@ -140,6 +161,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{tagKeyCreatedForClaimName: "testPVCName", tagKeyCreatedForClaimNamespace: "testPVCNamespace", tagKeyCreatedForVolumeName: "testPVName", tagKeyCreatedBy: "testDriver"}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { @@ -152,6 +174,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{}, Labels: map[string]string{"label-1": "label-value-1", "label-2": "label-value-2"}, + ResourceTags: map[string]string{}, }, }, { @@ -164,6 +187,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{}, Labels: map[string]string{"key1": "value1", "key2": "value2", "label-1": "label-value-1", "label-2": "label-value-2"}, + ResourceTags: map[string]string{}, }, }, { @@ -176,6 +200,56 @@ func TestExtractAndDefaultParameters(t *testing.T) { DiskEncryptionKMSKey: "", Tags: map[string]string{}, Labels: map[string]string{"key1": "value1", "label-1": "value-a", "label-2": "label-value-2"}, + ResourceTags: map[string]string{}, + }, + }, + { + name: "extra tags", + parameters: map[string]string{}, + extraTags: map[string]string{"parent1/key1": "value1", "parent2/key2": "value2"}, + expectParams: DiskParameters{ + DiskType: "pd-standard", + ReplicationType: "none", + DiskEncryptionKMSKey: "", + Tags: map[string]string{}, + Labels: map[string]string{}, + ResourceTags: map[string]string{ + "parent1/key1": "value1", + "parent2/key2": "value2", + }, + }, + }, + { + name: "resource-tags parameter and extra tags", + parameters: map[string]string{ParameterKeyResourceTags: "parent3/key3/value3"}, + extraTags: map[string]string{"parent1/key1": "value1", "parent2/key2": "value2"}, + expectParams: DiskParameters{ + DiskType: "pd-standard", + ReplicationType: "none", + DiskEncryptionKMSKey: "", + Tags: map[string]string{}, + Labels: map[string]string{}, + ResourceTags: map[string]string{ + "parent1/key1": "value1", + "parent2/key2": "value2", + "parent3/key3": "value3", + }, + }, + }, + { + name: "resource-tags parameter and extra labels, overlapping", + parameters: map[string]string{ParameterKeyResourceTags: "parent1/key1/value-a"}, + extraTags: map[string]string{"parent1/key1": "value-b", "parent2/key2": "value2"}, + expectParams: DiskParameters{ + DiskType: "pd-standard", + ReplicationType: "none", + DiskEncryptionKMSKey: "", + Tags: map[string]string{}, + Labels: map[string]string{}, + ResourceTags: map[string]string{ + "parent1/key1": "value-a", + "parent2/key2": "value2", + }, }, }, { @@ -187,6 +261,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { ForceAttach: true, Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, }, { @@ -197,13 +272,132 @@ func TestExtractAndDefaultParameters(t *testing.T) { ReplicationType: "none", Tags: map[string]string{}, Labels: map[string]string{}, + ResourceTags: map[string]string{}, + }, + }, + { + name: "storage pool parameters", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1,projects/my-project/zones/us-central1-b/storagePools/storagePool-2"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "hyperdisk-balanced", + ReplicationType: "none", + Tags: map[string]string{}, + Labels: map[string]string{}, + ResourceTags: map[string]string{}, + StoragePools: []StoragePool{ + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "my-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/my-project/zones/us-central1-b/storagePools/storagePool-2", + }, + }, }, }, + { + name: "invalid storage pool parameters, starts with /projects instead of projects", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "/projects/my-project/zones/us-central1-a/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "invalid storage pool parameters, missing projects", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "zones/us-central1-a/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "invalid storage pool parameters, missing zones", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "projects/my-project/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "invalid storage pool parameters, duplicate projects", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "projects/my-project/projects/my-project/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "invalid storage pool parameters, duplicate zones", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "zones/us-central1-a/zones/us-central1-a/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "invalid storage pool parameters, duplicate storagePools", + enableStoragePools: true, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "projects/my-project/storagePools/us-central1-a/storagePools/storagePool-1"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "storage pool parameters, enableStoragePools is false", + enableStoragePools: false, + parameters: map[string]string{ParameterKeyType: "hyperdisk-balanced", ParameterKeyStoragePools: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1,projects/my-project/zones/us-central1-b/storagePools/storagePool-2"}, + labels: map[string]string{}, + expectErr: true, + }, + { + name: "multi-zone-enable parameters, multi-zone label is set, multi-zone feature enabled", + parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "true"}, + labels: map[string]string{MultiZoneLabel: "true"}, + enableMultiZone: true, + expectParams: DiskParameters{ + DiskType: "hyperdisk-ml", + ReplicationType: "none", + Tags: map[string]string{}, + Labels: map[string]string{MultiZoneLabel: "true"}, + ResourceTags: map[string]string{}, + MultiZoneProvisioning: true, + }, + }, + { + name: "multi-zone-enable parameters, multi-zone label is false, multi-zone feature enabled", + parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "false"}, + enableMultiZone: true, + expectParams: DiskParameters{ + DiskType: "hyperdisk-ml", + ReplicationType: "none", + Tags: map[string]string{}, + ResourceTags: map[string]string{}, + Labels: map[string]string{}, + }, + }, + { + name: "multi-zone-enable parameters, invalid value, multi-zone feature enabled", + parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "unknown"}, + enableMultiZone: true, + expectErr: true, + }, + { + name: "multi-zone-enable parameters, multi-zone label is set, multi-zone feature disabled", + parameters: map[string]string{ParameterKeyType: "hyperdisk-ml", ParameterKeyEnableMultiZoneProvisioning: "true"}, + expectErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - p, err := ExtractAndDefaultParameters(tc.parameters, "testDriver", tc.labels) + pp := ParameterProcessor{ + DriverName: "testDriver", + EnableStoragePools: tc.enableStoragePools, + EnableMultiZone: tc.enableMultiZone, + } + p, err := pp.ExtractAndDefaultParameters(tc.parameters, tc.labels, tc.extraTags) if gotErr := err != nil; gotErr != tc.expectErr { t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr) } @@ -211,8 +405,8 @@ func TestExtractAndDefaultParameters(t *testing.T) { return } - if !reflect.DeepEqual(p, tc.expectParams) { - t.Errorf("ExtractAndDefaultParameters(%+v) = %v; expected params: %v", tc.parameters, p, tc.expectParams) + if diff := cmp.Diff(tc.expectParams, p); diff != "" { + t.Errorf("ExtractAndDefaultParameters(%+v): -want, +got \n%s", tc.parameters, diff) } }) } @@ -223,6 +417,7 @@ func TestExtractAndDefaultParameters(t *testing.T) { func TestSnapshotParameters(t *testing.T) { tests := []struct { desc string + extraTags map[string]string parameters map[string]string expectedSnapshotParames SnapshotParameters expectError bool @@ -237,6 +432,7 @@ func TestSnapshotParameters(t *testing.T) { ParameterKeyVolumeSnapshotContentName: "snapshot-content-name", ParameterKeyVolumeSnapshotNamespace: "snapshot-namespace", ParameterKeyLabels: "label-1=value-a,key1=value1", + ParameterKeyResourceTags: "parent1/key1/value1,parent2/key2/value2", }, expectedSnapshotParames: SnapshotParameters{ StorageLocations: []string{"asia"}, @@ -248,7 +444,8 @@ func TestSnapshotParameters(t *testing.T) { tagKeyCreatedForSnapshotNamespace: "snapshot-namespace", tagKeyCreatedBy: "test-driver", }, - Labels: map[string]string{"label-1": "value-a", "key1": "value1"}, + Labels: map[string]string{"label-1": "value-a", "key1": "value1"}, + ResourceTags: map[string]string{"parent1/key1": "value1", "parent2/key2": "value2"}, }, expectError: false, }, @@ -260,6 +457,7 @@ func TestSnapshotParameters(t *testing.T) { SnapshotType: DiskSnapshotType, Tags: make(map[string]string), Labels: map[string]string{}, + ResourceTags: map[string]string{}, }, expectError: false, }, @@ -271,7 +469,7 @@ func TestSnapshotParameters(t *testing.T) { } for _, tc := range tests { t.Run(tc.desc, func(t *testing.T) { - p, err := ExtractAndDefaultSnapshotParameters(tc.parameters, "test-driver") + p, err := ExtractAndDefaultSnapshotParameters(tc.parameters, "test-driver", tc.extraTags) if err != nil && !tc.expectError { t.Errorf("Got error %v; expect no error", err) } diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 368a853a..c3b71c1c 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -22,9 +22,13 @@ import ( "fmt" "net/http" "regexp" + "slices" "strings" + "time" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/googleapis/gax-go/v2/apierror" + "golang.org/x/time/rate" "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -70,6 +74,10 @@ const ( // Full or partial URL of the machine type resource, in the format: // zones/zone/machineTypes/machine-type machineTypePattern = "zones/[^/]+/machineTypes/([^/]+)$" + + // Full or partial URL of the zone resource, in the format: + // projects/{project}/zones/{zone} + zoneURIPattern = "projects/[^/]+/zones/([^/]+)$" ) var ( @@ -80,6 +88,10 @@ var ( // zones/zone/machineTypes/machine-type machineTypeRegex = regexp.MustCompile(machineTypePattern) + storagePoolFieldsRegex = regexp.MustCompile(`^projects/([^/]+)/zones/([^/]+)/storagePools/([^/]+)$`) + + zoneURIRegex = regexp.MustCompile(zoneURIPattern) + // userErrorCodeMap tells how API error types are translated to error codes. userErrorCodeMap = map[int]codes.Code{ http.StatusForbidden: codes.PermissionDenied, @@ -87,6 +99,13 @@ var ( http.StatusTooManyRequests: codes.ResourceExhausted, http.StatusNotFound: codes.NotFound, } + + // Regular expressions for validating parent_id, key and value of a resource tag. + regexParent = regexp.MustCompile(`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)`) + regexKey = regexp.MustCompile(`^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$`) + regexValue = regexp.MustCompile(`^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\[\]{}\-\s]{0,61}[a-zA-Z0-9])?$`) + + csiRetryableErrorCodes = []codes.Code{codes.Canceled, codes.DeadlineExceeded, codes.Unavailable, codes.Aborted, codes.ResourceExhausted} ) func BytesToGbRoundDown(bytes int64) int64 { @@ -255,6 +274,86 @@ func ConvertLabelsStringToMap(labels string) (map[string]string, error) { return labelsMap, nil } +// ConvertTagsStringToMap converts the tags from string to Tag slice +// example: "parent_id1/tag_key1/tag_value1,parent_id2/tag_key2/tag_value2" gets +// converted into {"parent_id1/tag_key1":"tag_value1", "parent_id2/tag_key2":"tag_value2"} +// See https://cloud.google.com/resource-manager/docs/tags/tags-overview, +// https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details +func ConvertTagsStringToMap(tags string) (map[string]string, error) { + const tagsDelimiter = "," + const tagsParentIDKeyValueDelimiter = "/" + + tagsMap := make(map[string]string) + if tags == "" { + return nil, nil + } + + checkTagParentIDFn := func(tag, parentID string) error { + if !regexParent.MatchString(parentID) { + return fmt.Errorf("tag parent_id %q for tag %q is invalid. parent_id can have a maximum of 32 characters and cannot be empty. parent_id can be either OrganizationID or ProjectID. OrganizationID must consist of decimal numbers, and cannot have leading zeroes and ProjectID must be 6 to 30 characters in length, can only contain lowercase letters, numbers, and hyphens, and must start with a letter, and cannot end with a hyphen", parentID, tag) + } + return nil + } + + checkTagKeyFn := func(tag, key string) error { + if !regexKey.MatchString(key) { + return fmt.Errorf("tag key %q for tag %q is invalid. Tag key can have a maximum of 63 characters and cannot be empty. Tag key must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `._-`", key, tag) + } + return nil + } + + checkTagValueFn := func(tag, value string) error { + if !regexValue.MatchString(value) { + return fmt.Errorf("tag value %q for tag %q is invalid. Tag value can have a maximum of 63 characters and cannot be empty. Tag value must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `_-.@%%=+:,*#&(){}[]` and spaces", value, tag) + } + + return nil + } + + checkTagParentIDKey := sets.String{} + parentIDkeyValueStrings := strings.Split(tags, tagsDelimiter) + for _, parentIDkeyValueString := range parentIDkeyValueStrings { + parentIDKeyValue := strings.Split(parentIDkeyValueString, tagsParentIDKeyValueDelimiter) + + if len(parentIDKeyValue) != 3 { + return nil, fmt.Errorf("tag %q is invalid, correct format: 'parent_id1/key1/value1,parent_id2/key2/value2'", parentIDkeyValueString) + } + + parentID := strings.TrimSpace(parentIDKeyValue[0]) + if err := checkTagParentIDFn(parentIDkeyValueString, parentID); err != nil { + return nil, err + } + + key := strings.TrimSpace(parentIDKeyValue[1]) + if err := checkTagKeyFn(parentIDkeyValueString, key); err != nil { + return nil, err + } + + value := strings.TrimSpace(parentIDKeyValue[2]) + if err := checkTagValueFn(parentIDkeyValueString, value); err != nil { + return nil, err + } + + parentIDKeyStr := fmt.Sprintf("%s/%s", parentID, key) + if checkTagParentIDKey.Has(parentIDKeyStr) { + return nil, fmt.Errorf("tag parent_id & key combination %q exists more than once", parentIDKeyStr) + } + checkTagParentIDKey.Insert(parentIDKeyStr) + + tagsMap[parentIDKeyStr] = value + } + + // The maximum number of tags allowed per resource is 50. For more details check the following: + // https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing#attaching + // https://cloud.google.com/resource-manager/docs/limits#tag-limits + const maxNumberOfTags = 50 + if len(tagsMap) > maxNumberOfTags { + return nil, fmt.Errorf("more than %d tags is not allowed, given: %d", maxNumberOfTags, len(tagsMap)) + } + + return tagsMap, nil +} + // ProcessStorageLocations trims and normalizes storage location to lower letters. func ProcessStorageLocations(storageLocations string) ([]string, error) { normalizedLoc := strings.ToLower(strings.TrimSpace(storageLocations)) @@ -334,19 +433,19 @@ func CodeForError(sourceError error) codes.Code { if sourceError == nil { return codes.Internal } - + if code, err := isUserMultiAttachError(sourceError); err == nil { + return code + } if code, err := existingErrorCode(sourceError); err == nil { return code } if code, err := isContextError(sourceError); err == nil { return code } - - var apiErr *googleapi.Error - if !errors.As(sourceError, &apiErr) { - return codes.Internal + if code, err := isConnectionResetError(sourceError); err == nil { + return code } - if code, ok := userErrorCodeMap[apiErr.Code]; ok { + if code, err := isGoogleAPIError(sourceError); err == nil { return code } @@ -372,19 +471,120 @@ func isContextError(err error) (codes.Code, error) { return codes.Unknown, fmt.Errorf("Not a context error: %w", err) } +// isConnectionResetError returns the grpc error code Unavailable if the +// passed in error contains the "connection reset by peer" string. +func isConnectionResetError(err error) (codes.Code, error) { + if err == nil { + return codes.Unknown, fmt.Errorf("null error") + } + + errStr := err.Error() + if strings.Contains(errStr, "connection reset by peer") { + return codes.Unavailable, nil + } + return codes.Unknown, fmt.Errorf("Not a connection reset error: %w", err) +} + +// isUserMultiAttachError returns an InvalidArgument if the error is +// multi-attach detected from the API server. If we get this error from the API +// server, it means that the kubelet doesn't know about the multiattch so it is +// due to user configuration. +func isUserMultiAttachError(err error) (codes.Code, error) { + if strings.Contains(err.Error(), "The disk resource") && strings.Contains(err.Error(), "is already being used") { + return codes.InvalidArgument, nil + } + return codes.Unknown, fmt.Errorf("Not a user multiattach error: %w", err) +} + +// existingErrorCode returns the existing gRPC Status error code for the given error, if one exists, +// or an error if one doesn't exist. Since github.com/googleapis/gax-go/v2/apierror now wraps googleapi +// errors (returned from GCE API calls), and sets their status error code to Unknown, we now have to +// make sure we only return existing error codes from errors that are either TemporaryErrors, or errors +// that do not wrap googleAPI errors. Otherwise, we will return Unknown for all GCE API calls that +// return googleapi errors. func existingErrorCode(err error) (codes.Code, error) { if err == nil { return codes.Unknown, fmt.Errorf("null error") } - if status, ok := status.FromError(err); ok { - return status.Code(), nil + var tmpError *TemporaryError + // This explicitly checks our error is a temporary error before extracting its + // status, as there can be other errors that can qualify as statusable + // while not necessarily being temporary. + if errors.As(err, &tmpError) { + if status, ok := status.FromError(err); ok { + return status.Code(), nil + } + } + // We want to make sure we catch other error types that are statusable. + // (eg. grpc-go/internal/status/status.go Error struct that wraps a status) + var googleErr *googleapi.Error + if !errors.As(err, &googleErr) { + if status, ok := status.FromError(err); ok { + return status.Code(), nil + } } + return codes.Unknown, fmt.Errorf("no existing error code for %w", err) } -func LoggedError(msg string, err error) error { +// isGoogleAPIError returns the gRPC status code for the given googleapi error by mapping +// the googleapi error's HTTP code to the corresponding gRPC error code. If the error is +// wrapped in an APIError (github.com/googleapis/gax-go/v2/apierror), it maps the wrapped +// googleAPI error's HTTP code to the corresponding gRPC error code. Returns an error if +// the given error is not a googleapi error. +func isGoogleAPIError(err error) (codes.Code, error) { + var googleErr *googleapi.Error + if !errors.As(err, &googleErr) { + return codes.Unknown, fmt.Errorf("error %w is not a googleapi.Error", err) + } + var sourceCode int + var apiErr *apierror.APIError + if errors.As(err, &apiErr) { + // When googleapi.Err is used as a wrapper, we return the error code of the wrapped contents. + sourceCode = apiErr.HTTPCode() + } else { + // Rely on error code in googleapi.Err when it is our primary error. + sourceCode = googleErr.Code + } + // Map API error code to user error code. + if code, ok := userErrorCodeMap[sourceCode]; ok { + return code, nil + } + + return codes.Unknown, fmt.Errorf("googleapi.Error %w does not map to any known errors", err) +} + +func loggedErrorForCode(msg string, code codes.Code, err error) error { klog.Errorf(msg+"%v", err.Error()) - return status.Errorf(CodeForError(err), msg+"%v", err.Error()) + return status.Errorf(code, msg+"%v", err.Error()) +} + +func LoggedError(msg string, err error) error { + return loggedErrorForCode(msg, CodeForError(err), err) +} + +// NewCombinedError tries to return an appropriate wrapped error that captures +// useful information as an error code +// If there are multiple errors, it extracts the first "retryable" error +// as interpreted by the CSI sidecar. +func NewCombinedError(msg string, errs []error) error { + // If there is only one error, return it as the single error code + if len(errs) == 1 { + LoggedError(msg, errs[0]) + } + + for _, err := range errs { + code := CodeForError(err) + if slices.Contains(csiRetryableErrorCodes, code) { + // Return this as a TemporaryError to lock-in the retryable code + // This will invoke the "existing" error code check in CodeForError + return NewTemporaryError(code, fmt.Errorf("%s: %w", msg, err)) + } + } + + // None of these error codes were retryable. Just return a combined error + // The first matching error (based on our CodeForError) logic will be returned. + return LoggedError(msg, errors.Join(errs...)) } func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool { @@ -392,3 +592,106 @@ func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool { kmsKeyPattern := regexp.MustCompile("projects/[^/]+/locations/([^/]+)/keyRings/[^/]+/cryptoKeys/[^/]+") return kmsKeyPattern.MatchString(DiskEncryptionKmsKey) } + +func ParseZoneFromURI(zoneURI string) (string, error) { + zoneMatch := zoneURIRegex.FindStringSubmatch(zoneURI) + if zoneMatch == nil { + return "", fmt.Errorf("failed to parse zone URI. Expected projects/{project}/zones/{zone}. Got: %s", zoneURI) + } + return zoneMatch[1], nil +} + +// ParseStoragePools returns an error if none of the given storagePools +// (delimited by a comma) are in the format +// projects/project/zones/zone/storagePools/storagePool. +func ParseStoragePools(storagePools string) ([]StoragePool, error) { + spSlice := strings.Split(storagePools, ",") + parsedStoragePools := []StoragePool{} + for _, sp := range spSlice { + project, location, spName, err := fieldsFromStoragePoolResourceName(sp) + if err != nil { + return nil, err + } + spObj := StoragePool{Project: project, Zone: location, Name: spName, ResourceName: sp} + parsedStoragePools = append(parsedStoragePools, spObj) + + } + return parsedStoragePools, nil +} + +// fieldsFromResourceName returns the project, zone, and Storage Pool name from the given +// Storage Pool resource name. The resource name must be in the format +// projects/project/zones/zone/storagePools/storagePool. +// All other formats are invalid, and an error will be returned. +func fieldsFromStoragePoolResourceName(resourceName string) (project, location, spName string, err error) { + fieldMatches := storagePoolFieldsRegex.FindStringSubmatch(resourceName) + // Field matches should have 4 strings: [resourceName, project, zone, storagePool]. The first + // match is the entire string. + if len(fieldMatches) != 4 { + err := fmt.Errorf("invalid Storage Pool resource name. Got %s, expected projects/project/zones/zone/storagePools/storagePool", resourceName) + return "", "", "", err + } + project = fieldMatches[1] + location = fieldMatches[2] + spName = fieldMatches[3] + return +} + +// StoragePoolZones returns the unique zones of the given storage pool resource names. +// Returns an error if multiple storage pools in 1 zone are found. +func StoragePoolZones(storagePools []StoragePool) ([]string, error) { + zonesSet := sets.String{} + var zones []string + for _, sp := range storagePools { + if zonesSet.Has(sp.Zone) { + return nil, fmt.Errorf("found multiple storage pools in zone %s. Only one storage pool per zone is allowed", sp.Zone) + } + zonesSet.Insert(sp.Zone) + zones = append(zones, sp.Zone) + } + return zones, nil +} + +func StoragePoolInZone(storagePools []StoragePool, zone string) *StoragePool { + for _, pool := range storagePools { + if zone == pool.Zone { + return &pool + } + } + return nil +} + +func UnorderedSlicesEqual(slice1 []string, slice2 []string) bool { + set1 := sets.NewString(slice1...) + set2 := sets.NewString(slice2...) + spZonesNotInReq := set1.Difference(set2) + if spZonesNotInReq.Len() != 0 { + return false + } + return true +} + +func VolumeIdAsMultiZone(volumeId string) (string, error) { + splitId := strings.Split(volumeId, "/") + if len(splitId) != volIDTotalElements { + return "", fmt.Errorf("failed to get id components. Expected projects/{project}/zones/{zone}/disks/{name}. Got: %s", volumeId) + } + if splitId[volIDToplogyKey] != "zones" { + return "", fmt.Errorf("expected id to be zonal. Got: %s", volumeId) + } + splitId[volIDToplogyValue] = MultiZoneValue + return strings.Join(splitId, "/"), nil +} + +// NewLimiter returns a token bucket based request rate limiter after initializing +// the passed values for limit, burst (or token bucket) size. If opted for emptyBucket +// all initial tokens are reserved for the first burst. +func NewLimiter(limit, burst int, emptyBucket bool) *rate.Limiter { + limiter := rate.NewLimiter(rate.Every(time.Second/time.Duration(limit)), burst) + + if emptyBucket { + limiter.AllowN(time.Now(), burst) + } + + return limiter +} diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index 3f5993a0..1b9b44e3 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -22,9 +22,12 @@ import ( "fmt" "net/http" "reflect" + "syscall" "testing" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/google/go-cmp/cmp" + "github.com/googleapis/gax-go/v2/apierror" "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -541,6 +544,235 @@ func TestConvertLabelsStringToMap(t *testing.T) { } +func TestConvertTagsStringToMap(t *testing.T) { + t.Run("parsing tags string into slice", func(t *testing.T) { + testCases := []struct { + name string + tags string + expectedOutput map[string]string + expectedError bool + }{ + { + name: "should return empty slice when tags string is empty", + tags: "", + expectedOutput: nil, + expectedError: false, + }, + { + name: "single tag string", + tags: "parent/key/value", + expectedOutput: map[string]string{"parent/key": "value"}, + expectedError: false, + }, + { + name: "multiple tag string", + tags: "parent1/key1/value1,parent2/key2/value2", + expectedOutput: map[string]string{"parent1/key1": "value1", "parent2/key2": "value2"}, + expectedError: false, + }, + { + name: "multiple tags string with whitespaces gets trimmed", + tags: "parent1/key1/value1, parent2/key2/value2", + expectedOutput: map[string]string{"parent1/key1": "value1", "parent2/key2": "value2"}, + expectedError: false, + }, + { + name: "malformed tags string (no parent_ids, keys and values)", + tags: ",,", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed tags string (incorrect format)", + tags: "foo,bar", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed tags string (missing parent_id)", + tags: "parent1/key1/value1,/key2/value2", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed tags string (missing key)", + tags: "parent1//value1,parent2/key2/value2", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed tags string (missing value)", + tags: "parent1/key1/value1,parent2/key2/", + expectedOutput: nil, + expectedError: true, + }, + { + name: "same tag parent_id, key and value string used more than once", + tags: "parent1/key1/value1,parent1/key1/value1", + expectedOutput: nil, + expectedError: true, + }, + { + name: "same tag parent_id & key string used more than once", + tags: "parent1/key1/value1,parent1/key1/value2", + expectedOutput: nil, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + output, err := ConvertTagsStringToMap(tc.tags) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + + if !tc.expectedError && err != nil { + t.Errorf("Did not expect error but got: %v", err) + } + + if err == nil && !reflect.DeepEqual(output, tc.expectedOutput) { + t.Errorf("Got tags %v, but expected %v", output, tc.expectedOutput) + } + } + }) + + t.Run("checking google requirements", func(t *testing.T) { + testCases := []struct { + name string + tags string + expectedError bool + }{ + { + name: "50 tags at most", + tags: `p1/k/v,p2/k/v,p3/k/v,p4/k/v,p5/k/v,p6/k/v,p7/k/v,p8/k/v,p9/k/v,p10/k/v,p11/k/v,p12/k/v,p13/k/v,p14/k/v,p15/k/v,p16/k/v,p17/k/v, + p18/k/v,p19/k/v,p20/k/v,p21/k/v,p22/k/v,p23/k/v,p24/k/v,p25/k/v,p26/k/v,p27/k/v,p28/k/v,p29/k/v,p30/k/v,p31/k/v,p32/k/v,p33/k/v, + p34/k/v,p35/k/v,p36/k/v,p37/k/v,p38/k/v,p39/k/v,p40/k/v,p41/k/v,p42/k/v,p43/k/v,p44/k/v,p45/k/v,p46/k/v,p47/k/v,p48/k/v,p49/k/v, + p50/k/v,p51/k/v`, + expectedError: true, + }, + { + name: "tag parent_id must start with non-zero decimal when OrganizationID is used (leading zeroes case)", + tags: "01/k/v", + expectedError: true, + }, + { + name: "tag parent_id may not have more than 32 characters when OrganizationID is used", + tags: "123546789012345678901234567890123/k/v", + expectedError: true, + }, + { + name: "tag parent_id can have decimal characters when OrganizationID is used", + tags: "1234567890/k/v", + expectedError: false, + }, + { + name: "tag parent_id may not have less than 6 characters when ProjectID is used", + tags: "abcde/k/v", + expectedError: true, + }, + { + name: "tag parent_id must start with lowercase char when ProjectID is used (decimal case)", + tags: "1parent/k/v", + expectedError: true, + }, + { + name: "tag parent_id must start with lowercase char when ProjectID is used (- case)", + tags: "-parent/k/v", + expectedError: true, + }, + { + name: "tag parent_id must end with lowercase alphanumeric char when ProjectID is used (- case)", + tags: "parent-/k/v", + expectedError: true, + }, + { + name: "tag parent_id may not have more than 30 characters when ProjectID is used", + tags: "abcdefghijklmnopqrstuvwxyz12345/k/v", + expectedError: true, + }, + { + name: "tag parent_id can contain lowercase alphanumeric characters and hyphens when ProjectID is used", + tags: "parent-id-100/k/v", + expectedError: false, + }, + { + name: "tag key must start with alphanumeric char (. case)", + tags: "parent/.k/v", + expectedError: true, + }, + { + name: "tag key must start with alphanumeric char (_ case)", + tags: "parent/_k/v", + expectedError: true, + }, + { + name: "tag key must start with alphanumeric char (- case)", + tags: "parent/-k/v", + expectedError: true, + }, + { + name: "tag key can only contain uppercase, lowercase alphanumeric characters, and the following special characters '._-'", + tags: "parent/k*/v", + expectedError: true, + }, + { + name: "tag key may not have over 63 characters", + tags: "parent/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234/v", + expectedError: true, + }, + { + name: "tag key can contain uppercase, lowercase alphanumeric characters, and the following special characters '._-'", + tags: "parent/Type_of.cloud-platform/v", + expectedError: false, + }, + { + name: "tag value must start with alphanumeric char (. case)", + tags: "parent/k/.v", + expectedError: true, + }, + { + name: "tag value must start with alphanumeric char (_ case)", + tags: "parent/k/_v", + expectedError: true, + }, + { + name: "tag value must start with alphanumeric char (- case)", + tags: "parent/k/-v", + expectedError: true, + }, + { + name: "tag value can only contain uppercase, lowercase alphanumeric characters, and the following special characters `_-.@%%=+:,*#&(){}[]` and spaces", + tags: "parent/k/v*", + expectedError: true, + }, + { + name: "tag value may not have over 63 characters", + tags: "parent/k/abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234", + expectedError: true, + }, + { + name: "tag key can contain uppercase, lowercase alphanumeric characters, and the following special characters `_-.@%%=+:,*#&(){}[]` and spaces", + tags: "parent/k/Special@value[10]{20}(30)-example", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + _, err := ConvertTagsStringToMap(tc.tags) + + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + + if !tc.expectedError && err != nil { + t.Errorf("Did not expect error but got: %v", err) + } + } + }) +} + func TestSnapshotStorageLocations(t *testing.T) { tests := []struct { desc string @@ -975,6 +1207,17 @@ func TestParseMachineType(t *testing.T) { } func TestCodeForError(t *testing.T) { + getGoogleAPIWrappedError := func(err error) *googleapi.Error { + apierr, _ := apierror.ParseError(err, false) + wrappedError := &googleapi.Error{} + wrappedError.Wrap(apierr) + + return wrappedError + } + getAPIError := func(err error) *apierror.APIError { + apierror, _ := apierror.ParseError(err, true) + return apierror + } testCases := []struct { name string inputErr error @@ -990,6 +1233,36 @@ func TestCodeForError(t *testing.T) { inputErr: &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"}, expCode: codes.InvalidArgument, }, + { + name: "googleapi.Error that wraps apierror.APIError of http kind", + inputErr: getGoogleAPIWrappedError(&googleapi.Error{ + Code: 404, + Message: "data requested not found error", + }), + expCode: codes.NotFound, + }, + { + name: "googleapi.Error that wraps apierror.APIError of status kind", + inputErr: getGoogleAPIWrappedError(status.New( + codes.Internal, "Internal status error", + ).Err()), + expCode: codes.Internal, + }, + { + name: "apierror.APIError of http kind", + inputErr: getAPIError(&googleapi.Error{ + Code: 404, + Message: "data requested not found error", + }), + expCode: codes.NotFound, + }, + { + name: "apierror.APIError of status kind", + inputErr: getAPIError(status.New( + codes.Canceled, "Internal status error", + ).Err()), + expCode: codes.Canceled, + }, { name: "googleapi.Error but not a user error", inputErr: &googleapi.Error{Code: http.StatusInternalServerError, Message: "Internal error"}, @@ -1005,6 +1278,16 @@ func TestCodeForError(t *testing.T) { inputErr: context.DeadlineExceeded, expCode: codes.DeadlineExceeded, }, + { + name: "connection reset error", + inputErr: fmt.Errorf("failed to getDisk: connection reset by peer"), + expCode: codes.Unavailable, + }, + { + name: "wrapped connection reset error", + inputErr: fmt.Errorf("received error: %v", syscall.ECONNRESET), + expCode: codes.Unavailable, + }, { name: "status error with Aborted error code", inputErr: status.Error(codes.Aborted, "aborted error"), @@ -1015,6 +1298,31 @@ func TestCodeForError(t *testing.T) { inputErr: nil, expCode: codes.Internal, }, + { + name: "user multiattach error", + inputErr: fmt.Errorf("The disk resource 'projects/foo/disk/bar' is already being used by 'projects/foo/instances/1'"), + expCode: codes.InvalidArgument, + }, + { + name: "TemporaryError that wraps googleapi error", + inputErr: &TemporaryError{code: codes.Unavailable, err: &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"}}, + expCode: codes.Unavailable, + }, + { + name: "TemporaryError that wraps fmt.Errorf, which wraps googleapi error", + inputErr: &TemporaryError{code: codes.Aborted, err: fmt.Errorf("got error: %w", &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"})}, + expCode: codes.Aborted, + }, + { + name: "TemporaryError that wraps status error", + inputErr: &TemporaryError{code: codes.Aborted, err: status.Error(codes.Aborted, "aborted error")}, + expCode: codes.Aborted, + }, + { + name: "TemporaryError that wraps context canceled error", + inputErr: &TemporaryError{code: codes.Aborted, err: context.Canceled}, + expCode: codes.Aborted, + }, } for _, tc := range testCases { @@ -1076,6 +1384,34 @@ func TestIsContextError(t *testing.T) { } } +func TestIsUserMultiAttachError(t *testing.T) { + cases := []struct { + errorString string + expectedCode codes.Code + expectCode bool + }{ + { + errorString: "The disk resource 'projects/foo/disk/bar' is already being used by 'projects/foo/instance/biz'", + expectedCode: codes.InvalidArgument, + expectCode: true, + }, + { + errorString: "The disk resource is ok!", + expectCode: false, + }, + } + for _, test := range cases { + code, err := isUserMultiAttachError(fmt.Errorf(test.errorString)) + if test.expectCode { + if err != nil || code != test.expectedCode { + t.Errorf("Failed with non-nil error %v or bad code %v: %s", err, code, test.errorString) + } + } else if err == nil { + t.Errorf("Expected error for test but got none: %s", test.errorString) + } + } +} + func TestIsValidDiskEncryptionKmsKey(t *testing.T) { cases := []struct { diskEncryptionKmsKey string @@ -1101,3 +1437,300 @@ func TestIsValidDiskEncryptionKmsKey(t *testing.T) { } } } + +func TestFieldsFromResourceName(t *testing.T) { + testcases := []struct { + name string + resourceName string + expectedProject string + expectedZone string + expectedName string + expectedErr bool + }{ + { + name: "StoragePool_WithValidResourceName_ReturnsFields", + resourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + expectedProject: "my-project", + expectedZone: "us-central1-a", + expectedName: "storagePool-1", + }, + { + name: "StoragePool_WithFullResourceURL_ReturnsError", + resourceName: "https://www.googleapis.com/compute/v1/projects/project/zones/zone/storagePools/storagePool", + expectedErr: true, + }, + { + name: "StoragePool_WithMissingProject_ReturnsError", + resourceName: "zones/us-central1-a/storagePools/storagePool-1", + expectedErr: true, + }, + { + name: "StoragePool_WithMissingZone_ReturnsError", + resourceName: "projects/my-project/storagePools/storagePool-1", + expectedErr: true, + }, + { + name: "StoragePool_WithMissingStoragePoolName_ReturnsError", + resourceName: "projects/my-project/zones/us-central1-a/storagePool-1", + expectedErr: true, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + project, zone, name, err := fieldsFromStoragePoolResourceName(tc.resourceName) + input := fmt.Sprintf("fieldsFromStoragePoolResourceName(%q)", tc.resourceName) + gotErr := err != nil + if gotErr != tc.expectedErr { + t.Errorf("%s error presence = %v, expected error presence = %v", input, gotErr, tc.expectedErr) + } + if project != tc.expectedProject || zone != tc.expectedZone || name != tc.expectedName { + t.Errorf("%s returned {project: %q, zone: %q, name: %q}, expected {project: %q, zone: %q, name: %q}", input, project, zone, name, tc.expectedProject, tc.expectedZone, tc.expectedName) + } + }) + } +} + +func TestZones(t *testing.T) { + testcases := []struct { + name string + storagePools []StoragePool + expectedZones []string + expectedErr bool + }{ + { + name: "StoragePools_WithValidResourceNames_ReturnsZones", + storagePools: []StoragePool{ + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "my-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/my-project/zones/us-central1-b/storagePools/storagePool-2", + }, + }, + expectedZones: []string{"us-central1-a", "us-central1-b"}, + }, + { + name: "StoragePools_WithDuplicateZone_ReturnsError", + storagePools: []StoragePool{ + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-2", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-2", + }, + }, + expectedErr: true, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + zones, err := StoragePoolZones(tc.storagePools) + input := fmt.Sprintf("StoragePoolZones(%q)", tc.storagePools) + gotErr := err != nil + if gotErr != tc.expectedErr { + t.Errorf("%s error presence = %v, expected error presence = %v", input, gotErr, tc.expectedErr) + } + if diff := cmp.Diff(tc.expectedZones, zones); diff != "" { + t.Errorf("%s: -want err, +got err\n%s", input, diff) + } + }) + } +} + +func TestStoragePoolInZone(t *testing.T) { + testcases := []struct { + name string + storagePools []StoragePool + zone string + expectedStoragePool *StoragePool + expectedErr bool + }{ + { + name: "ValidStoragePools_ReturnsStoragePoolInZone", + storagePools: []StoragePool{ + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "my-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/my-project/zones/us-central1-b/storagePools/storagePool-2", + }, + }, + zone: "us-central1-a", + expectedStoragePool: &StoragePool{ + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + { + name: "StoragePoolNotInZone_ReturnsNil", + storagePools: []StoragePool{ + { + Project: "my-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/my-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + zone: "us-central1-b", + expectedStoragePool: nil, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + sp := StoragePoolInZone(tc.storagePools, tc.zone) + input := fmt.Sprintf("StoragePoolInZone(%q)", tc.storagePools) + if diff := cmp.Diff(tc.expectedStoragePool, sp); diff != "" { + t.Errorf("%s: -want, +got \n%s", input, diff) + } + }) + } +} + +func TestUnorderedSlicesEqual(t *testing.T) { + testcases := []struct { + name string + slice1 []string + slice2 []string + expectedSlicesEqual bool + }{ + { + name: "OrderedSlicesEqual_ReturnsTrue", + slice1: []string{"us-central1-a", "us-central1-b"}, + slice2: []string{"us-central1-a", "us-central1-b"}, + expectedSlicesEqual: true, + }, + { + name: "UnorderedSlicesEqual_ReturnsTrue", + slice1: []string{"us-central1-a", "us-central1-b"}, + slice2: []string{"us-central1-b", "us-central1-a"}, + expectedSlicesEqual: true, + }, + { + name: "SlicesNotEqualSameLength_ReturnsFalse", + slice1: []string{"us-central1-a", "us-central1-b"}, + slice2: []string{"us-central1-a", "us-central1-a"}, + expectedSlicesEqual: false, + }, + { + name: "SlicesNotEqualDifferentLength_ReturnsFalse", + slice1: []string{"us-central1-a"}, + slice2: []string{}, + expectedSlicesEqual: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + slicesEqual := UnorderedSlicesEqual(tc.slice1, tc.slice2) + input := fmt.Sprintf("UnorderedSlicesEqual(%v, %v)", tc.slice1, tc.slice2) + if diff := cmp.Diff(tc.expectedSlicesEqual, slicesEqual); diff != "" { + t.Errorf("%s: -want, +got \n%s", input, diff) + } + }) + } +} + +func TestParseZoneFromURI(t *testing.T) { + testcases := []struct { + name string + zoneURI string + wantZone string + expectErr bool + }{ + { + name: "ParseZoneFromURI_FullURI", + zoneURI: "https://www.googleapis.com/compute/v1/projects/psch-gke-dev/zones/us-east4-a", + wantZone: "us-east4-a", + }, + { + name: "ParseZoneFromURI_ProjectZoneString", + zoneURI: "projects/psch-gke-dev/zones/us-east4-a", + wantZone: "us-east4-a", + }, + { + name: "ParseZoneFromURI_Malformed", + zoneURI: "projects/psch-gke-dev/regions/us-east4", + expectErr: true, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + gotZone, err := ParseZoneFromURI(tc.zoneURI) + if err != nil && !tc.expectErr { + t.Fatalf("Unexpected error: %v", err) + } + if err == nil && tc.expectErr { + t.Fatalf("Expected err, but none was returned. Zone result: %v", gotZone) + } + if gotZone != tc.wantZone { + t.Errorf("ParseZoneFromURI(%v): got %v, want %v", tc.zoneURI, gotZone, tc.wantZone) + } + }) + } +} + +func TestNewCombinedError(t *testing.T) { + testcases := []struct { + name string + errors []error + wantCode codes.Code + }{ + { + name: "single generic error", + errors: []error{fmt.Errorf("my internal error")}, + wantCode: codes.Internal, + }, + { + name: "single retryable error", + errors: []error{&googleapi.Error{Code: http.StatusTooManyRequests, Message: "Resource Exhausted"}}, + wantCode: codes.ResourceExhausted, + }, + { + name: "multi generic error", + errors: []error{fmt.Errorf("my internal error"), fmt.Errorf("my other internal error")}, + wantCode: codes.Internal, + }, + { + name: "multi retryable error", + errors: []error{fmt.Errorf("my internal error"), &googleapi.Error{Code: http.StatusTooManyRequests, Message: "Resource Exhausted"}}, + wantCode: codes.ResourceExhausted, + }, + { + name: "multi retryable error", + errors: []error{fmt.Errorf("my internal error"), &googleapi.Error{Code: http.StatusGatewayTimeout, Message: "connection reset by peer"}, fmt.Errorf("my other internal error")}, + wantCode: codes.Unavailable, + }, + { + name: "multi retryable error", + errors: []error{fmt.Errorf("The disk resource is already being used"), &googleapi.Error{Code: http.StatusGatewayTimeout, Message: "connection reset by peer"}}, + wantCode: codes.Unavailable, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + gotCode := CodeForError(NewCombinedError("message", tc.errors)) + if gotCode != tc.wantCode { + t.Errorf("NewCombinedError(%v): got %v, want %v", tc.errors, gotCode, tc.wantCode) + } + }) + } +} diff --git a/pkg/deviceutils/device-utils.go b/pkg/deviceutils/device-utils.go index 0835f0ba..c9ce8988 100644 --- a/pkg/deviceutils/device-utils.go +++ b/pkg/deviceutils/device-utils.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "k8s.io/mount-utils" pathutils "k8s.io/utils/path" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" ) @@ -36,9 +37,10 @@ const ( diskScsiGooglePrefix = "scsi-0Google_PersistentDisk_" diskPartitionSuffix = "-part" diskSDPath = "/dev/sd" - diskSDPattern = "/dev/sd*" + diskSDGlob = "/dev/sd*" diskNvmePath = "/dev/nvme" - diskNvmePattern = "/dev/nvme*" + diskNvmePattern = "^/dev/nvme[0-9]+n[0-9]+$" + diskNvmeGlob = "/dev/nvme*" // How many times to retry for a consistent read of /proc/mounts. maxListTries = 3 // Number of fields per line in /proc/mounts as per the fstab man page. @@ -69,6 +71,8 @@ var ( scsiRegex = regexp.MustCompile(scsiPattern) // regex to parse google_nvme_id output and extract the serial nvmeRegex = regexp.MustCompile(nvmePattern) + // regex to filter for disk drive paths from filepath.Glob output of diskNvmeGlob + diskNvmeRegex = regexp.MustCompile(diskNvmePattern) ) // DeviceUtils are a collection of methods that act on the devices attached @@ -82,12 +86,15 @@ type DeviceUtils interface { // exists on the machine, or an empty string if none exists VerifyDevicePath(devicePaths []string, deviceName string) (string, error) - // DisableDevice performs necessary disabling prior to a device being - // detached from a node. The path is that from GetDiskByIdPaths. - DisableDevice(devicePath string) error - // Resize returns whether or not a device needs resizing. Resize(resizer resizefs.Resizefs, devicePath string, deviceMountPath string) (bool, error) + + // IsDeviceFilesystemInUse returns if a device path with the specified fstype + // TODO: Mounter is passed in in order to call GetDiskFormat() + // This is currently only implemented in mounter_linux, not mounter_windows. + // Refactor this interface and function call up the stack to the caller once it is + // implemented in mounter_windows. + IsDeviceFilesystemInUse(mounter *mount.SafeFormatAndMount, devicePath, devFsPath string) (bool, error) } type deviceUtils struct { @@ -306,15 +313,28 @@ func getDevFsSerial(devFsPath string) (string, error) { } } +func filterAvailableNvmeDevFsPaths(devNvmePaths []string) []string { + // Devices under /dev/nvme need to be filtered for disk drive paths only. + diskNvmePaths := []string{} + for _, devNvmePath := range devNvmePaths { + if diskNvmeRegex.MatchString(devNvmePath) { + diskNvmePaths = append(diskNvmePaths, devNvmePath) + } + } + return diskNvmePaths +} + func findAvailableDevFsPaths() ([]string, error) { - diskSDPaths, err := filepath.Glob(diskSDPattern) + diskSDPaths, err := filepath.Glob(diskSDGlob) if err != nil { - return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskSDPattern, err) + return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskSDGlob, err) } - diskNvmePaths, err := filepath.Glob(diskNvmePattern) + devNvmePaths, err := filepath.Glob(diskNvmeGlob) if err != nil { - return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskNvmePattern, err) + return nil, fmt.Errorf("failed to filepath.Glob(\"%s\"): %w", diskNvmeGlob, err) } + // Devices under /dev/nvme need to be filtered for disk drive paths only. + diskNvmePaths := filterAvailableNvmeDevFsPaths(devNvmePaths) return append(diskSDPaths, diskNvmePaths...), nil } diff --git a/pkg/deviceutils/device-utils_linux.go b/pkg/deviceutils/device-utils_linux.go index 3e1cda5c..5878a28f 100644 --- a/pkg/deviceutils/device-utils_linux.go +++ b/pkg/deviceutils/device-utils_linux.go @@ -20,9 +20,26 @@ import ( "fmt" "os" "path/filepath" + + "k8s.io/mount-utils" ) -func (_ *deviceUtils) DisableDevice(devicePath string) error { - deviceName := filepath.Base(devicePath) - return os.WriteFile(fmt.Sprintf("/sys/block/%s/device/state", deviceName), []byte("offline"), 0644) +func (_ *deviceUtils) IsDeviceFilesystemInUse(mounter *mount.SafeFormatAndMount, devicePath, devFsPath string) (bool, error) { + fstype, err := mounter.GetDiskFormat(devicePath) + if err != nil { + return false, fmt.Errorf("failed to get disk format for %s (aka %s): %v", devicePath, devFsPath, err) + } + + devFsName := filepath.Base(devFsPath) + sysFsTypePath := fmt.Sprintf("/sys/fs/%s/%s", fstype, devFsName) + stat, err := os.Stat(sysFsTypePath) + if err != nil { + if os.IsNotExist(err) { + // Path doesn't exist, indicating the device is NOT in use + return false, nil + } + return false, err + } + + return stat.IsDir(), nil } diff --git a/pkg/deviceutils/device-utils_test.go b/pkg/deviceutils/device-utils_test.go index 630ab4c7..1026e3c1 100644 --- a/pkg/deviceutils/device-utils_test.go +++ b/pkg/deviceutils/device-utils_test.go @@ -1,7 +1,12 @@ package deviceutils import ( + "fmt" + "path/filepath" "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" ) func TestParseNvmeSerial(t *testing.T) { @@ -45,3 +50,66 @@ func TestParseNvmeSerial(t *testing.T) { } } } + +func less(a, b fmt.Stringer) bool { + return a.String() < b.String() +} + +// Test that the NVMe Regex matches expected paths +// Note that this only tests the regex, not the actual path finding codepath. +// The real codepath uses filepath.Glob(), which doesn't have an easy way to mock out local +// directory paths (eg: using a subdirectory prefix). +// We could use a recursive child process, setting a root UID, and use Chroot (which requires superuser). +// This is done in upstream golang tests, but this adds additional complexity +// and may prevent our tests from running on all platforms. See the following test for an example: +// https://github.com/golang/go/blob/d33548d178016122726342911f8e15016a691472/src/syscall/exec_linux_test.go#L250 +func TestDiskNvmePattern(t *testing.T) { + testCases := []struct { + paths []string + wantPaths []string + }{ + { + paths: []string{ + "/dev/nvme0n1p15", + "/dev/nvme0n1p14", + "/dev/nvme0n1p1", + "/dev/nvme0n1", + "/dev/nvme0n2", + "/dev/nvme0", + }, + wantPaths: []string{ + "/dev/nvme0n1", + "/dev/nvme0n2", + }, + }, + { + paths: []string{ + "/dev/nvme1", + "/dev/nvme0n1p15", + "/dev/nvme0n1p14", + "/dev/nvme0n1p1", + "/dev/nvme2", + }, + wantPaths: []string{}, + }, + } + + for _, tc := range testCases { + devPaths := []string{} + + for _, path := range tc.paths { + ok, err := filepath.Match(diskNvmeGlob, path) + if err != nil { + t.Errorf("Error encountered matching path: %v", path) + } + + if ok { + devPaths = append(devPaths, path) + } + } + gotPaths := filterAvailableNvmeDevFsPaths(devPaths) + if diff := cmp.Diff(gotPaths, tc.wantPaths, cmpopts.SortSlices(less)); diff != "" { + t.Errorf("Unexpected NVMe device paths (-got, +want):\n%s", diff) + } + } +} diff --git a/pkg/deviceutils/device-utils_windows.go b/pkg/deviceutils/device-utils_windows.go index 8b2055a1..b147217b 100644 --- a/pkg/deviceutils/device-utils_windows.go +++ b/pkg/deviceutils/device-utils_windows.go @@ -16,7 +16,15 @@ limitations under the License. package deviceutils +import "k8s.io/mount-utils" + func (_ *deviceUtils) DisableDevice(devicePath string) error { // No disabling is necessary on windows. return nil } + +func (_ *deviceUtils) IsDeviceFilesystemInUse(mounter *mount.SafeFormatAndMount, devicePath, devFsPath string) (bool, error) { + // We don't support checking if a device filesystem is captured elsewhere by the system + // Return false, to skip this check. Assume the filesystem is not in use. + return false, nil +} diff --git a/pkg/deviceutils/fake-device-utils.go b/pkg/deviceutils/fake-device-utils.go index d17c43e5..1b0c641e 100644 --- a/pkg/deviceutils/fake-device-utils.go +++ b/pkg/deviceutils/fake-device-utils.go @@ -14,7 +14,10 @@ limitations under the License. package deviceutils -import "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" +import ( + "k8s.io/mount-utils" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs" +) type fakeDeviceUtils struct { skipResize bool @@ -50,3 +53,9 @@ func (du *fakeDeviceUtils) Resize(resizer resizefs.Resizefs, devicePath string, } return resizer.Resize(devicePath, deviceMountPath) } + +func (_ *fakeDeviceUtils) IsDeviceFilesystemInUse(mounter *mount.SafeFormatAndMount, devicePath, devFsPath string) (bool, error) { + // We don't support checking if a device filesystem is captured elsewhere by the system + // Return false, to skip this check. + return false, nil +} diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go index 1d59cdd0..02b70d5d 100644 --- a/pkg/gce-cloud-provider/compute/cloud-disk.go +++ b/pkg/gce-cloud-provider/compute/cloud-disk.go @@ -227,10 +227,43 @@ func (d *CloudDisk) GetMultiWriter() bool { func (d *CloudDisk) GetEnableConfidentialCompute() bool { switch { case d.disk != nil: - return false + return d.disk.EnableConfidentialCompute case d.betaDisk != nil: return d.betaDisk.EnableConfidentialCompute default: return false } } + +func (d *CloudDisk) GetEnableStoragePools() bool { + switch { + case d.disk != nil: + return d.disk.StoragePool != "" + case d.betaDisk != nil: + return d.betaDisk.StoragePool != "" + default: + return false + } +} + +func (d *CloudDisk) GetLabels() map[string]string { + switch { + case d.disk != nil: + return d.disk.Labels + case d.betaDisk != nil: + return d.betaDisk.Labels + default: + return nil + } +} + +func (d *CloudDisk) GetAccessMode() string { + switch { + case d.disk != nil: + return d.disk.AccessMode + case d.betaDisk != nil: + return d.betaDisk.AccessMode + default: + return "" + } +} diff --git a/pkg/gce-cloud-provider/compute/cloud-disk_test.go b/pkg/gce-cloud-provider/compute/cloud-disk_test.go index a37584ea..e2bea29d 100644 --- a/pkg/gce-cloud-provider/compute/cloud-disk_test.go +++ b/pkg/gce-cloud-provider/compute/cloud-disk_test.go @@ -20,6 +20,7 @@ package gcecloudprovider import ( "testing" + "github.com/google/go-cmp/cmp" computebeta "google.golang.org/api/compute/v0.beta" computev1 "google.golang.org/api/compute/v1" ) @@ -55,12 +56,12 @@ func TestGetEnableConfidentialCompute(t *testing.T) { expectedEnableConfidentialCompute: true, }, { - name: "test disk withpit enableConfidentialCompute", + name: "test disk without enableConfidentialCompute", diskVersion: CreateDiskWithConfidentialCompute(false, false, "hyperdisk-balanced"), expectedEnableConfidentialCompute: false, }, { - name: "test disk withpit enableConfidentialCompute", + name: "test disk without enableConfidentialCompute", diskVersion: CreateDiskWithConfidentialCompute(false, false, "pd-standard"), expectedEnableConfidentialCompute: false, }, @@ -77,3 +78,120 @@ func TestGetEnableConfidentialCompute(t *testing.T) { } } } + +func TestGetEnableStoragePools(t *testing.T) { + testCases := []struct { + name string + cloudDisk *CloudDisk + expectedEnableStoragePools bool + }{ + { + name: "v1 disk returns false", + cloudDisk: &CloudDisk{ + disk: &computev1.Disk{}, + }, + expectedEnableStoragePools: false, + }, + { + name: "beta disk returns false", + cloudDisk: &CloudDisk{ + betaDisk: &computebeta.Disk{}, + }, + expectedEnableStoragePools: false, + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + input := "GetEnableStoragePools()" + enableStoragePools := tc.cloudDisk.GetEnableStoragePools() + if enableStoragePools != tc.expectedEnableStoragePools { + t.Fatalf("%s got confidentialCompute value %t expected %t", input, enableStoragePools, tc.expectedEnableStoragePools) + } + if enableStoragePools != tc.expectedEnableStoragePools { + t.Fatalf("%s got confidentialCompute value %t expected %t", input, enableStoragePools, tc.expectedEnableStoragePools) + } + } +} + +func TestGetLabels(t *testing.T) { + testCases := []struct { + name string + cloudDisk *CloudDisk + wantLabels map[string]string + }{ + { + name: "v1 disk labels", + cloudDisk: &CloudDisk{ + disk: &computev1.Disk{ + Labels: map[string]string{"foo": "v1", "goog-gke-multi-zone": "true"}, + }, + }, + wantLabels: map[string]string{"foo": "v1", "goog-gke-multi-zone": "true"}, + }, + { + name: "beta disk labels", + cloudDisk: &CloudDisk{ + betaDisk: &computebeta.Disk{ + Labels: map[string]string{"bar": "beta", "goog-gke-multi-zone": "true"}, + }, + }, + wantLabels: map[string]string{"bar": "beta", "goog-gke-multi-zone": "true"}, + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + gotLabels := tc.cloudDisk.GetLabels() + if diff := cmp.Diff(tc.wantLabels, gotLabels); diff != "" { + t.Errorf("GetLabels() returned unexpected difference (-want +got):\n%s", diff) + } + } +} + +func TestGetAccessMode(t *testing.T) { + testCases := []struct { + name string + cloudDisk *CloudDisk + wantAccessMode string + }{ + { + name: "v1 disk accessMode", + cloudDisk: &CloudDisk{ + disk: &computev1.Disk{ + AccessMode: "READ_WRITE_SINGLE", + }, + }, + wantAccessMode: "READ_WRITE_SINGLE", + }, + { + name: "beta disk accessMode", + cloudDisk: &CloudDisk{ + betaDisk: &computebeta.Disk{ + AccessMode: "READ_ONLY_MANY", + }, + }, + wantAccessMode: "READ_ONLY_MANY", + }, + { + name: "unset disk accessMode", + cloudDisk: &CloudDisk{ + betaDisk: &computebeta.Disk{}, + }, + wantAccessMode: "", + }, + { + name: "unset disk", + cloudDisk: &CloudDisk{}, + wantAccessMode: "", + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + gotAccessMode := tc.cloudDisk.GetAccessMode() + if gotAccessMode != tc.wantAccessMode { + t.Errorf("GetAccessMode() got %v, want %v", gotAccessMode, tc.wantAccessMode) + } + } +} diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index 14fbaf44..f64408f0 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -35,6 +35,8 @@ package gcecloudprovider import ( "context" "fmt" + "net/http" + "regexp" "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" @@ -57,6 +59,15 @@ const ( imageURITemplateGlobal = "projects/%s/global/images/%s" //{gce.projectID}/global/images/{image.Name}" ) +var ( + // Snaphsot and Image Regex must comply with RFC1035 + rfc1035Regex = regexp.MustCompile("^[a-z]([-a-z0-9]*[a-z0-9])?$") +) + +func isRFC1035(value string) bool { + return rfc1035Regex.MatchString(strings.ToLower(value)) +} + type FakeCloudProvider struct { project string zone string @@ -86,7 +97,11 @@ func CreateFakeCloudProvider(project, zone string, cloudDisks []*CloudDisk) (*Fa mockDiskStatus: "READY", } for _, d := range cloudDisks { - fcp.disks[d.GetName()] = d + diskZone := d.GetZone() + if diskZone == "" { + diskZone = zone + } + fcp.disks[meta.ZonalKey(d.GetName(), diskZone).String()] = d } return fcp, nil } @@ -108,8 +123,8 @@ func (cloud *FakeCloudProvider) RepairUnderspecifiedVolumeKey(ctx context.Contex if volumeKey.Zone != common.UnspecifiedValue { return project, volumeKey, nil } - for name, d := range cloud.disks { - if name == volumeKey.Name { + for diskVolKey, d := range cloud.disks { + if diskVolKey == volumeKey.String() { volumeKey.Zone = d.GetZone() return project, volumeKey, nil } @@ -134,7 +149,16 @@ func (cloud *FakeCloudProvider) ListZones(ctx context.Context, region string) ([ return []string{cloud.zone, "country-region-fakesecondzone"}, nil } -func (cloud *FakeCloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, string, error) { +func (cloud *FakeCloudProvider) ListCompatibleDiskTypeZones(ctx context.Context, project string, zones []string, diskType string) ([]string, error) { + // Assume all zones are compatible + return zones, nil +} + +func (cloud *FakeCloudProvider) ListDisksWithFilter(ctx context.Context, fields []googleapi.Field, filter string) ([]*computev1.Disk, string, error) { + return cloud.ListDisks(ctx, fields) +} + +func (cloud *FakeCloudProvider) ListDisks(ctx context.Context, fields []googleapi.Field) ([]*computev1.Disk, string, error) { d := []*computev1.Disk{} for _, cd := range cloud.disks { d = append(d, cd.disk) @@ -142,6 +166,14 @@ func (cloud *FakeCloudProvider) ListDisks(ctx context.Context) ([]*computev1.Dis return d, "", nil } +func (cloud *FakeCloudProvider) ListInstances(ctx context.Context, fields []googleapi.Field) ([]*computev1.Instance, string, error) { + instances := []*computev1.Instance{} + for _, instance := range cloud.instances { + instances = append(instances, instance) + } + return instances, "", nil +} + func (cloud *FakeCloudProvider) ListSnapshots(ctx context.Context, filter string) ([]*computev1.Snapshot, string, error) { var sourceDisk string snapshots := []*computev1.Snapshot{} @@ -166,7 +198,7 @@ func (cloud *FakeCloudProvider) ListSnapshots(ctx context.Context, filter string // Disk Methods func (cloud *FakeCloudProvider) GetDisk(ctx context.Context, project string, volKey *meta.Key, api GCEAPIVersion) (*CloudDisk, error) { - disk, ok := cloud.disks[volKey.Name] + disk, ok := cloud.disks[volKey.String()] if !ok { return nil, notFoundError() } @@ -202,8 +234,8 @@ func (cloud *FakeCloudProvider) ValidateExistingDisk(ctx context.Context, resp * return ValidateDiskParameters(resp, params) } -func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error { - if disk, ok := cloud.disks[volKey.Name]; ok { +func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) error { + if disk, ok := cloud.disks[volKey.String()]; ok { err := cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), int64(capacityRange.GetLimitBytes()), @@ -256,17 +288,17 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, } if containsBetaDiskType(hyperdiskTypes, params.DiskType) { - betaDisk := convertV1DiskToBetaDisk(computeDisk, params.ProvisionedThroughputOnCreate) + betaDisk := convertV1DiskToBetaDisk(computeDisk) betaDisk.EnableConfidentialCompute = params.EnableConfidentialCompute - cloud.disks[volKey.Name] = CloudDiskFromBeta(betaDisk) + cloud.disks[volKey.String()] = CloudDiskFromBeta(betaDisk) } else { - cloud.disks[volKey.Name] = CloudDiskFromV1(computeDisk) + cloud.disks[volKey.String()] = CloudDiskFromV1(computeDisk) } return nil } func (cloud *FakeCloudProvider) DeleteDisk(ctx context.Context, project string, volKey *meta.Key) error { - delete(cloud.disks, volKey.Name) + delete(cloud.disks, volKey.String()) return nil } @@ -306,6 +338,22 @@ func (cloud *FakeCloudProvider) DetachDisk(ctx context.Context, project, deviceN return nil } +func (cloud *FakeCloudProvider) SetDiskAccessMode(ctx context.Context, project string, volKey *meta.Key, accessMode string) error { + disk, ok := cloud.disks[volKey.String()] + if !ok { + return fmt.Errorf("disk %v not found", volKey) + } + + if disk.disk != nil { + disk.disk.AccessMode = accessMode + } + if disk.betaDisk != nil { + disk.betaDisk.AccessMode = accessMode + } + + return nil +} + func (cloud *FakeCloudProvider) GetDiskTypeURI(project string, volKey *meta.Key, diskType string) string { switch volKey.Type() { case meta.Zonal: @@ -325,7 +373,7 @@ func (cloud *FakeCloudProvider) getRegionalDiskTypeURI(project, region, diskType return fmt.Sprintf(diskTypeURITemplateRegional, project, region, diskType) } -func (cloud *FakeCloudProvider) WaitForAttach(ctx context.Context, project string, volKey *meta.Key, instanceZone, instanceName string) error { +func (cloud *FakeCloudProvider) WaitForAttach(ctx context.Context, project string, volKey *meta.Key, diskType, instanceZone, instanceName string) error { return nil } @@ -350,6 +398,9 @@ func (cloud *FakeCloudProvider) GetInstanceOrError(ctx context.Context, instance // Snapshot Methods func (cloud *FakeCloudProvider) GetSnapshot(ctx context.Context, project, snapshotName string) (*computev1.Snapshot, error) { + if !isRFC1035(snapshotName) { + return nil, fmt.Errorf("invalid snapshot name %v: %w", snapshotName, invalidError()) + } snapshot, ok := cloud.snapshots[snapshotName] if !ok { return nil, notFoundError() @@ -386,7 +437,7 @@ func (cloud *FakeCloudProvider) CreateSnapshot(ctx context.Context, project stri } func (cloud *FakeCloudProvider) ResizeDisk(ctx context.Context, project string, volKey *meta.Key, requestBytes int64) (int64, error) { - disk, ok := cloud.disks[volKey.Name] + disk, ok := cloud.disks[volKey.String()] if !ok { return -1, notFoundError() } @@ -428,6 +479,9 @@ func (cloud *FakeCloudProvider) ListImages(ctx context.Context, filter string) ( } func (cloud *FakeCloudProvider) GetImage(ctx context.Context, project, imageName string) (*computev1.Image, error) { + if !isRFC1035(imageName) { + return nil, fmt.Errorf("invalid image name %v: %w", imageName, invalidError()) + } image, ok := cloud.images[imageName] if !ok { return nil, notFoundError() @@ -585,6 +639,7 @@ func notFoundError() *googleapi.Error { func invalidError() *googleapi.Error { return &googleapi.Error{ + Code: http.StatusBadRequest, Errors: []googleapi.ErrorItem{ { Reason: "invalid", diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index 9626787b..13f00fcc 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -17,19 +17,29 @@ package gcecloudprovider import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "regexp" "strings" "time" + rscmgr "cloud.google.com/go/resourcemanager/apiv3" + rscmgrpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/googleapis/gax-go/v2" + "github.com/googleapis/gax-go/v2/apierror" + "golang.org/x/oauth2" computebeta "google.golang.org/api/compute/v0.beta" computev1 "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" ) @@ -51,10 +61,12 @@ type GCEAPIVersion string const ( // V1 key type GCEAPIVersionV1 GCEAPIVersion = "v1" - // Alpha key type + // Beta key type GCEAPIVersionBeta GCEAPIVersion = "beta" ) +var GCEAPIVersions = []GCEAPIVersion{GCEAPIVersionBeta, GCEAPIVersionV1} + // AttachDiskBackoff is backoff used to wait for AttachDisk to complete. // Default values are similar to Poll every 5 seconds with 2 minute timeout. var AttachDiskBackoff = wait.Backoff{ @@ -90,15 +102,19 @@ type GCECompute interface { GetDisk(ctx context.Context, project string, volumeKey *meta.Key, gceAPIVersion GCEAPIVersion) (*CloudDisk, error) RepairUnderspecifiedVolumeKey(ctx context.Context, project string, volumeKey *meta.Key) (string, *meta.Key, error) ValidateExistingDisk(ctx context.Context, disk *CloudDisk, params common.DiskParameters, reqBytes, limBytes int64, multiWriter bool) error - InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error + InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) error DeleteDisk(ctx context.Context, project string, volumeKey *meta.Key) error AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error DetachDisk(ctx context.Context, project, deviceName, instanceZone, instanceName string) error + SetDiskAccessMode(ctx context.Context, project string, volKey *meta.Key, accessMode string) error + ListCompatibleDiskTypeZones(ctx context.Context, project string, zones []string, diskType string) ([]string, error) GetDiskSourceURI(project string, volKey *meta.Key) string GetDiskTypeURI(project string, volKey *meta.Key, diskType string) string - WaitForAttach(ctx context.Context, project string, volKey *meta.Key, instanceZone, instanceName string) error + WaitForAttach(ctx context.Context, project string, volKey *meta.Key, diskType, instanceZone, instanceName string) error ResizeDisk(ctx context.Context, project string, volKey *meta.Key, requestBytes int64) (int64, error) - ListDisks(ctx context.Context) ([]*computev1.Disk, string, error) + ListDisks(ctx context.Context, fields []googleapi.Field) ([]*computev1.Disk, string, error) + ListDisksWithFilter(ctx context.Context, fields []googleapi.Field, filter string) ([]*computev1.Disk, string, error) + ListInstances(ctx context.Context, fields []googleapi.Field) ([]*computev1.Instance, string, error) // Regional Disk Methods GetReplicaZoneURI(project string, zone string) string // Instance Methods @@ -127,7 +143,16 @@ func (cloud *CloudProvider) GetDefaultZone() string { // ListDisks lists disks based on maxEntries and pageToken only in the project // and region that the driver is running in. -func (cloud *CloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, string, error) { +func (cloud *CloudProvider) ListDisks(ctx context.Context, fields []googleapi.Field) ([]*computev1.Disk, string, error) { + filter := "" + return cloud.listDisksInternal(ctx, fields, filter) +} + +func (cloud *CloudProvider) ListDisksWithFilter(ctx context.Context, fields []googleapi.Field, filter string) ([]*computev1.Disk, string, error) { + return cloud.listDisksInternal(ctx, fields, filter) +} + +func (cloud *CloudProvider) listDisksInternal(ctx context.Context, fields []googleapi.Field, filter string) ([]*computev1.Disk, string, error) { region, err := common.GetRegionFromZones([]string{cloud.zone}) if err != nil { return nil, "", fmt.Errorf("failed to get region from zones: %w", err) @@ -140,6 +165,8 @@ func (cloud *CloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, s // listing out regional disks in the region rlCall := cloud.service.RegionDisks.List(cloud.project, region) + rlCall.Fields(fields...) + rlCall.Filter(filter) nextPageToken := "pageToken" for nextPageToken != "" { rDiskList, err := rlCall.Do() @@ -154,6 +181,8 @@ func (cloud *CloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, s // listing out zonal disks in all zones of the region for _, zone := range zones { lCall := cloud.service.Disks.List(cloud.project, zone) + lCall.Fields(fields...) + lCall.Filter(filter) nextPageToken := "pageToken" for nextPageToken != "" { diskList, err := lCall.Do() @@ -168,6 +197,41 @@ func (cloud *CloudProvider) ListDisks(ctx context.Context) ([]*computev1.Disk, s return items, "", nil } +// ListInstances lists instances based on maxEntries and pageToken for the project and region +// that the driver is running in. Filters from cloud.listInstancesConfig.Filters are applied +// to the request. +func (cloud *CloudProvider) ListInstances(ctx context.Context, fields []googleapi.Field) ([]*computev1.Instance, string, error) { + region, err := common.GetRegionFromZones([]string{cloud.zone}) + if err != nil { + return nil, "", fmt.Errorf("failed to get region from zones: %w", err) + } + zones, err := cloud.ListZones(ctx, region) + if err != nil { + return nil, "", err + } + items := []*computev1.Instance{} + + for _, zone := range zones { + lCall := cloud.service.Instances.List(cloud.project, zone) + for _, filter := range cloud.listInstancesConfig.Filters { + lCall = lCall.Filter(filter) + } + lCall = lCall.Fields(fields...) + nextPageToken := "pageToken" + for nextPageToken != "" { + instancesList, err := lCall.Do() + if err != nil { + return nil, "", err + } + items = append(items, instancesList.Items...) + nextPageToken = instancesList.NextPageToken + lCall.PageToken(nextPageToken) + } + } + + return items, "", nil +} + // RepairUnderspecifiedVolumeKey will query the cloud provider and check each zone for the disk specified // by the volume key and return a volume key with a correct zone func (cloud *CloudProvider) RepairUnderspecifiedVolumeKey(ctx context.Context, project string, volumeKey *meta.Key) (string, *meta.Key, error) { @@ -272,7 +336,7 @@ func (cloud *CloudProvider) GetDisk(ctx context.Context, project string, key *me } case meta.Regional: if gceAPIVersion == GCEAPIVersionBeta { - disk, err := cloud.getRegionalAlphaDiskOrError(ctx, project, key.Region, key.Name) + disk, err := cloud.getRegionalBetaDiskOrError(ctx, project, key.Region, key.Name) return CloudDiskFromBeta(disk), err } else { disk, err := cloud.getRegionalDiskOrError(ctx, project, key.Region, key.Name) @@ -307,7 +371,7 @@ func (cloud *CloudProvider) getZonalBetaDiskOrError(ctx context.Context, project return disk, nil } -func (cloud *CloudProvider) getRegionalAlphaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) { +func (cloud *CloudProvider) getRegionalBetaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) { disk, err := cloud.betaService.RegionDisks.Get(project, volumeRegion, volumeName).Context(ctx).Do() if err != nil { return nil, err @@ -371,7 +435,7 @@ func ValidateDiskParameters(disk *CloudDisk, params common.DiskParameters) error return nil } -func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error { +func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) error { klog.V(5).Infof("Inserting disk %v", volKey) description, err := encodeTags(params.Tags) @@ -384,7 +448,7 @@ func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volK if description == "" { description = "Disk created by GCE-PD CSI Driver" } - return cloud.insertZonalDisk(ctx, project, volKey, params, capBytes, capacityRange, snapshotID, volumeContentSourceVolumeID, description, multiWriter) + return cloud.insertZonalDisk(ctx, project, volKey, params, capBytes, capacityRange, snapshotID, volumeContentSourceVolumeID, description, multiWriter, accessMode) case meta.Regional: if description == "" { description = "Regional disk created by GCE-PD CSI Driver" @@ -405,13 +469,28 @@ func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey } } -func convertV1DiskToBetaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCreate int64) *computebeta.Disk { +func convertV1DiskParamsToBeta(v1DiskParams *computev1.DiskParams) *computebeta.DiskParams { + resourceManagerTags := make(map[string]string) + for k, v := range v1DiskParams.ResourceManagerTags { + resourceManagerTags[k] = v + } + return &computebeta.DiskParams{ + ResourceManagerTags: resourceManagerTags, + } +} + +func convertV1DiskToBetaDisk(v1Disk *computev1.Disk) *computebeta.Disk { var dek *computebeta.CustomerEncryptionKey = nil if v1Disk.DiskEncryptionKey != nil { dek = convertV1CustomerEncryptionKeyToBeta(v1Disk.DiskEncryptionKey) } + var params *computebeta.DiskParams = nil + if v1Disk.Params != nil { + params = convertV1DiskParamsToBeta(v1Disk.Params) + } + // Note: this is an incomplete list. It only includes the fields we use for disk creation. betaDisk := &computebeta.Disk{ Name: v1Disk.Name, @@ -429,13 +508,20 @@ func convertV1DiskToBetaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCrea Region: v1Disk.Region, Status: v1Disk.Status, SelfLink: v1Disk.SelfLink, + Params: params, + AccessMode: v1Disk.AccessMode, } + + // Hyperdisk doesn't currently support multiWriter (https://cloud.google.com/compute/docs/disks/hyperdisks#limitations), + // but if multiWriter + hyperdisk is supported in the future, we want the PDCSI driver to support this feature without + // any additional code change. if v1Disk.ProvisionedIops > 0 { betaDisk.ProvisionedIops = v1Disk.ProvisionedIops } - if provisionedThroughputOnCreate > 0 { - betaDisk.ProvisionedThroughput = provisionedThroughputOnCreate + if v1Disk.ProvisionedThroughput > 0 { + betaDisk.ProvisionedThroughput = v1Disk.ProvisionedThroughput } + betaDisk.StoragePool = v1Disk.StoragePool return betaDisk } @@ -495,10 +581,20 @@ func (cloud *CloudProvider) insertRegionalDisk( KmsKeyName: params.DiskEncryptionKMSKey, } } + resourceTags, err := getResourceManagerTags(ctx, cloud.tokenSource, params.ResourceTags) + if err != nil { + return err + } + + if len(resourceTags) > 0 { + diskToCreate.Params = &computev1.DiskParams{ + ResourceManagerTags: resourceTags, + } + } if gceAPIVersion == GCEAPIVersionBeta { var insertOp *computebeta.Operation - betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate, 0) + betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate) betaDiskToCreate.MultiWriter = multiWriter insertOp, err = cloud.betaService.RegionDisks.Insert(project, volKey.Region, betaDiskToCreate).Context(ctx).Do() if insertOp != nil { @@ -515,7 +611,9 @@ func (cloud *CloudProvider) insertRegionalDisk( if IsGCEError(err, "alreadyExists") { disk, err := cloud.GetDisk(ctx, project, volKey, gceAPIVersion) if err != nil { - return err + // failed to GetDisk, however the Disk may already exist + // the error code should be non-Final + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("error when getting disk: %w", err)) } err = cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), @@ -527,16 +625,19 @@ func (cloud *CloudProvider) insertRegionalDisk( klog.Warningf("GCE PD %s already exists, reusing", volKey.Name) return nil } - return status.Error(codes.Internal, fmt.Sprintf("unknown Insert disk error: %v", err.Error())) + // if the error code is considered "final", RegionDisks.Insert might not be retried + return fmt.Errorf("unknown Insert Regional disk error: %w", err) } klog.V(5).Infof("InsertDisk operation %s for disk %s", opName, diskToCreate.Name) err = cloud.waitForRegionalOp(ctx, project, opName, volKey.Region) + // failed to wait for Op to finish, however, the Op possibly is still running as expected + // the error code returned should be non-final if err != nil { if IsGCEError(err, "alreadyExists") { disk, err := cloud.GetDisk(ctx, project, volKey, gceAPIVersion) if err != nil { - return err + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("error when getting disk: %w", err)) } err = cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), @@ -548,7 +649,7 @@ func (cloud *CloudProvider) insertRegionalDisk( klog.Warningf("GCE PD %s already exists after wait, reusing", volKey.Name) return nil } - return fmt.Errorf("unknown Insert disk operation error: %w", err) + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("unknown error when polling the operation: %w", err)) } return nil } @@ -563,13 +664,14 @@ func (cloud *CloudProvider) insertZonalDisk( snapshotID string, volumeContentSourceVolumeID string, description string, - multiWriter bool) error { + multiWriter bool, + accessMode string) error { var ( err error opName string gceAPIVersion = GCEAPIVersionV1 ) - if multiWriter || containsBetaDiskType(hyperdiskTypes, params.DiskType) { + if multiWriter { gceAPIVersion = GCEAPIVersionBeta } @@ -584,6 +686,17 @@ func (cloud *CloudProvider) insertZonalDisk( if params.ProvisionedIOPSOnCreate > 0 { diskToCreate.ProvisionedIops = params.ProvisionedIOPSOnCreate } + if params.ProvisionedThroughputOnCreate > 0 { + diskToCreate.ProvisionedThroughput = params.ProvisionedThroughputOnCreate + } + + if params.StoragePools != nil { + sp := common.StoragePoolInZone(params.StoragePools, volKey.Zone) + if sp == nil { + return status.Errorf(codes.InvalidArgument, "cannot create disk in zone %q: no Storage Pools exist in zone", volKey.Zone) + } + diskToCreate.StoragePool = sp.ResourceName + } if snapshotID != "" { _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) @@ -608,12 +721,24 @@ func (cloud *CloudProvider) insertZonalDisk( KmsKeyName: params.DiskEncryptionKMSKey, } } + diskToCreate.EnableConfidentialCompute = params.EnableConfidentialCompute + + resourceTags, err := getResourceManagerTags(ctx, cloud.tokenSource, params.ResourceTags) + if err != nil { + return err + } + + if len(resourceTags) > 0 { + diskToCreate.Params = &computev1.DiskParams{ + ResourceManagerTags: resourceTags, + } + } + diskToCreate.AccessMode = accessMode if gceAPIVersion == GCEAPIVersionBeta { var insertOp *computebeta.Operation - betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate, params.ProvisionedThroughputOnCreate) + betaDiskToCreate := convertV1DiskToBetaDisk(diskToCreate) betaDiskToCreate.MultiWriter = multiWriter - betaDiskToCreate.EnableConfidentialCompute = params.EnableConfidentialCompute insertOp, err = cloud.betaService.Disks.Insert(project, volKey.Zone, betaDiskToCreate).Context(ctx).Do() if insertOp != nil { opName = insertOp.Name @@ -630,7 +755,9 @@ func (cloud *CloudProvider) insertZonalDisk( if IsGCEError(err, "alreadyExists") { disk, err := cloud.GetDisk(ctx, project, volKey, gceAPIVersion) if err != nil { - return err + // failed to GetDisk, however the Disk may already exist + // the error code should be non-Final + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("error when getting disk: %w", err)) } err = cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), @@ -642,6 +769,7 @@ func (cloud *CloudProvider) insertZonalDisk( klog.Warningf("GCE PD %s already exists, reusing", volKey.Name) return nil } + // if the error code is considered "final", Disks.Insert might not be retried return fmt.Errorf("unknown Insert disk error: %w", err) } klog.V(5).Infof("InsertDisk operation %s for disk %s", opName, diskToCreate.Name) @@ -649,10 +777,12 @@ func (cloud *CloudProvider) insertZonalDisk( err = cloud.waitForZonalOp(ctx, project, opName, volKey.Zone) if err != nil { + // failed to wait for Op to finish, however, the Op possibly is still running as expected + // the error code returned should be non-final if IsGCEError(err, "alreadyExists") { disk, err := cloud.GetDisk(ctx, project, volKey, gceAPIVersion) if err != nil { - return err + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("error when getting disk: %w", err)) } err = cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), @@ -664,7 +794,7 @@ func (cloud *CloudProvider) insertZonalDisk( klog.Warningf("GCE PD %s already exists after wait, reusing", volKey.Name) return nil } - return fmt.Errorf("unknown Insert disk operation error: %w", err) + return common.NewTemporaryError(codes.Unavailable, fmt.Errorf("unknown error when polling the operation: %w", err)) } return nil } @@ -765,6 +895,73 @@ func (cloud *CloudProvider) DetachDisk(ctx context.Context, project, deviceName, return nil } +func (cloud *CloudProvider) SetDiskAccessMode(ctx context.Context, project string, volKey *meta.Key, accessMode string) error { + diskMask := &computev1.Disk{ + AccessMode: accessMode, + Name: volKey.Name, + } + switch volKey.Type() { + case meta.Zonal: + op, err := cloud.service.Disks.Update(project, volKey.Zone, volKey.Name, diskMask).Context(ctx).Paths("accessMode").Do() + if err != nil { + return fmt.Errorf("failed to set access mode for zonal volume %v: %w", volKey, err) + } + klog.V(5).Infof("SetDiskAccessMode operation %s for disk %s", op.Name, volKey.Name) + + err = cloud.waitForZonalOp(ctx, project, op.Name, volKey.Zone) + if err != nil { + return fmt.Errorf("failed waiting for op for zonal disk update for %v: %w", volKey, err) + } + case meta.Regional: + op, err := cloud.service.RegionDisks.Update(project, volKey.Region, volKey.Name, diskMask).Context(ctx).Paths("accessMode").Do() + if err != nil { + return fmt.Errorf("failed to set access mode for regional volume %v: %w", volKey, err) + } + klog.V(5).Infof("SetDiskAccessMode operation %s for disk %s", op.Name, volKey.Name) + + err = cloud.waitForRegionalOp(ctx, project, op.Name, volKey.Region) + if err != nil { + return fmt.Errorf("failed waiting for op for regional disk update for %v: %w", volKey, err) + } + default: + return fmt.Errorf("volume key %v not zonal nor regional", volKey.Name) + } + + return nil +} + +func (cloud *CloudProvider) ListCompatibleDiskTypeZones(ctx context.Context, project string, zones []string, diskType string) ([]string, error) { + diskTypeFilter := fmt.Sprintf("name=%s", diskType) + filters := []string{diskTypeFilter} + diskTypeListCall := cloud.service.DiskTypes.AggregatedList(project).Context(ctx).Filter(strings.Join(filters, " ")) + + supportedZones := []string{} + nextPageToken := "pageToken" + for nextPageToken != "" { + diskTypeList, err := diskTypeListCall.Do() + if err != nil { + return nil, err + } + for _, item := range diskTypeList.Items { + for _, diskType := range item.DiskTypes { + zone, err := common.ParseZoneFromURI(diskType.Zone) + if err != nil { + klog.Warningf("Failed to parse zone %q from diskTypes API: %v", diskType.Zone, err) + continue + } + if slices.Contains(zones, zone) { + supportedZones = append(supportedZones, zone) + } + } + } + + nextPageToken = diskTypeList.NextPageToken + diskTypeListCall.PageToken(nextPageToken) + } + + return supportedZones, nil +} + func (cloud *CloudProvider) GetDiskSourceURI(project string, volKey *meta.Key) string { switch volKey.Type() { case meta.Zonal: @@ -849,18 +1046,46 @@ func (cloud *CloudProvider) waitForGlobalOp(ctx context.Context, project, opName }) } -func (cloud *CloudProvider) WaitForAttach(ctx context.Context, project string, volKey *meta.Key, instanceZone, instanceName string) error { +func (cloud *CloudProvider) waitForAttachOnInstance(ctx context.Context, project string, volKey *meta.Key, instanceZone, instanceName string) error { klog.V(5).Infof("Waiting for attach of disk %v to instance %v to complete...", volKey.Name, instanceName) start := time.Now() return wait.ExponentialBackoff(AttachDiskBackoff, func() (bool, error) { - klog.V(6).Infof("Polling for attach of disk %v to instance %v to complete for %v", volKey.Name, instanceName, time.Since(start)) + klog.V(6).Infof("Polling instances.get for attach of disk %v to instance %v to complete for %v", volKey.Name, instanceName, time.Since(start)) + instance, err := cloud.GetInstanceOrError(ctx, instanceZone, instanceName) + if err != nil { + return false, fmt.Errorf("GetInstance failed to get instance: %w", err) + } + + if instance == nil { + return false, fmt.Errorf("instance %v could not be found", instanceName) + } + + for _, disk := range instance.Disks { + deviceName, err := common.GetDeviceName(volKey) + if err != nil { + return false, fmt.Errorf("failed to get disk device name for %s: %w", volKey, err) + } + + if deviceName == disk.DeviceName { + return true, nil + } + } + return false, nil + }) +} + +func (cloud *CloudProvider) waitForAttachOnDisk(ctx context.Context, project string, volKey *meta.Key, instanceZone, instanceName string) error { + klog.V(5).Infof("Waiting for attach of disk %v to instance %v to complete...", volKey.Name, instanceName) + start := time.Now() + return wait.ExponentialBackoff(AttachDiskBackoff, func() (bool, error) { + klog.V(6).Infof("Polling disks.get for attach of disk %v to instance %v to complete for %v", volKey.Name, instanceName, time.Since(start)) disk, err := cloud.GetDisk(ctx, project, volKey, GCEAPIVersionV1) if err != nil { return false, fmt.Errorf("GetDisk failed to get disk: %w", err) } if disk == nil { - return false, fmt.Errorf("Disk %v could not be found", volKey.Name) + return false, fmt.Errorf("disk %v could not be found", volKey.Name) } for _, user := range disk.GetUsers() { @@ -872,6 +1097,14 @@ func (cloud *CloudProvider) WaitForAttach(ctx context.Context, project string, v }) } +func (cloud *CloudProvider) WaitForAttach(ctx context.Context, project string, volKey *meta.Key, diskType, instanceZone, instanceName string) error { + if cloud.waitForAttachConfig.ShouldUseGetInstanceAPI(diskType) { + return cloud.waitForAttachOnInstance(ctx, project, volKey, instanceZone, instanceName) + } else { + return cloud.waitForAttachOnDisk(ctx, project, volKey, instanceZone, instanceName) + } +} + func wrapOpErr(name string, opErr *computev1.OperationErrorErrors) error { if opErr == nil { return nil @@ -889,7 +1122,8 @@ func wrapOpErr(name string, opErr *computev1.OperationErrorErrors) error { } // codeForGCEOpError return the grpc error code for the passed in -// gce operation error. +// gce operation error. All of these error codes are filtered out from our SLO, +// but will be monitored by the stockout reporting dashboard. func codeForGCEOpError(err computev1.OperationErrorErrors) codes.Code { userErrors := map[string]codes.Code{ "RESOURCE_NOT_FOUND": codes.NotFound, @@ -902,6 +1136,7 @@ func codeForGCEOpError(err computev1.OperationErrorErrors) codes.Code { "REGION_QUOTA_EXCEEDED": codes.ResourceExhausted, "RATE_LIMIT_EXCEEDED": codes.ResourceExhausted, "INVALID_USAGE": codes.InvalidArgument, + "UNSUPPORTED_OPERATION": codes.InvalidArgument, } if code, ok := userErrors[err.Code]; ok { return code @@ -1011,7 +1246,13 @@ func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, vol return nil, err } - return cloud.waitForImageCreation(ctx, project, imageName) + newImage, err := cloud.waitForImageCreation(ctx, project, imageName) + + if err == nil { + err = cloud.attachTagsToResource(ctx, snapshotParams.ResourceTags, project, newImage.Id, imagesType, "", false, resourceManagerHostSubPath) + } + + return newImage, err } func (cloud *CloudProvider) waitForImageCreation(ctx context.Context, project, imageName string) (*computev1.Image, error) { @@ -1162,7 +1403,13 @@ func (cloud *CloudProvider) createZonalDiskSnapshot(ctx context.Context, project return nil, err } - return cloud.waitForSnapshotCreation(ctx, project, snapshotName) + snapshot, err := cloud.waitForSnapshotCreation(ctx, project, snapshotName) + + if err == nil { + err = cloud.attachTagsToResource(ctx, snapshotParams.ResourceTags, project, snapshot.Id, snapshotsType, "", false, resourceManagerHostSubPath) + } + + return snapshot, err } func (cloud *CloudProvider) createRegionalDiskSnapshot(ctx context.Context, project string, volKey *meta.Key, snapshotName string, snapshotParams common.SnapshotParameters, description string) (*computev1.Snapshot, error) { @@ -1178,7 +1425,13 @@ func (cloud *CloudProvider) createRegionalDiskSnapshot(ctx context.Context, proj return nil, err } - return cloud.waitForSnapshotCreation(ctx, project, snapshotName) + snapshot, err := cloud.waitForSnapshotCreation(ctx, project, snapshotName) + + if err == nil { + err = cloud.attachTagsToResource(ctx, snapshotParams.ResourceTags, project, snapshot.Id, snapshotsType, "", false, resourceManagerHostSubPath) + } + + return snapshot, err } @@ -1209,6 +1462,160 @@ func (cloud *CloudProvider) waitForSnapshotCreation(ctx context.Context, project } } +// getResourceManagerTags returns the map of tag keys and values. The tag keys are in the form `tagKeys/{tag_key_id}` +// and the tag values are in the format `tagValues/456`. +func getResourceManagerTags(ctx context.Context, tokenSource oauth2.TokenSource, tagsMap map[string]string) (map[string]string, error) { + if len(tagsMap) <= 0 { + return nil, nil + } + + tagValuesClient, err := createTagValuesClient(ctx, tokenSource, resourceManagerHostSubPath) + if err != nil { + return nil, err + } + defer tagValuesClient.Close() + + tagKeyValueMap := make(map[string]string, len(tagsMap)) + for tagParentIDKey, tagValue := range tagsMap { + getTagValuesReq := &rscmgrpb.GetNamespacedTagValueRequest{ + Name: fmt.Sprintf("%s/%s", tagParentIDKey, tagValue), + } + value, err := tagValuesClient.GetNamespacedTagValue(ctx, getTagValuesReq, getRetryCallOptions()...) + if err != nil { + return nil, err + } + tagKeyValueMap[value.Parent] = value.Name + } + + return tagKeyValueMap, nil +} + +// getRetryCallOptions returns a list of additional call options. If the +// call encounters TooManyRequests error then it will be retried with an +// exponential backoff. +func getRetryCallOptions() []gax.CallOption { + return []gax.CallOption{ + gax.WithRetry(func() gax.Retryer { + return gax.OnHTTPCodes(gax.Backoff{ + Initial: 90 * time.Second, + Max: 5 * time.Minute, + Multiplier: 2, + }, + http.StatusTooManyRequests) + }), + } +} + +// getFilteredTagsMap returns the map of tag keys and the tag values to apply on the resources after +// filtering the tags already existing on a given resource. +func getFilteredTagsMap(ctx context.Context, client *rscmgr.TagBindingsClient, parent string, tagsMap map[string]string) map[string]string { + if len(tagsMap) <= 0 { + klog.Infof("getFilteredTagsMap: tags map is empty for %s compute", parent) + return nil + } + + filteredTagsMap := make(map[string]string, len(tagsMap)) + for key, value := range tagsMap { + filteredTagsMap[key] = value + } + + listBindingsReq := &rscmgrpb.ListEffectiveTagsRequest{ + Parent: parent, + } + bindings := client.ListEffectiveTags(ctx, listBindingsReq) + for i := 0; i < 50; i++ { + binding, err := bindings.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil || binding == nil { + klog.Errorf("failed to list effective tags on %s compute resource: %v: %v", parent, binding, err) + break + } + namespacedTagKey := binding.GetNamespacedTagKey() + if _, exists := filteredTagsMap[namespacedTagKey]; exists { + delete(filteredTagsMap, namespacedTagKey) + klog.V(1).Infof("getFilteredTagsMap: skipping tag %s already exists on the %s compute resource", namespacedTagKey, parent) + } + } + + return filteredTagsMap +} + +// attachTagsToResource attaches tags to a compute resource. As GCP has a rate limit of 600 +// requests per minute, the tags list is filtered so that only the tags which are not attached +// to the resource are attached. The calls to create the tag binding resource for the tags are +// rate limited to 8 requests per second. +func (cloud *CloudProvider) attachTagsToResource( + ctx context.Context, + tagsMap map[string]string, + project string, + resourceID uint64, + resourceType ResourceType, + location string, + isZonal bool, + resourceManagerHostSubPath string) error { + if len(tagsMap) <= 0 { + return nil + } + + tagBindingsClient, err := createTagBindingsClient(ctx, cloud.tokenSource, location, resourceManagerHostSubPath) + if err != nil || tagBindingsClient == nil { + return fmt.Errorf("failed to create tag binding client for adding tags to %d compute %s: %w", resourceID, resourceType, err) + } + defer tagBindingsClient.Close() + + var fullResourceID string + if location != "" { + if isZonal { + fullResourceID = fmt.Sprintf(zonalOrRegionalComputeParentPathFmt, project, "zones", location, resourceType, resourceID) + } else { + fullResourceID = fmt.Sprintf(zonalOrRegionalComputeParentPathFmt, project, "regions", location, resourceType, resourceID) + } + } else { + fullResourceID = fmt.Sprintf(globalComputeParentPathFmt, project, resourceType, resourceID) + } + + filteredTagsMap := getFilteredTagsMap(ctx, tagBindingsClient, fullResourceID, tagsMap) + + errFlag := false + for tagParentIDKey, tagValue := range filteredTagsMap { + if err := cloud.tagsRateLimiter.Wait(ctx); err != nil { + errFlag = true + klog.Errorf("rate limiting request to add %s tag to %d compute %s failed: %v", tagValue, resourceID, resourceType, err) + continue + } + + tagBindingReq := &rscmgrpb.CreateTagBindingRequest{ + TagBinding: &rscmgrpb.TagBinding{ + Parent: fullResourceID, + TagValueNamespacedName: fmt.Sprintf("%s/%s", tagParentIDKey, tagValue), + }, + } + + result, err := tagBindingsClient.CreateTagBinding(ctx, tagBindingReq, getRetryCallOptions()...) + if err != nil { + e, ok := err.(*apierror.APIError) + if ok && e.HTTPCode() == http.StatusConflict { + klog.Infof("tag binding %d: %s/%s already exists", resourceID, tagParentIDKey, tagValue) + continue + } + errFlag = true + klog.Errorf("request to add %s/%s tag to %d compute %s failed: %v", tagParentIDKey, tagValue, resourceID, resourceType, err) + continue + } + + if _, err = result.Wait(ctx); err != nil { + errFlag = true + klog.Errorf("failed to add %s/%s tag to %d compute %s: %v", tagParentIDKey, tagValue, resourceID, resourceType, err) + } + } + if errFlag { + return fmt.Errorf("failed to add tags to %d compute %s", resourceID, resourceType) + } + return nil +} + // kmsKeyEqual returns true if fetchedKMSKey and storageClassKMSKey refer to the same key. // fetchedKMSKey - key returned by the server // diff --git a/pkg/gce-cloud-provider/compute/gce-compute_test.go b/pkg/gce-cloud-provider/compute/gce-compute_test.go index 652b111a..563fb602 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute_test.go +++ b/pkg/gce-cloud-provider/compute/gce-compute_test.go @@ -162,7 +162,7 @@ func TestCodeForGCEOpError(t *testing.T) { { name: "UNSUPPORTED_OPERATION error", inputErr: computev1.OperationErrorErrors{Code: "UNSUPPORTED_OPERATION"}, - expCode: codes.Internal, + expCode: codes.InvalidArgument, }, } diff --git a/pkg/gce-cloud-provider/compute/gce.go b/pkg/gce-cloud-provider/compute/gce.go index 0921eca0..2013a382 100644 --- a/pkg/gce-cloud-provider/compute/gce.go +++ b/pkg/gce-cloud-provider/compute/gce.go @@ -37,23 +37,30 @@ import ( "errors" "fmt" "net/http" + "net/url" "os" "runtime" "time" "golang.org/x/oauth2/google" + "golang.org/x/time/rate" "google.golang.org/api/option" "gopkg.in/gcfg.v1" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" "cloud.google.com/go/compute/metadata" + rscmgr "cloud.google.com/go/resourcemanager/apiv3" "golang.org/x/oauth2" computebeta "google.golang.org/api/compute/v0.beta" "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" ) +type Environment string + const ( TokenURL = "https://accounts.google.com/o/oauth2/token" diskSourceURITemplateSingleZone = "projects/%s/zones/%s/disks/%s" // {gce.projectID}/zones/{disk.Zone}/disks/{disk.Name}" @@ -63,16 +70,55 @@ const ( regionURITemplate = "projects/%s/regions/%s" - replicaZoneURITemplateSingleZone = "projects/%s/zones/%s" // {gce.projectID}/zones/{disk.Zone} + replicaZoneURITemplateSingleZone = "projects/%s/zones/%s" // {gce.projectID}/zones/{disk.Zone} + EnvironmentStaging Environment = "staging" + EnvironmentProduction Environment = "production" + + // resourceManagerHostSubPath is the endpoint for tag requests. + resourceManagerHostSubPath = "cloudresourcemanager.googleapis.com" + + // zonalOrRegionalComputeParentPathFmt is the string format for the full path of compute resource. + // belonging to a zone or a region + zonalOrRegionalComputeParentPathFmt = "//compute.googleapis.com/projects/%s/%s/%s/%s/%d" + + // globalComputeParentPathFmt is the string format for the full path of global compute resource. + globalComputeParentPathFmt = "//compute.googleapis.com/projects/%s/global/%s/%d" + + // gcpTagsRequestRateLimit is the tag request rate limit per second. + gcpTagsRequestRateLimit = 8 + + // gcpTagsRequestTokenBucketSize is the burst/token bucket size used + // for limiting API requests. + gcpTagsRequestTokenBucketSize = 8 ) +// ResourceType indicates the type of a compute resource. +type ResourceType string + +var ( + // snapshotsType is the resource type of compute snapshots. + snapshotsType ResourceType = "snapshots" + // imagesType is the resource type of compute images. + imagesType ResourceType = "images" +) + +// CloudProvider only supports GCE v1/beta Disk APIs. See +// https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/1524 +// for how to add GCE alpha Disk support. type CloudProvider struct { service *compute.Service betaService *computebeta.Service + tokenSource oauth2.TokenSource project string zone string zonesCache map[string][]string + + waitForAttachConfig WaitForAttachConfig + + tagsRateLimiter *rate.Limiter + + listInstancesConfig ListInstancesConfig } var _ GCECompute = &CloudProvider{} @@ -81,6 +127,21 @@ type ConfigFile struct { Global ConfigGlobal `gcfg:"global"` } +type ListInstancesConfig struct { + Filters []string +} + +type WaitForAttachConfig struct { + // A set of disk types that should use the compute instances.get API instead of the + // disks.get API. For certain disk types, using the instances.get API is preferred + // based on the response characteristics of the API. + UseInstancesAPIForDiskTypes []string +} + +func (cfg WaitForAttachConfig) ShouldUseGetInstanceAPI(diskType string) bool { + return slices.Contains(cfg.UseInstancesAPIForDiskTypes, diskType) +} + type ConfigGlobal struct { TokenURL string `gcfg:"token-url"` TokenBody string `gcfg:"token-body"` @@ -88,7 +149,7 @@ type ConfigGlobal struct { Zone string `gcfg:"zone"` } -func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath string, computeEndpoint string) (*CloudProvider, error) { +func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath string, computeEndpoint *url.URL, computeEnvironment Environment, waitForAttachConfig WaitForAttachConfig, listInstancesConfig ListInstancesConfig) (*CloudProvider, error) { configFile, err := readConfig(configPath) if err != nil { return nil, err @@ -103,15 +164,17 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s return nil, err } - svc, err := createCloudService(ctx, vendorVersion, tokenSource, computeEndpoint) + svc, err := createCloudService(ctx, vendorVersion, tokenSource, computeEndpoint, computeEnvironment) if err != nil { return nil, err } + klog.Infof("Compute endpoint for V1 version: %s", svc.BasePath) - betasvc, err := createBetaCloudService(ctx, vendorVersion, tokenSource, computeEndpoint) + betasvc, err := createBetaCloudService(ctx, vendorVersion, tokenSource, computeEndpoint, computeEnvironment) if err != nil { return nil, err } + klog.Infof("Compute endpoint for Beta version: %s", betasvc.BasePath) project, zone, err := getProjectAndZone(configFile) if err != nil { @@ -119,11 +182,17 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s } return &CloudProvider{ - service: svc, - betaService: betasvc, - project: project, - zone: zone, - zonesCache: make(map[string]([]string)), + service: svc, + betaService: betasvc, + tokenSource: tokenSource, + project: project, + zone: zone, + zonesCache: make(map[string]([]string)), + waitForAttachConfig: waitForAttachConfig, + listInstancesConfig: listInstancesConfig, + // GCP has a rate limit of 600 requests per minute, restricting + // here to 8 requests per second. + tagsRateLimiter: common.NewLimiter(gcpTagsRequestRateLimit, gcpTagsRequestTokenBucketSize, true), }, nil } @@ -174,18 +243,25 @@ func readConfig(configPath string) (*ConfigFile, error) { return cfg, nil } -func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*computebeta.Service, error) { - client, err := newOauthClient(ctx, tokenSource) +func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint *url.URL, computeEnvironment Environment) (*computebeta.Service, error) { + computeOpts, err := getComputeVersion(ctx, tokenSource, computeEndpoint, computeEnvironment, GCEAPIVersionBeta) + if err != nil { + klog.Errorf("Failed to get compute endpoint: %s", err) + } + service, err := computebeta.NewService(ctx, computeOpts...) if err != nil { return nil, err } + service.UserAgent = fmt.Sprintf("GCE CSI Driver/%s (%s %s)", vendorVersion, runtime.GOOS, runtime.GOARCH) + return service, nil +} - computeOpts := []option.ClientOption{option.WithHTTPClient(client)} - if computeEndpoint != "" { - betaEndpoint := fmt.Sprintf("%s/compute/beta/", computeEndpoint) - computeOpts = append(computeOpts, option.WithEndpoint(betaEndpoint)) +func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint *url.URL, computeEnvironment Environment) (*compute.Service, error) { + computeOpts, err := getComputeVersion(ctx, tokenSource, computeEndpoint, computeEnvironment, GCEAPIVersionV1) + if err != nil { + klog.Errorf("Failed to get compute endpoint: %s", err) } - service, err := computebeta.NewService(ctx, computeOpts...) + service, err := compute.NewService(ctx, computeOpts...) if err != nil { return nil, err } @@ -193,28 +269,61 @@ func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSour return service, nil } -func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) { - svc, err := createCloudServiceWithDefaultServiceAccount(ctx, vendorVersion, tokenSource, computeEndpoint) - return svc, err +func getComputeVersion(ctx context.Context, tokenSource oauth2.TokenSource, computeEndpoint *url.URL, computeEnvironment Environment, computeVersion GCEAPIVersion) ([]option.ClientOption, error) { + client, err := newOauthClient(ctx, tokenSource) + if err != nil { + return nil, err + } + computeOpts := []option.ClientOption{option.WithHTTPClient(client)} + + if computeEndpoint != nil { + computeEnvironmentSuffix := constructComputeEndpointPath(computeEnvironment, computeVersion) + computeEndpoint.Path = computeEnvironmentSuffix + endpoint := computeEndpoint.String() + computeOpts = append(computeOpts, option.WithEndpoint(endpoint)) + } + return computeOpts, nil } -func createCloudServiceWithDefaultServiceAccount(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) { +func constructComputeEndpointPath(env Environment, version GCEAPIVersion) string { + prefix := "" + if env == EnvironmentStaging { + prefix = fmt.Sprintf("%s_", env) + } + return fmt.Sprintf("compute/%s%s/", prefix, version) +} + +func createTagValuesClient(ctx context.Context, tokenSource oauth2.TokenSource, resourceManagerHostSubPath string) (*rscmgr.TagValuesClient, error) { client, err := newOauthClient(ctx, tokenSource) if err != nil { return nil, err } - computeOpts := []option.ClientOption{option.WithHTTPClient(client)} - if computeEndpoint != "" { - v1Endpoint := fmt.Sprintf("%s/compute/v1/", computeEndpoint) - computeOpts = append(computeOpts, option.WithEndpoint(v1Endpoint)) + endpoint := fmt.Sprintf("https://%s", resourceManagerHostSubPath) + opts := []option.ClientOption{ + option.WithHTTPClient(client), + option.WithEndpoint(endpoint), } - service, err := compute.NewService(ctx, computeOpts...) + return rscmgr.NewTagValuesRESTClient(ctx, opts...) +} + +func createTagBindingsClient(ctx context.Context, tokenSource oauth2.TokenSource, location string, resourceManagerHostSubPath string) (*rscmgr.TagBindingsClient, error) { + client, err := newOauthClient(ctx, tokenSource) if err != nil { return nil, err } - service.UserAgent = fmt.Sprintf("GCE CSI Driver/%s (%s %s)", vendorVersion, runtime.GOOS, runtime.GOARCH) - return service, nil + + var endpoint string + if location != "" { + endpoint = fmt.Sprintf("https://%s-%s", location, resourceManagerHostSubPath) + } else { + endpoint = fmt.Sprintf("https://%s", resourceManagerHostSubPath) + } + opts := []option.ClientOption{ + option.WithHTTPClient(client), + option.WithEndpoint(endpoint), + } + return rscmgr.NewTagBindingsRESTClient(ctx, opts...) } func newOauthClient(ctx context.Context, tokenSource oauth2.TokenSource) (*http.Client, error) { diff --git a/pkg/gce-cloud-provider/compute/gce_test.go b/pkg/gce-cloud-provider/compute/gce_test.go index 5bb2aed8..94bb4fd9 100644 --- a/pkg/gce-cloud-provider/compute/gce_test.go +++ b/pkg/gce-cloud-provider/compute/gce_test.go @@ -18,14 +18,30 @@ limitations under the License. package gcecloudprovider import ( + "context" "errors" "fmt" "net/http" + "net/url" "testing" + "time" + "golang.org/x/oauth2" + + "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" ) +type mockTokenSource struct{} + +func (*mockTokenSource) Token() (*oauth2.Token, error) { + return &oauth2.Token{ + AccessToken: "access", + TokenType: "Bearer", + RefreshToken: "refresh", + Expiry: time.Now().Add(1 * time.Hour), + }, nil +} func TestIsGCEError(t *testing.T) { testCases := []struct { name string @@ -84,3 +100,61 @@ func TestIsGCEError(t *testing.T) { } } } + +func TestGetComputeVersion(t *testing.T) { + testCases := []struct { + name string + computeEndpoint *url.URL + computeEnvironment Environment + computeVersion GCEAPIVersion + expectedEndpoint string + expectError bool + }{ + + { + name: "check for production environment", + computeEndpoint: convertStringToURL("https://compute.googleapis.com"), + computeEnvironment: EnvironmentProduction, + computeVersion: GCEAPIVersionBeta, + expectedEndpoint: "https://compute.googleapis.com/compute/beta/", + expectError: false, + }, + { + name: "check for staging environment", + computeEndpoint: convertStringToURL("https://compute.googleapis.com"), + computeEnvironment: EnvironmentStaging, + computeVersion: GCEAPIVersionV1, + expectedEndpoint: "https://compute.googleapis.com/compute/staging_v1/", + expectError: false, + }, + { + name: "check for random string as endpoint", + computeEndpoint: convertStringToURL(""), + computeEnvironment: "prod", + computeVersion: "v1", + expectedEndpoint: "compute/v1/", + expectError: true, + }, + } + for _, tc := range testCases { + ctx := context.Background() + computeOpts, err := getComputeVersion(ctx, &mockTokenSource{}, tc.computeEndpoint, tc.computeEnvironment, tc.computeVersion) + service, _ := compute.NewService(ctx, computeOpts...) + gotEndpoint := service.BasePath + if err != nil && !tc.expectError { + t.Fatalf("Got error %v", err) + } + if gotEndpoint != tc.expectedEndpoint && !tc.expectError { + t.Fatalf("expected endpoint %s, got endpoint %s", tc.expectedEndpoint, gotEndpoint) + } + } + +} + +func convertStringToURL(urlString string) *url.URL { + parsedURL, err := url.ParseRequestURI(urlString) + if err != nil { + return nil + } + return parsedURL +} diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index 78359b15..a2d57000 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" csi "github.com/container-storage-interface/spec/lib/go/csi" compute "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" @@ -47,8 +48,8 @@ type GCEControllerServer struct { CloudProvider gce.GCECompute Metrics metrics.MetricsManager - disks []*compute.Disk - seen map[string]int + volumeEntries []*csi.ListVolumesResponse_Entry + volumeEntriesSeen map[string]int snapshots []*csi.ListSnapshotsResponse_Entry snapshotTokens map[string]int @@ -91,6 +92,54 @@ type GCEControllerServer struct { // publish/unpublish call will clear the backoff condition for a node and // disk. errorBackoff *csiErrorBackoff + + // Requisite zones to fallback to when provisioning a disk. + // If there are an insufficient number of zones available in the union + // of preferred/requisite topology, this list is used instead of + // the passed in requisite topology. + // The main use case of this field is to support Regional Persistent Disk + // provisioning in GKE Autopilot, where a GKE cluster to + // be scaled down to 1 zone. + fallbackRequisiteZones []string + + // If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools. + enableStoragePools bool + + multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig + + listVolumesConfig ListVolumesConfig +} + +type MultiZoneVolumeHandleConfig struct { + // A set of supported disk types that are compatible with multi-zone volumeHandles. + // The disk type is only validated on ControllerPublish. + // Other operations that interacti with volumeHandle (ListVolumes/ControllerUnpublish) + // don't validate the disk type. This ensures existing published multi-zone volumes + // are listed and unpublished correctly. This allows this flag + // to be ratcheted to be more restricted without affecting volumes that are already + // published. + DiskTypes []string + + // If set to true, the CSI driver will enable the multi-zone volumeHandle feature. + // If set to false, volumeHandles that contain 'multi-zone' will not be translated + // to their respective attachment zone (based on the node), which will result in + // an "Unknown zone" error on ControllerPublish/ControllerUnpublish. + Enable bool +} + +type ListVolumesConfig struct { + UseInstancesAPIForPublishedNodes bool +} + +func (c ListVolumesConfig) listDisksFields() []googleapi.Field { + if c.UseInstancesAPIForPublishedNodes { + // If we are using the instances.list API in ListVolumes, + // don't include the users field in the response, as an optimization. + // We rely on instances.list items.disks for attachment pairings. + return listDisksFieldsWithoutUsers + } + + return listDisksFieldsWithUsers } type csiErrorBackoffId string @@ -144,10 +193,31 @@ const ( resourceApiScheme = "https" resourceApiService = "compute" resourceProject = "projects" + + listDisksUsersField = googleapi.Field("items/users") + + readOnlyManyAccessMode = "READ_ONLY_MANY" ) var ( - validResourceApiVersions = map[string]bool{"v1": true, "alpha": true, "beta": true} + validResourceApiVersions = map[string]bool{"v1": true, "alpha": true, "beta": true, "staging_v1": true, "staging_beta": true, "staging_alpha": true} + + // By default GCE returns a lot of data for each instance. Request only a subset of the fields. + listInstancesFields = []googleapi.Field{ + "items/disks/deviceName", + "items/disks/source", + "items/selfLink", + "nextPageToken", + } + + // By default GCE returns a lot of data for each disk. Request only a subset of the fields. + listDisksFieldsWithoutUsers = []googleapi.Field{ + "items/labels", + "items/selfLink", + "nextPageToken", + } + listDisksFieldsWithUsers = append(listDisksFieldsWithoutUsers, "items/users") + disksWithModifiableAccessMode = []string{"hyperdisk-ml"} ) func isDiskReady(disk *gce.CloudDisk) (bool, error) { @@ -212,25 +282,34 @@ func useVolumeCloning(req *csi.CreateVolumeRequest) bool { } func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + response, err := gceCS.createVolumeInternal(ctx, req) + if err != nil && req != nil { + klog.V(4).Infof("CreateVolume succeeded for volume %v", req.Name) + } + + return response, err +} + +func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("CreateVolume", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("CreateVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() // Validate arguments volumeCapabilities := req.GetVolumeCapabilities() - name := req.GetName() capacityRange := req.GetCapacityRange() - if len(name) == 0 { + if len(req.GetName()) == 0 { return nil, status.Error(codes.InvalidArgument, "CreateVolume Name must be provided") } if volumeCapabilities == nil || len(volumeCapabilities) == 0 { return nil, status.Error(codes.InvalidArgument, "CreateVolume Volume capabilities must be provided") } - capBytes, err := getRequestCapacity(capacityRange) - if err != nil { + // Validate request capacity early + if _, err := getRequestCapacity(capacityRange); err != nil { return nil, status.Errorf(codes.InvalidArgument, "CreateVolume Request Capacity is invalid: %v", err.Error()) } @@ -241,17 +320,36 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // Apply Parameters (case-insensitive). We leave validation of // the values to the cloud provider. - params, err := common.ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.name, gceCS.Driver.extraVolumeLabels) + params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) diskTypeForMetric = params.DiskType enableConfidentialCompute = strconv.FormatBool(params.EnableConfidentialCompute) + hasStoragePools := len(params.StoragePools) > 0 + enableStoragePools = strconv.FormatBool(hasStoragePools) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } - // Determine multiWriter - gceAPIVersion := gce.GCEAPIVersionV1 - multiWriter, _ := getMultiWriterFromCapabilities(volumeCapabilities) - if multiWriter { - gceAPIVersion = gce.GCEAPIVersionBeta + // Validate multiwriter + if _, err := getMultiWriterFromCapabilities(volumeCapabilities); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error()) + } + + err = validateStoragePools(req, params, gceCS.CloudProvider.GetDefaultProject()) + if err != nil { + // Reassign error so that all errors are reported as InvalidArgument to RecordOperationErrorMetrics. + err = status.Errorf(codes.InvalidArgument, "CreateVolume failed to validate storage pools: %v", err) + return nil, err + } + + // Validate VolumeContentSource is set when access mode is read only + readonly, _ := getReadOnlyFromCapabilities(volumeCapabilities) + if readonly && req.GetVolumeContentSource() == nil { + return nil, status.Error(codes.InvalidArgument, "VolumeContentSource must be provided when AccessMode is set to read only") + } + + // Validate multi-zone provisioning configuration + err = gceCS.validateMultiZoneProvisioning(req, params) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to validate multi-zone provisioning request: %v", err) } // Verify that the regional availability class is only used on regional disks. @@ -259,6 +357,159 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre return nil, status.Errorf(codes.InvalidArgument, "invalid availabilty class for zonal disk") } + if gceCS.multiZoneVolumeHandleConfig.Enable && params.MultiZoneProvisioning { + // Create multi-zone disk, that may have up to N disks. + return gceCS.createMultiZoneDisk(ctx, req, params) + } + + // Create single device zonal or regional disk + return gceCS.createSingleDeviceDisk(ctx, req, params) +} + +func (gceCS *GCEControllerServer) getSupportedZonesForPDType(ctx context.Context, zones []string, diskType string) ([]string, error) { + project := gceCS.CloudProvider.GetDefaultProject() + zones, err := gceCS.CloudProvider.ListCompatibleDiskTypeZones(ctx, project, zones, diskType) + if err != nil { + return nil, err + } + return zones, nil +} + +func (gceCS *GCEControllerServer) getMultiZoneProvisioningZones(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) ([]string, error) { + top := req.GetAccessibilityRequirements() + if top == nil { + return nil, status.Errorf(codes.InvalidArgument, "no topology specified") + } + prefZones, err := getZonesFromTopology(top.GetPreferred()) + if err != nil { + return nil, fmt.Errorf("could not get zones from preferred topology: %w", err) + } + reqZones, err := getZonesFromTopology(top.GetRequisite()) + if err != nil { + return nil, fmt.Errorf("could not get zones from requisite topology: %w", err) + } + prefSet := sets.NewString(prefZones...) + reqSet := sets.NewString(reqZones...) + prefAndReqSet := prefSet.Union(reqSet) + availableZones := prefAndReqSet.List() + if prefAndReqSet.Len() == 0 { + // If there are no specified zones, this means that there were no aggregate + // zones (eg: no nodes running) in the cluster + availableZones = gceCS.fallbackRequisiteZones + } + + supportedZones, err := gceCS.getSupportedZonesForPDType(ctx, availableZones, params.DiskType) + if err != nil { + return nil, fmt.Errorf("could not get supported zones for disk type %v from zone list %v: %w", params.DiskType, prefAndReqSet.List(), err) + } + + // It's possible that the provided requisite zones shifted since the last time that + // CreateVolume was called (eg: due to a node being removed in a zone) + // Ensure that we combine the supportedZones with any existing zones to get the full set. + existingZones, err := gceCS.getZonesWithDiskNameAndType(ctx, req.Name, params.DiskType) + if err != nil { + return nil, common.LoggedError(fmt.Sprintf("failed to check existing list of zones for request: %v", req.Name), err) + } + + supportedSet := sets.NewString(supportedZones...) + existingSet := sets.NewString(existingZones...) + combinedZones := existingSet.Union(supportedSet) + + return combinedZones.List(), nil +} + +func (gceCS *GCEControllerServer) createMultiZoneDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { + // Determine the zones that are needed. + var err error + + // For multi-zone, we either select: + // 1) The zones specified in requisite topology requirements + // 2) All zones in the region that are compatible with the selected disk type + zones, err := gceCS.getMultiZoneProvisioningZones(ctx, req, params) + if err != nil { + return nil, err + } + + multiZoneVolKey := meta.ZonalKey(req.GetName(), common.MultiZoneValue) + volumeID, err := common.KeyToVolumeID(multiZoneVolKey, gceCS.CloudProvider.GetDefaultProject()) + if err != nil { + return nil, err + } + if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { + return nil, status.Errorf(codes.Aborted, common.VolumeOperationAlreadyExistsFmt, volumeID) + } + defer gceCS.volumeLocks.Release(volumeID) + + createDiskErrs := []error{} + createdDisks := make([]*gce.CloudDisk, 0, len(zones)) + for _, zone := range zones { + volKey := meta.ZonalKey(req.GetName(), zone) + klog.V(4).Infof("Creating single zone disk for zone %q and volume: %v", zone, volKey) + disk, err := gceCS.createSingleDisk(ctx, req, params, volKey, []string{zone}) + if err != nil { + createDiskErrs = append(createDiskErrs, err) + continue + } + + createdDisks = append(createdDisks, disk) + } + if len(createDiskErrs) > 0 { + return nil, common.LoggedError("Failed to create multi-zone disk: ", errors.Join(createDiskErrs...)) + } + + if len(createdDisks) == 0 { + return nil, status.Errorf(codes.Internal, "could not create any disks for request: %v", req) + } + + // Use the first response as a template + volumeId := fmt.Sprintf("projects/%s/zones/%s/disks/%s", gceCS.CloudProvider.GetDefaultProject(), common.MultiZoneValue, req.GetName()) + klog.V(4).Infof("CreateVolume succeeded for multi-zone disks in zones %s: %v", zones, multiZoneVolKey) + return generateCreateVolumeResponseWithVolumeId(createdDisks[0], zones, params, volumeId), nil +} + +func (gceCS *GCEControllerServer) getZonesWithDiskNameAndType(ctx context.Context, name string, diskType string) ([]string, error) { + zoneOnlyFields := []googleapi.Field{"items/zone", "items/type"} + nameAndRegionFilter := fmt.Sprintf("name=%s", name) + disksWithZone, _, err := gceCS.CloudProvider.ListDisksWithFilter(ctx, zoneOnlyFields, nameAndRegionFilter) + if err != nil { + return nil, fmt.Errorf("failed to check existing zones for disk name %v: %w", name, err) + } + zones := []string{} + for _, disk := range disksWithZone { + if !strings.Contains(disk.Type, diskType) || disk.Zone == "" { + continue + } + diskZone, err := common.ParseZoneFromURI(disk.Zone) + if err != nil { + klog.Warningf("Malformed zone URI %v for disk %v from ListDisks call. Skipping", disk.Zone, name) + continue + } + zones = append(zones, diskZone) + } + return zones, nil +} + +func (gceCS *GCEControllerServer) updateAccessModeIfNecessary(ctx context.Context, volKey *meta.Key, disk *gce.CloudDisk, readonly bool) error { + if !slices.Contains(disksWithModifiableAccessMode, disk.GetPDType()) { + // If this isn't a disk that has access mode (eg: Hyperdisk ML), return + // So far, HyperdiskML is the only disk type that allows the disk type to be modified. + return nil + } + if !readonly { + // Only update the access mode if we're converting from ReadWrite to ReadOnly + return nil + } + project := gceCS.CloudProvider.GetDefaultProject() + if disk.GetAccessMode() == readOnlyManyAccessMode { + // If the access mode is already readonly, return + return nil + } + + return gceCS.CloudProvider.SetDiskAccessMode(ctx, project, volKey, readOnlyManyAccessMode) +} + +func (gceCS *GCEControllerServer) createSingleDeviceDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { + var err error var locationTopReq *locationRequirements if useVolumeCloning(req) { locationTopReq, err = cloningLocationRequirements(req, params.ReplicationType) @@ -272,17 +523,17 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre var volKey *meta.Key switch params.ReplicationType { case replicationTypeNone: - zones, err = pickZones(ctx, gceCS, req.GetAccessibilityRequirements(), 1, locationTopReq) + zones, err = gceCS.pickZones(ctx, req.GetAccessibilityRequirements(), 1, locationTopReq) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to pick zones for disk: %v", err.Error()) } if len(zones) != 1 { return nil, status.Errorf(codes.Internal, "Failed to pick exactly 1 zone for zonal disk, got %v instead", len(zones)) } - volKey = meta.ZonalKey(name, zones[0]) + volKey = meta.ZonalKey(req.GetName(), zones[0]) case replicationTypeRegionalPD: - zones, err = pickZones(ctx, gceCS, req.GetAccessibilityRequirements(), 2, locationTopReq) + zones, err = gceCS.pickZones(ctx, req.GetAccessibilityRequirements(), 2, locationTopReq) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to pick zones for disk: %v", err.Error()) } @@ -290,7 +541,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if err != nil { return nil, status.Errorf(codes.InvalidArgument, "CreateVolume failed to get region from zones: %v", err.Error()) } - volKey = meta.RegionalKey(name, region) + volKey = meta.RegionalKey(req.GetName(), region) default: return nil, status.Errorf(codes.InvalidArgument, "CreateVolume replication type '%s' is not supported", params.ReplicationType) } @@ -303,12 +554,31 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre return nil, status.Errorf(codes.Aborted, common.VolumeOperationAlreadyExistsFmt, volumeID) } defer gceCS.volumeLocks.Release(volumeID) + disk, err := gceCS.createSingleDisk(ctx, req, params, volKey, zones) + + if err != nil { + return nil, common.LoggedError("CreateVolume failed: %v", err) + } + + return generateCreateVolumeResponseWithVolumeId(disk, zones, params, volumeID), err +} + +func (gceCS *GCEControllerServer) createSingleDisk(ctx context.Context, req *csi.CreateVolumeRequest, params common.DiskParameters, volKey *meta.Key, zones []string) (*gce.CloudDisk, error) { + capacityRange := req.GetCapacityRange() + capBytes, _ := getRequestCapacity(capacityRange) + multiWriter, _ := getMultiWriterFromCapabilities(req.GetVolumeCapabilities()) + readonly, _ := getReadOnlyFromCapabilities(req.GetVolumeCapabilities()) + accessMode := "" + if readonly && slices.Contains(disksWithModifiableAccessMode, params.DiskType) { + accessMode = readOnlyManyAccessMode + } // Validate if disk already exists - existingDisk, err := gceCS.CloudProvider.GetDisk(ctx, gceCS.CloudProvider.GetDefaultProject(), volKey, gceAPIVersion) + existingDisk, err := gceCS.CloudProvider.GetDisk(ctx, gceCS.CloudProvider.GetDefaultProject(), volKey, getGCEApiVersion(multiWriter)) if err != nil { if !gce.IsGCEError(err, "notFound") { - return nil, common.LoggedError("CreateVolume, failed to getDisk when validating: ", err) + // failed to GetDisk, however the Disk may already be created, the error code should be non-Final + return nil, common.LoggedError("CreateVolume, failed to getDisk when validating: ", status.Error(codes.Unavailable, err.Error())) } } if err == nil { @@ -323,15 +593,15 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre ready, err := isDiskReady(existingDisk) if err != nil { - return nil, common.LoggedError("CreateVolume disk "+volKey.String()+" had error checking ready status: ", err) + return nil, status.Errorf(codes.Aborted, "CreateVolume disk %q had error checking ready status: %v", volKey.String(), err.Error()) } if !ready { - return nil, status.Errorf(codes.Internal, "CreateVolume existing disk %v is not ready", volKey) + return nil, status.Errorf(codes.Aborted, "CreateVolume existing disk %v is not ready", volKey) } // If there is no validation error, immediately return success klog.V(4).Infof("CreateVolume succeeded for disk %v, it already exists and was compatible", volKey) - return generateCreateVolumeResponse(existingDisk, zones, params) + return existingDisk, nil } snapshotID := "" @@ -359,7 +629,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre } // Verify that the volume in VolumeContentSource exists. - diskFromSourceVolume, err := gceCS.CloudProvider.GetDisk(ctx, project, sourceVolKey, gceAPIVersion) + diskFromSourceVolume, err := gceCS.CloudProvider.GetDisk(ctx, project, sourceVolKey, getGCEApiVersion(multiWriter)) if err != nil { if gce.IsGCEError(err, "notFound") { return nil, status.Errorf(codes.NotFound, "CreateVolume source volume %s does not exist", volumeContentSourceVolumeID) @@ -401,26 +671,24 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // Verify the source disk is ready. ready, err := isDiskReady(diskFromSourceVolume) if err != nil { - return nil, common.LoggedError("CreateVolume disk from source volume "+sourceVolKey.String()+" had error checking ready status: ", err) + return nil, status.Errorf(codes.Aborted, "CreateVolume disk from source volume %q had error checking ready status: %v", sourceVolKey.String(), err.Error()) } if !ready { - return nil, status.Errorf(codes.Internal, "CreateVolume disk from source volume %v is not ready", sourceVolKey) + return nil, status.Errorf(codes.Aborted, "CreateVolume disk from source volume %v is not ready", sourceVolKey) } } - } else { // if VolumeContentSource is nil, validate access mode is not read only - if readonly, _ := getReadOnlyFromCapabilities(volumeCapabilities); readonly { - return nil, status.Error(codes.InvalidArgument, "VolumeContentSource must be provided when AccessMode is set to read only") - } } // Create the disk var disk *gce.CloudDisk + name := req.GetName() + switch params.ReplicationType { case replicationTypeNone: if len(zones) != 1 { return nil, status.Errorf(codes.Internal, "CreateVolume failed to get a single zone for creating zonal disk, instead got: %v", zones) } - disk, err = createSingleZoneDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) + disk, err = createSingleZoneDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter, accessMode) if err != nil { return nil, common.LoggedError("CreateVolume failed to create single zonal disk "+name+": ", err) } @@ -428,7 +696,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if len(zones) != 2 { return nil, status.Errorf(codes.Internal, "CreateVolume failed to get a 2 zones for creating regional disk, instead got: %v", zones) } - disk, err = createRegionalDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) + disk, err = createRegionalDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter, accessMode) if err != nil { return nil, common.LoggedError("CreateVolume failed to create regional disk "+name+": ", err) } @@ -445,17 +713,11 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre } klog.V(4).Infof("CreateVolume succeeded for disk %v", volKey) - return generateCreateVolumeResponse(disk, zones, params) - + return disk, nil } func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { var err error - diskTypeForMetric := metrics.DefaultDiskTypeForMetric - enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute - defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("DeleteVolume", err, diskTypeForMetric, enableConfidentialCompute) - }() // Validate arguments volumeID := req.GetVolumeId() if len(volumeID) == 0 { @@ -470,6 +732,78 @@ func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.Del return &csi.DeleteVolumeResponse{}, nil } + volumeIsMultiZone := isMultiZoneVolKey(volKey) + if gceCS.multiZoneVolumeHandleConfig.Enable && volumeIsMultiZone { + // Delete multi-zone disk, that may have up to N disks. + return gceCS.deleteMultiZoneDisk(ctx, req, project, volKey) + } + + // Delete zonal or regional disk + return gceCS.deleteSingleDeviceDisk(ctx, req, project, volKey) +} + +func getGCEApiVersion(multiWriter bool) gce.GCEAPIVersion { + if multiWriter { + return gce.GCEAPIVersionBeta + } + + return gce.GCEAPIVersionV1 +} + +func (gceCS *GCEControllerServer) deleteMultiZoneDisk(ctx context.Context, req *csi.DeleteVolumeRequest, project string, volKey *meta.Key) (*csi.DeleteVolumeResponse, error) { + // List disks with same name + var err error + diskTypeForMetric := metrics.DefaultDiskTypeForMetric + enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools + defer func() { + gceCS.Metrics.RecordOperationErrorMetrics("DeleteVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) + }() + existingZones := []string{gceCS.CloudProvider.GetDefaultZone()} + zones, err := getDefaultZonesInRegion(ctx, gceCS, existingZones) + if err != nil { + return nil, fmt.Errorf("failed to list default zones: %w", err) + } + + volumeID := req.GetVolumeId() + if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { + return nil, status.Errorf(codes.Aborted, common.VolumeOperationAlreadyExistsFmt, volumeID) + } + defer gceCS.volumeLocks.Release(volumeID) + + deleteDiskErrs := []error{} + for _, zone := range zones { + zonalVolKey := &meta.Key{ + Name: volKey.Name, + Region: volKey.Region, + Zone: zone, + } + disk, _ := gceCS.CloudProvider.GetDisk(ctx, project, zonalVolKey, gce.GCEAPIVersionV1) + // TODO: Consolidate the parameters here, rather than taking the last. + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) + err := gceCS.CloudProvider.DeleteDisk(ctx, project, zonalVolKey) + if err != nil { + deleteDiskErrs = append(deleteDiskErrs, gceCS.CloudProvider.DeleteDisk(ctx, project, volKey)) + } + } + + if len(deleteDiskErrs) > 0 { + return nil, common.LoggedError("Failed to delete multi-zone disk: ", errors.Join(deleteDiskErrs...)) + } + + klog.V(4).Infof("DeleteVolume succeeded for disk %v", volKey) + return &csi.DeleteVolumeResponse{}, nil +} + +func (gceCS *GCEControllerServer) deleteSingleDeviceDisk(ctx context.Context, req *csi.DeleteVolumeRequest, project string, volKey *meta.Key) (*csi.DeleteVolumeResponse, error) { + var err error + diskTypeForMetric := metrics.DefaultDiskTypeForMetric + enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools + defer func() { + gceCS.Metrics.RecordOperationErrorMetrics("DeleteVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) + }() + volumeID := req.GetVolumeId() project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { @@ -484,7 +818,7 @@ func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.Del } defer gceCS.volumeLocks.Release(volumeID) disk, _ := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(disk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) err = gceCS.CloudProvider.DeleteDisk(ctx, project, volKey) if err != nil { return nil, common.LoggedError("Failed to delete disk: ", err) @@ -498,8 +832,9 @@ func (gceCS *GCEControllerServer) ControllerPublishVolume(ctx context.Context, r var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("ControllerPublishVolume", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("ControllerPublishVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() // Only valid requests will be accepted _, _, _, err = gceCS.validateControllerPublishVolumeRequest(ctx, req) @@ -513,7 +848,7 @@ func (gceCS *GCEControllerServer) ControllerPublishVolume(ctx context.Context, r } resp, err, disk := gceCS.executeControllerPublishVolume(ctx, req) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(disk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) if err != nil { klog.Infof("For node %s adding backoff due to error for volume %s: %v", req.NodeId, req.VolumeId, err) gceCS.errorBackoff.next(backoffId, common.CodeForError(err)) @@ -566,6 +901,52 @@ func parseMachineType(machineTypeUrl string) string { return machineType } +func convertMultiZoneVolKeyToZoned(volumeKey *meta.Key, instanceZone string) *meta.Key { + volumeKey.Zone = instanceZone + return volumeKey +} + +func (gceCS *GCEControllerServer) validateMultiZoneDisk(volumeID string, disk *gce.CloudDisk) error { + if !slices.Contains(gceCS.multiZoneVolumeHandleConfig.DiskTypes, disk.GetPDType()) { + return status.Errorf(codes.InvalidArgument, "Multi-zone volumeID %q points to disk with unsupported disk type %q: %v", volumeID, disk.GetPDType(), disk.GetSelfLink()) + } + if _, ok := disk.GetLabels()[common.MultiZoneLabel]; !ok { + return status.Errorf(codes.InvalidArgument, "Multi-zone volumeID %q points to disk that is missing label %q: %v", volumeID, common.MultiZoneLabel, disk.GetSelfLink()) + } + return nil +} + +func (gceCS *GCEControllerServer) validateMultiZoneProvisioning(req *csi.CreateVolumeRequest, params common.DiskParameters) error { + if !gceCS.multiZoneVolumeHandleConfig.Enable { + return nil + } + if !params.MultiZoneProvisioning { + return nil + } + + // For volume populator, we want to allow multiple RWO disks to be created + // with the same name, so they can be hydrated across multiple zones. + + // We don't have support volume cloning from an existing PVC + if useVolumeCloning(req) { + return fmt.Errorf("%q parameter does not support volume cloning", common.ParameterKeyEnableMultiZoneProvisioning) + } + + if readonly, _ := getReadOnlyFromCapabilities(req.GetVolumeCapabilities()); !readonly && req.GetVolumeContentSource() != nil { + return fmt.Errorf("%q parameter does not support specifying volume content source in readwrite mode", common.ParameterKeyEnableMultiZoneProvisioning) + } + + if !slices.Contains(gceCS.multiZoneVolumeHandleConfig.DiskTypes, params.DiskType) { + return fmt.Errorf("%q parameter with unsupported disk type: %v", common.ParameterKeyEnableMultiZoneProvisioning, params.DiskType) + } + + return nil +} + +func isMultiZoneVolKey(volumeKey *meta.Key) bool { + return volumeKey.Type() == meta.Zonal && volumeKey.Zone == common.MultiZoneValue +} + func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error, *gce.CloudDisk) { project, volKey, pdcsiContext, err := gceCS.validateControllerPublishVolumeRequest(ctx, req) if err != nil { @@ -581,6 +962,21 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con PublishContext: nil, } + instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) + if err != nil { + return nil, status.Errorf(codes.NotFound, "could not split nodeID: %v", err.Error()), nil + } + + volumeIsMultiZone := isMultiZoneVolKey(volKey) + if gceCS.multiZoneVolumeHandleConfig.Enable && volumeIsMultiZone { + // Only allow read-only attachment for "multi-zone" volumes + if !readOnly { + return nil, status.Errorf(codes.InvalidArgument, "'multi-zone' volume only supports 'readOnly': %v", volumeID), nil + } + + volKey = convertMultiZoneVolKeyToZoned(volKey, instanceZone) + } + project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { @@ -603,10 +999,6 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con } return nil, common.LoggedError("Failed to getDisk: ", err), disk } - instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) - if err != nil { - return nil, status.Errorf(codes.NotFound, "could not split nodeID: %v", err.Error()), disk - } instance, err := gceCS.CloudProvider.GetInstanceOrError(ctx, instanceZone, instanceName) if err != nil { if gce.IsGCENotFoundError(err) { @@ -615,6 +1007,12 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con return nil, common.LoggedError("Failed to get instance: ", err), disk } + if gceCS.multiZoneVolumeHandleConfig.Enable && volumeIsMultiZone { + if err := gceCS.validateMultiZoneDisk(volumeID, disk); err != nil { + return nil, err, disk + } + } + readWrite := "READ_WRITE" if readOnly { readWrite = "READ_ONLY" @@ -634,6 +1032,9 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con klog.V(4).Infof("ControllerPublishVolume succeeded for disk %v to instance %v, already attached.", volKey, nodeID) return pubVolResp, nil, disk } + if err := gceCS.updateAccessModeIfNecessary(ctx, volKey, disk, readOnly); err != nil { + return nil, common.LoggedError("Failed to update access mode: ", err), disk + } instanceZone, instanceName, err = common.NodeIDToZoneAndName(nodeID) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()), disk @@ -650,7 +1051,7 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con return nil, common.LoggedError("Failed to Attach: ", err), disk } - err = gceCS.CloudProvider.WaitForAttach(ctx, project, volKey, instanceZone, instanceName) + err = gceCS.CloudProvider.WaitForAttach(ctx, project, volKey, disk.GetPDType(), instanceZone, instanceName) if err != nil { return nil, common.LoggedError("Errored during WaitForAttach: ", err), disk } @@ -663,8 +1064,9 @@ func (gceCS *GCEControllerServer) ControllerUnpublishVolume(ctx context.Context, var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("ControllerUnpublishVolume", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("ControllerUnpublishVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() _, _, err = gceCS.validateControllerUnpublishVolumeRequest(ctx, req) if err != nil { @@ -677,7 +1079,7 @@ func (gceCS *GCEControllerServer) ControllerUnpublishVolume(ctx context.Context, return nil, status.Errorf(gceCS.errorBackoff.code(backoffId), "ControllerUnpublish not permitted on node %q due to backoff condition", req.NodeId) } resp, err, disk := gceCS.executeControllerUnpublishVolume(ctx, req) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(disk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) if err != nil { klog.Infof("For node %s adding backoff due to error for volume %s: %v", req.NodeId, req.VolumeId, err) gceCS.errorBackoff.next(backoffId, common.CodeForError(err)) @@ -716,6 +1118,16 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C volumeID := req.GetVolumeId() nodeID := req.GetNodeId() + + instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()), nil + } + + if gceCS.multiZoneVolumeHandleConfig.Enable && isMultiZoneVolKey(volKey) { + volKey = convertMultiZoneVolKeyToZoned(volKey, instanceZone) + } + project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { @@ -733,10 +1145,6 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C } defer gceCS.volumeLocks.Release(lockingVolumeID) diskToUnpublish, _ := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) - instanceZone, instanceName, err := common.NodeIDToZoneAndName(nodeID) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()), diskToUnpublish - } instance, err := gceCS.CloudProvider.GetInstanceOrError(ctx, instanceZone, instanceName) if err != nil { if gce.IsGCENotFoundError(err) { @@ -768,12 +1176,21 @@ func (gceCS *GCEControllerServer) executeControllerUnpublishVolume(ctx context.C return &csi.ControllerUnpublishVolumeResponse{}, nil, diskToUnpublish } +func (gceCS *GCEControllerServer) parameterProcessor() *common.ParameterProcessor { + return &common.ParameterProcessor{ + DriverName: gceCS.Driver.name, + EnableStoragePools: gceCS.enableStoragePools, + EnableMultiZone: gceCS.multiZoneVolumeHandleConfig.Enable, + } +} + func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("ValidateVolumeCapabilities", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("ValidateVolumeCapabilities", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() if req.GetVolumeCapabilities() == nil || len(req.GetVolumeCapabilities()) == 0 { return nil, status.Error(codes.InvalidArgument, "Volume Capabilities must be provided") @@ -786,6 +1203,7 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context if err != nil { return nil, status.Errorf(codes.InvalidArgument, "Volume ID is invalid: %v", err.Error()) } + project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) if err != nil { if gce.IsGCENotFoundError(err) { @@ -800,7 +1218,7 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context defer gceCS.volumeLocks.Release(volumeID) disk, err := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(disk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) if err != nil { if gce.IsGCENotFoundError(err) { return nil, status.Errorf(codes.NotFound, "Could not find disk %v: %v", volKey.Name, err.Error()) @@ -814,7 +1232,7 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context } // Validate the disk parameters match the disk we GET - params, err := common.ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.name, gceCS.Driver.extraVolumeLabels) + params, err := gceCS.parameterProcessor().ExtractAndDefaultParameters(req.GetParameters(), gceCS.Driver.extraVolumeLabels, gceCS.Driver.extraTags) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error()) } @@ -848,6 +1266,22 @@ func generateFailedValidationMessage(format string, a ...interface{}) *csi.Valid } } +func (gceCS *GCEControllerServer) listVolumeEntries(ctx context.Context) ([]*csi.ListVolumesResponse_Entry, error) { + diskList, _, err := gceCS.CloudProvider.ListDisks(ctx, gceCS.listVolumesConfig.listDisksFields()) + if err != nil { + return nil, err + } + + var instanceList []*compute.Instance = nil + if gceCS.listVolumesConfig.UseInstancesAPIForPublishedNodes { + instanceList, _, err = gceCS.CloudProvider.ListInstances(ctx, listInstancesFields) + if err != nil { + return nil, err + } + } + return gceCS.disksAndInstancesToVolumeEntries(diskList, instanceList), nil +} + func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { // https://cloud.google.com/compute/docs/reference/beta/disks/list if req.MaxEntries < 0 { @@ -855,20 +1289,21 @@ func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.List "ListVolumes got max entries request %v. GCE only supports values >0", req.MaxEntries) } - offset := 0 + offsetLow := 0 var ok bool if req.StartingToken == "" { - diskList, _, err := gceCS.CloudProvider.ListDisks(ctx) + volumeEntries, err := gceCS.listVolumeEntries(ctx) if err != nil { if gce.IsGCEInvalidError(err) { return nil, status.Errorf(codes.Aborted, "ListVolumes error with invalid request: %v", err.Error()) } - return nil, common.LoggedError("Failed to list disk: ", err) + return nil, common.LoggedError("Failed to list volumes: ", err) } - gceCS.disks = diskList - gceCS.seen = map[string]int{} + + gceCS.volumeEntries = volumeEntries + gceCS.volumeEntriesSeen = map[string]int{} } else { - offset, ok = gceCS.seen[req.StartingToken] + offsetLow, ok = gceCS.volumeEntriesSeen[req.StartingToken] if !ok { return nil, status.Errorf(codes.Aborted, "ListVolumes error with invalid startingToken: %s", req.StartingToken) } @@ -879,43 +1314,109 @@ func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.List maxEntries = maxListVolumesResponseEntries } - entries := []*csi.ListVolumesResponse_Entry{} - for i := 0; i+offset < len(gceCS.disks) && i < maxEntries; i++ { - d := gceCS.disks[i+offset] - diskRsrc, err := getResourceId(d.SelfLink) + nextToken := "" + offsetHigh := offsetLow + maxEntries + if offsetHigh < len(gceCS.volumeEntries) { + nextToken = string(uuid.NewUUID()) + gceCS.volumeEntriesSeen[nextToken] = offsetHigh + } else { + offsetHigh = len(gceCS.volumeEntries) + } + + return &csi.ListVolumesResponse{ + Entries: gceCS.volumeEntries[offsetLow:offsetHigh], + NextToken: nextToken, + }, nil +} + +// isMultiZoneDisk returns the multi-zone volumeId of a disk if it is +// "multi-zone", otherwise returns an empty string +// The second parameter indiciates if it is a "multi-zone" disk +func isMultiZoneDisk(diskRsrc string, diskLabels map[string]string) (string, bool) { + isMultiZoneDisk := false + for l := range diskLabels { + if l == common.MultiZoneLabel { + isMultiZoneDisk = true + } + } + if !isMultiZoneDisk { + return "", false + } + + multiZoneVolumeId, err := common.VolumeIdAsMultiZone(diskRsrc) + if err != nil { + klog.Warningf("Error converting multi-zone volume handle for disk %s, skipped: %v", diskRsrc, err) + return "", false + } + return multiZoneVolumeId, true +} + +// disksAndInstancesToVolumeEntries converts a list of disks and instances to a list +// of CSI ListVolumeResponse entries. +// It appends "multi-zone" volumeHandles at the end. These are volumeHandles which +// map to multiple volumeHandles in different zones +func (gceCS *GCEControllerServer) disksAndInstancesToVolumeEntries(disks []*compute.Disk, instances []*compute.Instance) []*csi.ListVolumesResponse_Entry { + nodesByVolumeId := map[string][]string{} + multiZoneVolumeIdsByVolumeId := map[string]string{} + for _, d := range disks { + volumeId, err := getResourceId(d.SelfLink) if err != nil { klog.Warningf("Bad ListVolumes disk resource %s, skipped: %v (%+v)", d.SelfLink, err, d) continue } - users := []string{} + + instanceIds := make([]string, len(d.Users)) for _, u := range d.Users { - rsrc, err := getResourceId(u) + instanceId, err := getResourceId(u) if err != nil { klog.Warningf("Bad ListVolumes user %s, skipped: %v", u, err) } else { - users = append(users, rsrc) + instanceIds = append(instanceIds, instanceId) + } + } + + nodesByVolumeId[volumeId] = instanceIds + + if gceCS.multiZoneVolumeHandleConfig.Enable { + if multiZoneVolumeId, isMultiZone := isMultiZoneDisk(volumeId, d.Labels); isMultiZone { + multiZoneVolumeIdsByVolumeId[volumeId] = multiZoneVolumeId + nodesByVolumeId[multiZoneVolumeId] = append(nodesByVolumeId[multiZoneVolumeId], instanceIds...) } } + } + + entries := []*csi.ListVolumesResponse_Entry{} + for _, instance := range instances { + instanceId, err := getResourceId(instance.SelfLink) + if err != nil { + klog.Warningf("Bad ListVolumes instance resource %s, skipped: %v (%+v)", instance.SelfLink, err, instance) + continue + } + for _, disk := range instance.Disks { + volumeId, err := getResourceId(disk.Source) + if err != nil { + klog.Warningf("Bad ListVolumes instance disk source %s, skipped: %v (%+v)", disk.Source, err, instance) + continue + } + + nodesByVolumeId[volumeId] = append(nodesByVolumeId[volumeId], instanceId) + if multiZoneVolumeId, isMultiZone := multiZoneVolumeIdsByVolumeId[volumeId]; isMultiZone { + nodesByVolumeId[multiZoneVolumeId] = append(nodesByVolumeId[multiZoneVolumeId], instanceId) + } + } + } + + for volumeId, nodeIds := range nodesByVolumeId { entries = append(entries, &csi.ListVolumesResponse_Entry{ Volume: &csi.Volume{ - VolumeId: diskRsrc, + VolumeId: volumeId, }, Status: &csi.ListVolumesResponse_VolumeStatus{ - PublishedNodeIds: users, + PublishedNodeIds: nodeIds, }, }) } - - nextToken := "" - if len(entries)+offset < len(gceCS.disks) { - nextToken = string(uuid.NewUUID()) - gceCS.seen[nextToken] = len(entries) + offset - } - - return &csi.ListVolumesResponse{ - Entries: entries, - NextToken: nextToken, - }, nil + return entries } func (gceCS *GCEControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { @@ -935,8 +1436,9 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("CreateSnapshot", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("CreateSnapshot", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() // Validate arguments volumeID := req.GetSourceVolumeId() @@ -951,6 +1453,11 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C return nil, status.Errorf(codes.InvalidArgument, "CreateSnapshot Volume ID is invalid: %v", err.Error()) } + volumeIsMultiZone := isMultiZoneVolKey(volKey) + if gceCS.multiZoneVolumeHandleConfig.Enable && volumeIsMultiZone { + return nil, status.Errorf(codes.InvalidArgument, "CreateSnapshot for volume %v failed. Snapshots are not supported with the multi-zone PV volumeHandle feature", volumeID) + } + if acquired := gceCS.volumeLocks.TryAcquire(volumeID); !acquired { return nil, status.Errorf(codes.Aborted, common.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -958,7 +1465,7 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C // Check if volume exists disk, err := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(disk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(disk) if err != nil { if gce.IsGCENotFoundError(err) { return nil, status.Errorf(codes.NotFound, "CreateSnapshot could not find disk %v: %v", volKey.String(), err.Error()) @@ -966,7 +1473,7 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C return nil, common.LoggedError("CreateSnapshot, failed to getDisk: ", err) } - snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters(), gceCS.Driver.name) + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters(), gceCS.Driver.name, gceCS.Driver.extraTags) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "Invalid snapshot parameters: %v", err.Error()) } @@ -996,6 +1503,7 @@ func (gceCS *GCEControllerServer) createPDSnapshot(ctx context.Context, project if err != nil { return nil, status.Errorf(codes.InvalidArgument, "Invalid volume key: %v", volKey) } + // Check if PD snapshot already exists var snapshot *compute.Snapshot snapshot, err = gceCS.CloudProvider.GetSnapshot(ctx, project, snapshotName) @@ -1187,8 +1695,9 @@ func (gceCS *GCEControllerServer) DeleteSnapshot(ctx context.Context, req *csi.D var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("DeleteSnapshot", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("DeleteSnapshot", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() // Validate arguments snapshotID := req.GetSnapshotId() @@ -1274,11 +1783,13 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li } func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + var err error diskTypeForMetric := metrics.DefaultDiskTypeForMetric enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute + enableStoragePools := metrics.DefaultEnableStoragePools defer func() { - gceCS.Metrics.RecordOperationErrorMetrics("ControllerExpandVolume", err, diskTypeForMetric, enableConfidentialCompute) + gceCS.Metrics.RecordOperationErrorMetrics("ControllerExpandVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools) }() volumeID := req.GetVolumeId() if len(volumeID) == 0 { @@ -1295,15 +1806,23 @@ func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, re return nil, status.Errorf(codes.InvalidArgument, "ControllerExpandVolume Volume ID is invalid: %v", err.Error()) } project, volKey, err = gceCS.CloudProvider.RepairUnderspecifiedVolumeKey(ctx, project, volKey) + if err != nil { if gce.IsGCENotFoundError(err) { return nil, status.Errorf(codes.NotFound, "ControllerExpandVolume could not find volume with ID %v: %v", volumeID, err.Error()) } return nil, common.LoggedError("ControllerExpandVolume error repairing underspecified volume key: ", err) } + + volumeIsMultiZone := isMultiZoneVolKey(volKey) + if gceCS.multiZoneVolumeHandleConfig.Enable && volumeIsMultiZone { + return nil, status.Errorf(codes.InvalidArgument, "ControllerExpandVolume is not supported with the multi-zone PVC volumeHandle feature. Please re-create the volume %v from source if you want a larger size", volumeID) + } + sourceDisk, err := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionV1) - diskTypeForMetric, enableConfidentialCompute = metrics.GetMetricParameters(sourceDisk) + diskTypeForMetric, enableConfidentialCompute, enableStoragePools = metrics.GetMetricParameters(sourceDisk) resizedGb, err := gceCS.CloudProvider.ResizeDisk(ctx, project, volKey, reqBytes) + if err != nil { return nil, common.LoggedError("ControllerExpandVolume failed to resize disk: ", err) } @@ -1391,6 +1910,7 @@ func (gceCS *GCEControllerServer) getSnapshotByID(ctx context.Context, snapshotI // return empty list if no snapshot is found return &csi.ListSnapshotsResponse{}, nil } + return nil, common.LoggedError("Failed to get image snapshot: ", err) } e, err := generateDiskImageEntry(image) if err != nil { @@ -1550,7 +2070,7 @@ func prependZone(zone string, zones []string) []string { return newZones } -func pickZonesFromTopology(top *csi.TopologyRequirement, numZones int, locationTopReq *locationRequirements) ([]string, error) { +func pickZonesFromTopology(top *csi.TopologyRequirement, numZones int, locationTopReq *locationRequirements, fallbackRequisiteZones []string) ([]string, error) { reqZones, err := getZonesFromTopology(top.GetRequisite()) if err != nil { return nil, fmt.Errorf("could not get zones from requisite topology: %w", err) @@ -1595,40 +2115,52 @@ func pickZonesFromTopology(top *csi.TopologyRequirement, numZones int, locationT if numZones <= len(prefZones) { return prefZones[0:numZones], nil - } else { - zones := sets.String{} - // Add all preferred zones into zones - zones.Insert(prefZones...) - remainingNumZones := numZones - len(prefZones) - // Take all of the remaining zones from requisite zones - reqSet := sets.NewString(reqZones...) - prefSet := sets.NewString(prefZones...) - remainingZones := reqSet.Difference(prefSet) - - if remainingZones.Len() < remainingNumZones { + } + + remainingNumZones := numZones - len(prefZones) + // Take all of the remaining zones from requisite zones + reqSet := sets.NewString(reqZones...) + prefSet := sets.NewString(prefZones...) + remainingZones := reqSet.Difference(prefSet) + + if remainingZones.Len() < remainingNumZones { + fallbackSet := sets.NewString(fallbackRequisiteZones...) + remainingFallbackZones := fallbackSet.Difference(prefSet) + if remainingFallbackZones.Len() >= remainingNumZones { + remainingZones = remainingFallbackZones + } else { return nil, fmt.Errorf("need %v zones from topology, only got %v unique zones", numZones, reqSet.Union(prefSet).Len()) } - // Add the remaining number of zones into the set - nSlice, err := pickRandAndConsecutive(remainingZones.List(), remainingNumZones) - if err != nil { - return nil, err - } - zones.Insert(nSlice...) - return zones.List(), nil } + + allZones := prefSet.Union(remainingZones).List() + sort.Strings(allZones) + var shiftIndex int + if len(prefZones) == 0 { + // Random shift the requisite zones, since there is no preferred start. + shiftIndex = rand.Intn(len(allZones)) + } else { + shiftIndex = slices.Index(allZones, prefZones[0]) + } + shiftedZones := append(allZones[shiftIndex:], allZones[:shiftIndex]...) + sortedShiftedReqZones := slices.Filter(nil, shiftedZones, func(v string) bool { return !prefSet.Has(v) }) + zones := make([]string, 0, numZones) + zones = append(zones, prefZones...) + zones = append(zones, sortedShiftedReqZones...) + return zones[:numZones], nil } func getZonesFromTopology(topList []*csi.Topology) ([]string, error) { zones := []string{} for _, top := range topList { if top.GetSegments() == nil { - return nil, fmt.Errorf("preferred topologies specified but no segments") + return nil, fmt.Errorf("topologies specified but no segments") } // GCE PD cloud provider Create has no restrictions so just create in top preferred zone zone, err := getZoneFromSegment(top.GetSegments()) if err != nil { - return nil, fmt.Errorf("could not get zone from preferred topology: %w", err) + return nil, fmt.Errorf("could not get zone from topology: %w", err) } zones = append(zones, zone) } @@ -1651,11 +2183,11 @@ func getZoneFromSegment(seg map[string]string) (string, error) { return zone, nil } -func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.TopologyRequirement, numZones int, locationTopReq *locationRequirements) ([]string, error) { +func (gceCS *GCEControllerServer) pickZones(ctx context.Context, top *csi.TopologyRequirement, numZones int, locationTopReq *locationRequirements) ([]string, error) { var zones []string var err error if top != nil { - zones, err = pickZonesFromTopology(top, numZones, locationTopReq) + zones, err = pickZonesFromTopology(top, numZones, locationTopReq, gceCS.fallbackRequisiteZones) if err != nil { return nil, fmt.Errorf("failed to pick zones from topology: %w", err) } @@ -1669,7 +2201,7 @@ func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.Topolog existingZones = []string{locationTopReq.srcVolZone} } // If topology is nil, then the Immediate binding mode was used without setting allowedTopologies in the storageclass. - zones, err = getDefaultZonesInRegion(ctx, gceCS, existingZones, numZones) + zones, err = getNumDefaultZonesInRegion(ctx, gceCS, existingZones, numZones) if err != nil { return nil, fmt.Errorf("failed to get default %v zones in region: %w", numZones, err) } @@ -1679,16 +2211,24 @@ func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.Topolog return zones, nil } -func getDefaultZonesInRegion(ctx context.Context, gceCS *GCEControllerServer, existingZones []string, numZones int) ([]string, error) { +func getDefaultZonesInRegion(ctx context.Context, gceCS *GCEControllerServer, existingZones []string) ([]string, error) { region, err := common.GetRegionFromZones(existingZones) if err != nil { return nil, fmt.Errorf("failed to get region from zones: %w", err) } - needToGet := numZones - len(existingZones) totZones, err := gceCS.CloudProvider.ListZones(ctx, region) if err != nil { return nil, fmt.Errorf("failed to list zones from cloud provider: %w", err) } + return totZones, nil +} + +func getNumDefaultZonesInRegion(ctx context.Context, gceCS *GCEControllerServer, existingZones []string, numZones int) ([]string, error) { + needToGet := numZones - len(existingZones) + totZones, err := getDefaultZonesInRegion(ctx, gceCS, existingZones) + if err != nil { + return nil, err + } remainingZones := sets.NewString(totZones...).Difference(sets.NewString(existingZones...)) l := remainingZones.List() if len(l) < needToGet { @@ -1730,12 +2270,7 @@ func extractVolumeContext(context map[string]string) (*PDCSIContext, error) { return info, nil } -func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string, params common.DiskParameters) (*csi.CreateVolumeResponse, error) { - volumeId, err := getResourceId(disk.GetSelfLink()) - if err != nil { - return nil, fmt.Errorf("cannot get volume id from %s: %w", disk.GetSelfLink(), err) - } - +func generateCreateVolumeResponseWithVolumeId(disk *gce.CloudDisk, zones []string, params common.DiskParameters, volumeId string) *csi.CreateVolumeResponse { tops := []*csi.Topology{} for _, zone := range zones { tops = append(tops, &csi.Topology{ @@ -1785,7 +2320,7 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string, params co } createResp.Volume.ContentSource = contentSource } - return createResp, nil + return createResp } func getResourceId(resourceLink string) (string, error) { @@ -1817,7 +2352,7 @@ func getResourceId(resourceLink string) (string, error) { return strings.Join(elts[3:], "/"), nil } -func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) (*gce.CloudDisk, error) { +func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) (*gce.CloudDisk, error) { project := cloudProvider.GetDefaultProject() region, err := common.GetRegionFromZones(zones) if err != nil { @@ -1830,7 +2365,7 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name fullyQualifiedReplicaZones, cloudProvider.GetReplicaZoneURI(project, replicaZone)) } - err = cloudProvider.InsertDisk(ctx, project, meta.RegionalKey(name, region), params, capBytes, capacityRange, fullyQualifiedReplicaZones, snapshotID, volumeContentSourceVolumeID, multiWriter) + err = cloudProvider.InsertDisk(ctx, project, meta.RegionalKey(name, region), params, capBytes, capacityRange, fullyQualifiedReplicaZones, snapshotID, volumeContentSourceVolumeID, multiWriter, accessMode) if err != nil { return nil, fmt.Errorf("failed to insert regional disk: %w", err) } @@ -1839,20 +2374,21 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name if multiWriter { gceAPIVersion = gce.GCEAPIVersionBeta } + // failed to GetDisk, however the Disk may already be created, the error code should be non-Final disk, err := cloudProvider.GetDisk(ctx, project, meta.RegionalKey(name, region), gceAPIVersion) if err != nil { - return nil, fmt.Errorf("failed to get disk after creating regional disk: %w", err) + return nil, common.NewTemporaryError(codes.Unavailable, fmt.Errorf("failed to get disk after creating regional disk: %w", err)) } return disk, nil } -func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) (*gce.CloudDisk, error) { +func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) (*gce.CloudDisk, error) { project := cloudProvider.GetDefaultProject() if len(zones) != 1 { return nil, fmt.Errorf("got wrong number of zones for zonal create volume: %v", len(zones)) } diskZone := zones[0] - err := cloudProvider.InsertDisk(ctx, project, meta.ZonalKey(name, diskZone), params, capBytes, capacityRange, nil, snapshotID, volumeContentSourceVolumeID, multiWriter) + err := cloudProvider.InsertDisk(ctx, project, meta.ZonalKey(name, diskZone), params, capBytes, capacityRange, nil, snapshotID, volumeContentSourceVolumeID, multiWriter, accessMode) if err != nil { return nil, fmt.Errorf("failed to insert zonal disk: %w", err) } @@ -1861,9 +2397,10 @@ func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, nam if multiWriter { gceAPIVersion = gce.GCEAPIVersionBeta } + // failed to GetDisk, however the Disk may already be created, the error code should be non-Final disk, err := cloudProvider.GetDisk(ctx, project, meta.ZonalKey(name, diskZone), gceAPIVersion) if err != nil { - return nil, err + return nil, common.NewTemporaryError(codes.Unavailable, fmt.Errorf("failed to get disk after creating zonal disk: %w", err)) } return disk, nil } @@ -1902,7 +2439,7 @@ func (b *csiErrorBackoff) code(id csiErrorBackoffId) codes.Code { // If we haven't recorded a code, return unavailable, which signals a problem with the driver // (ie, next() wasn't called correctly). klog.Errorf("using default code for %s", id) - return codes.Unavailable + return codes.Internal } func (b *csiErrorBackoff) next(id csiErrorBackoffId, code codes.Code) { diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index 8ea8065b..12d6430c 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math/rand" + "net/http" "reflect" "sort" "strconv" @@ -25,14 +26,19 @@ import ( "time" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/protobuf/types/known/timestamppb" compute "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" clock "k8s.io/utils/clock/testing" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/flowcontrol" + "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" csi "github.com/container-storage-interface/spec/lib/go/csi" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" @@ -66,6 +72,7 @@ var ( testVolumeID = fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, name) underspecifiedVolumeID = fmt.Sprintf("projects/UNSPECIFIED/zones/UNSPECIFIED/disks/%s", name) + multiZoneVolumeID = fmt.Sprintf("projects/%s/zones/multi-zone/disks/%s", project, name) region, _ = common.GetRegionFromZones([]string{zone}) testRegionalID = fmt.Sprintf("projects/%s/regions/%s/disks/%s", project, region, name) @@ -184,6 +191,36 @@ func TestCreateSnapshotArguments(t *testing.T) { }, expErrCode: codes.InvalidArgument, }, + { + name: "success with resource-tags parameter", + req: &csi.CreateSnapshotRequest{ + Name: name, + SourceVolumeId: testVolumeID, + Parameters: map[string]string{"resource-tags": "parent1/key1/value1,parent2/key2/value2"}, + }, + seedDisks: []*gce.CloudDisk{ + createZonalCloudDisk(name), + }, + expSnapshot: &csi.Snapshot{ + SnapshotId: testSnapshotID, + SourceVolumeId: testVolumeID, + CreationTime: tp, + SizeBytes: common.GbToBytes(gce.DiskSizeGb), + ReadyToUse: false, + }, + }, + { + name: "fail with malformed resource-tags parameter", + req: &csi.CreateSnapshotRequest{ + Name: name, + SourceVolumeId: testVolumeID, + Parameters: map[string]string{"resource-tags": "parent1/key1/value1,parent2/key2/"}, + }, + seedDisks: []*gce.CloudDisk{ + createZonalCloudDisk(name), + }, + expErrCode: codes.InvalidArgument, + }, } for _, tc := range testCases { @@ -221,6 +258,77 @@ func TestCreateSnapshotArguments(t *testing.T) { } } +func TestUnsupportedMultiZoneCreateSnapshot(t *testing.T) { + testCase := struct { + name string + req *csi.CreateSnapshotRequest + expErrCode codes.Code + }{ + name: "failed create snapshot for multi-zone PV", // Example values + req: &csi.CreateSnapshotRequest{ + Name: name, + SourceVolumeId: multiZoneVolumeID, + }, + expErrCode: codes.InvalidArgument, + } + + t.Logf("test case: %s", testCase.name) + + gceDriver := initGCEDriver(t, nil) + gceDriver.cs.multiZoneVolumeHandleConfig = MultiZoneVolumeHandleConfig{ + Enable: true, + } + + // Start Test + _, err := gceDriver.cs.CreateSnapshot(context.Background(), testCase.req) + if err != nil { + serverError, ok := status.FromError(err) + if !ok { + t.Fatalf("Could not get error status code from err: %v", serverError) + } + if serverError.Code() != testCase.expErrCode { + t.Fatalf("Expected error code: %v, got: %v. err : %v", testCase.expErrCode, serverError.Code(), err) + } + } else { + t.Fatalf("Expected error: %v, got no error", testCase.expErrCode) + } +} + +func TestUnsupportedMultiZoneControllerExpandVolume(t *testing.T) { + testCase := struct { + name string + req *csi.ControllerExpandVolumeRequest + expErrCode codes.Code + }{ + name: "failed create snapshot for multi-zone PV", // Example values + req: &csi.ControllerExpandVolumeRequest{ + VolumeId: multiZoneVolumeID, + }, + expErrCode: codes.InvalidArgument, + } + + t.Logf("test case: %s", testCase.name) + + gceDriver := initGCEDriver(t, nil) + gceDriver.cs.multiZoneVolumeHandleConfig = MultiZoneVolumeHandleConfig{ + Enable: true, + } + + // Start Test + _, err := gceDriver.cs.ControllerExpandVolume(context.Background(), testCase.req) + if err != nil { + serverError, ok := status.FromError(err) + if !ok { + t.Fatalf("Could not get error status code from err: %v", serverError) + } + if serverError.Code() != testCase.expErrCode { + t.Fatalf("Expected error code: %v, got: %v. err : %v", testCase.expErrCode, serverError.Code(), err) + } + } else { + t.Fatalf("Expected error: %v, got no error", testCase.expErrCode) + } +} + func TestDeleteSnapshot(t *testing.T) { testCases := []struct { name string @@ -293,6 +401,15 @@ func TestListSnapshotsArguments(t *testing.T) { numImages: 2, expectedCount: 1, }, + { + name: "valid image", + req: &csi.ListSnapshotsRequest{ + SnapshotId: testImageID + "0", + }, + numSnapshots: 3, + numImages: 2, + expectedCount: 1, + }, { name: "invalid id", req: &csi.ListSnapshotsRequest{ @@ -300,6 +417,29 @@ func TestListSnapshotsArguments(t *testing.T) { }, expectedCount: 0, }, + { + name: "invalid image id", + req: &csi.ListSnapshotsRequest{ + SnapshotId: testImageID + "/foo", + }, + expectedCount: 0, + }, + { + name: "invalid snapshot name", + req: &csi.ListSnapshotsRequest{ + SnapshotId: testSnapshotID + "-invalid-snapshot-", + }, + expectedCount: 0, + expErrCode: codes.InvalidArgument, + }, + { + name: "invalid image name", + req: &csi.ListSnapshotsRequest{ + SnapshotId: testImageID + "-invalid-image-", + }, + expectedCount: 0, + expErrCode: codes.InvalidArgument, + }, { name: "no id", req: &csi.ListSnapshotsRequest{ @@ -411,10 +551,11 @@ func TestListSnapshotsArguments(t *testing.T) { func TestCreateVolumeArguments(t *testing.T) { testCases := []struct { - name string - req *csi.CreateVolumeRequest - expVol *csi.Volume - expErrCode codes.Code + name string + req *csi.CreateVolumeRequest + enableStoragePools bool + expVol *csi.Volume + expErrCode codes.Code }{ { name: "success default", @@ -845,6 +986,109 @@ func TestCreateVolumeArguments(t *testing.T) { }, expErrCode: codes.InvalidArgument, }, + { + name: "success with storage pools parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"storage-pools": "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", "type": "hyperdisk-balanced"}, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + enableStoragePools: true, + expVol: &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: "projects/test-project/zones/us-central1-a/disks/test-name", + VolumeContext: nil, + AccessibleTopology: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + { + name: "fail with storage pools parameter, enableStoragePools is false", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"storage-pools": "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", "type": "hyperdisk-balanced"}, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + enableStoragePools: false, + expErrCode: codes.InvalidArgument, + }, + { + name: "fail with invalid storage pools parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"storage-pools": "zones/us-central1-a/storagePools/storagePool-1", "type": "hyperdisk-balanced"}, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + enableStoragePools: true, + expErrCode: codes.InvalidArgument, + }, + { + name: "success with resource-tags parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"resource-tags": "parent1/key1/value1,parent2/key2/value2"}, + }, + expVol: &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: testVolumeID, + VolumeContext: nil, + AccessibleTopology: stdTopology, + }, + }, + { + name: "fail with malformed resource-tags parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"resource-tags": "parent1/key1/value1,parent2/key2/"}, + }, + expErrCode: codes.InvalidArgument, + }, } // Run test cases @@ -852,7 +1096,7 @@ func TestCreateVolumeArguments(t *testing.T) { t.Logf("test case: %s", tc.name) // Setup new driver each time so no interference gceDriver := initGCEDriver(t, nil) - + gceDriver.cs.enableStoragePools = tc.enableStoragePools // Start Test resp, err := gceDriver.cs.CreateVolume(context.Background(), tc.req) if err != nil { @@ -890,44 +1134,621 @@ func TestCreateVolumeArguments(t *testing.T) { } } -func TestListVolumePagination(t *testing.T) { +func TestMultiZoneVolumeCreation(t *testing.T) { testCases := []struct { - name string - diskCount int - maxEntries int32 - expectedEntries []int + name string + req *csi.CreateVolumeRequest + enableStoragePools bool + fallbackZones []string + expZones []string + expErrCode codes.Code }{ { - name: "no pagination (implicit)", - diskCount: 325, - expectedEntries: []int{325}, - }, - { - name: "no pagination (explicit)", - diskCount: 2500, - maxEntries: 2500, - expectedEntries: []int{2500}, - }, - { - name: "pagination (implicit)", - diskCount: 1327, - expectedEntries: []int{500, 500, 327}, + name: "success single ROX multi-zone disk", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: testSnapshotID, + }, + }, + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + expZones: []string{"us-central1-a"}, }, { - name: "pagination (explicit)", - diskCount: 723, - maxEntries: 200, - expectedEntries: []int{200, 200, 200, 123}, + name: "single ROX multi-zone disk empty topology fallback zones", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: testSnapshotID, + }, + }, + }, + AccessibilityRequirements: &csi.TopologyRequirement{}, + }, + fallbackZones: []string{zone, secondZone}, + expZones: []string{zone, secondZone}, }, { - name: "pagination (explicit)", - diskCount: 3253, - maxEntries: 1000, - expectedEntries: []int{1000, 1000, 1000, 253}, - }, - } - - for _, tc := range testCases { + name: "success triple ROX multi-zone disk", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: testSnapshotID, + }, + }, + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + }, + }, + expZones: []string{"us-central1-a", "us-central1-b", "us-central1-c"}, + }, + { + name: "success triple rwo multi-zone disk", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + }, + }, + expZones: []string{"us-central1-a", "us-central1-b", "us-central1-c"}, + }, + { + name: "err single ROX multi-zone no topology", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: testSnapshotID, + }, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, + { + name: "err rwo access mode", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: testSnapshotID, + }, + }, + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, + { + name: "err no content source", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, + { + name: "err cloning not supported", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + }, + }, + }, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: testVolumeID, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, + } + + // Run test cases + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + // Setup new driver each time so no interference + fcp, err := gce.CreateFakeCloudProvider(project, zone, nil) + if err != nil { + t.Fatalf("Failed to create fake cloud provider: %v", err) + } + // Setup new driver each time so no interference + gceDriver := initGCEDriverWithCloudProvider(t, fcp) + gceDriver.cs.multiZoneVolumeHandleConfig.DiskTypes = []string{"hyperdisk-ml"} + gceDriver.cs.multiZoneVolumeHandleConfig.Enable = true + gceDriver.cs.fallbackRequisiteZones = tc.fallbackZones + + if tc.req.VolumeContentSource.GetType() != nil { + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil, gceDriver.name, nil) + if err != nil { + t.Errorf("Got error extracting snapshot parameters: %v", err) + } + if snapshotParams.SnapshotType == common.DiskSnapshotType { + fcp.CreateSnapshot(context.Background(), project, meta.ZonalKey(name, common.MultiZoneValue), name, snapshotParams) + } else { + t.Fatalf("No volume source mentioned in snapshot parameters %v", snapshotParams) + } + } + + // Start Test + resp, err := gceDriver.cs.CreateVolume(context.Background(), tc.req) + if err != nil { + serverError, ok := status.FromError(err) + if !ok { + t.Fatalf("Could not get error status code from err: %v", serverError) + } + if serverError.Code() != tc.expErrCode { + t.Fatalf("Expected error code: %v, got: %v. err : %v", tc.expErrCode, serverError.Code(), err) + } + continue + } + if tc.expErrCode != codes.OK { + t.Fatalf("Expected error: %v, got no error", tc.expErrCode) + } + + topologies := make([]*csi.Topology, 0, len(tc.expZones)) + for _, zone := range tc.expZones { + topologies = append(topologies, &csi.Topology{ + Segments: map[string]string{common.TopologyKeyZone: zone}, + }) + } + + expVol := &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: fmt.Sprintf("projects/%s/zones/multi-zone/disks/%s", project, name), + VolumeContext: nil, + AccessibleTopology: topologies, + ContentSource: tc.req.VolumeContentSource, + } + + // Make sure responses match + vol := resp.GetVolume() + if vol == nil { + // If one is nil but not both + t.Fatalf("Expected volume %v, got nil volume", expVol) + } + + klog.Warningf("Got accessible topology: %v", vol.GetAccessibleTopology()) + + sortTopologies := func(t1, t2 *csi.Topology) bool { + return t1.Segments[common.TopologyKeyZone] < t2.Segments[common.TopologyKeyZone] + } + if diff := cmp.Diff(expVol, vol, cmpopts.SortSlices(sortTopologies)); diff != "" { + t.Errorf("Accessible topologies mismatch (-want +got):\n%s", diff) + } + + for _, zone := range tc.expZones { + volumeKey := meta.ZonalKey(name, zone) + disk, err := fcp.GetDisk(context.Background(), project, volumeKey, gce.GCEAPIVersionBeta) + if err != nil { + t.Fatalf("Get Disk failed for created disk with error: %v", err) + } + if disk.GetLabels()[common.MultiZoneLabel] != "true" { + t.Fatalf("Expect %s disk to have %s label, got: %v", volumeKey, common.MultiZoneLabel, disk.GetLabels()) + } + } + } +} + +type FakeCloudProviderInsertDiskErr struct { + *gce.FakeCloudProvider + insertDiskErrors map[string]error +} + +func NewFakeCloudProviderInsertDiskErr(project, zone string) (*FakeCloudProviderInsertDiskErr, error) { + provider, err := gce.CreateFakeCloudProvider(project, zone, nil) + if err != nil { + return nil, err + } + return &FakeCloudProviderInsertDiskErr{ + FakeCloudProvider: provider, + insertDiskErrors: map[string]error{}, + }, nil +} + +func (cloud *FakeCloudProviderInsertDiskErr) AddDiskForErr(volKey *meta.Key, err error) { + cloud.insertDiskErrors[volKey.String()] = err +} + +func (cloud *FakeCloudProviderInsertDiskErr) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) error { + if err, ok := cloud.insertDiskErrors[volKey.String()]; ok { + return err + } + + return cloud.FakeCloudProvider.InsertDisk(ctx, project, volKey, params, capBytes, capacityRange, replicaZones, snapshotID, volumeContentSourceVolumeID, multiWriter, accessMode) +} + +func TestMultiZoneVolumeCreationErrHandling(t *testing.T) { + testCases := []struct { + name string + req *csi.CreateVolumeRequest + insertDiskErrs map[*meta.Key]error + expErrCode codes.Code + wantDisks []*meta.Key + }{ + { + name: "ResourceExhausted errors", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + insertDiskErrs: map[*meta.Key]error{ + meta.ZonalKey(name, "us-central1-b"): &googleapi.Error{Code: http.StatusTooManyRequests, Message: "Resource Exhausted"}, + }, + expErrCode: codes.ResourceExhausted, + wantDisks: []*meta.Key{ + meta.ZonalKey(name, "us-central1-a"), + meta.ZonalKey(name, "us-central1-c"), + }, + }, + { + name: "Unavailable errors", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + CapacityRange: stdCapRange, + VolumeCapabilities: []*csi.VolumeCapability{ + { + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{}, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + Parameters: map[string]string{ + common.ParameterKeyType: "hyperdisk-ml", + common.ParameterKeyEnableMultiZoneProvisioning: "true", + }, + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + insertDiskErrs: map[*meta.Key]error{ + meta.ZonalKey(name, "us-central1-b"): &googleapi.Error{Code: http.StatusGatewayTimeout, Message: "connection reset by peer"}, + meta.ZonalKey(name, "us-central1-c"): &googleapi.Error{Code: http.StatusTooManyRequests, Message: "Resource Exhausted"}, + }, + expErrCode: codes.Unavailable, + wantDisks: []*meta.Key{ + meta.ZonalKey(name, "us-central1-a"), + }, + }, + } + + // Run test cases + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + // Setup new driver each time so no interference + fcp, err := NewFakeCloudProviderInsertDiskErr(project, zone) + if err != nil { + t.Fatalf("Failed to create fake cloud provider: %v", err) + } + // Setup new driver each time so no interference + gceDriver := initGCEDriverWithCloudProvider(t, fcp) + gceDriver.cs.multiZoneVolumeHandleConfig.DiskTypes = []string{"hyperdisk-ml"} + gceDriver.cs.multiZoneVolumeHandleConfig.Enable = true + + for volKey, err := range tc.insertDiskErrs { + fcp.AddDiskForErr(volKey, err) + } + + // Start Test + _, err = gceDriver.cs.CreateVolume(context.Background(), tc.req) + + if err == nil { + t.Errorf("Expected error: %v, got no error", tc.expErrCode) + } + + serverError, ok := status.FromError(err) + if !ok { + t.Errorf("Could not get error status code from err: %v", serverError) + } + if serverError.Code() != tc.expErrCode { + t.Errorf("Expected error code: %v, got: %v. err : %v", tc.expErrCode, serverError.Code(), err) + } + + for _, volKey := range tc.wantDisks { + disk, err := fcp.GetDisk(context.Background(), project, volKey, gce.GCEAPIVersionV1) + if err != nil { + t.Errorf("Unexpected err fetching disk %v: %v", volKey, err) + } + if disk == nil { + t.Errorf("Expected disk for %v but got nil", volKey) + } + } + } +} + +func TestListVolumePagination(t *testing.T) { + testCases := []struct { + name string + diskCount int + maxEntries int32 + expectedEntries []int + }{ + { + name: "no pagination (implicit)", + diskCount: 325, + expectedEntries: []int{325}, + }, + { + name: "no pagination (explicit)", + diskCount: 2500, + maxEntries: 2500, + expectedEntries: []int{2500}, + }, + { + name: "pagination (implicit)", + diskCount: 1327, + expectedEntries: []int{500, 500, 327}, + }, + { + name: "pagination (explicit)", + diskCount: 723, + maxEntries: 200, + expectedEntries: []int{200, 200, 200, 123}, + }, + { + name: "pagination (explicit)", + diskCount: 3253, + maxEntries: 1000, + expectedEntries: []int{1000, 1000, 1000, 253}, + }, + } + + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Setup new driver each time so no interference var d []*gce.CloudDisk @@ -965,6 +1786,96 @@ func TestListVolumePagination(t *testing.T) { } } +func TestListAttachedVolumePagination(t *testing.T) { + testCases := []struct { + name string + diskCount int + maxEntries int32 + expectedEntries []int + }{ + { + name: "no pagination (implicit)", + diskCount: 325, + expectedEntries: []int{325}, + }, + { + name: "no pagination (explicit)", + diskCount: 2500, + maxEntries: 2500, + expectedEntries: []int{2500}, + }, + { + name: "pagination (implicit)", + diskCount: 1327, + expectedEntries: []int{500, 500, 327}, + }, + { + name: "pagination (explicit)", + diskCount: 723, + maxEntries: 200, + expectedEntries: []int{200, 200, 200, 123}, + }, + { + name: "pagination (explicit)", + diskCount: 3253, + maxEntries: 1000, + expectedEntries: []int{1000, 1000, 1000, 253}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Setup new driver each time so no interference + var d []*gce.CloudDisk + fakeCloudProvider, err := gce.CreateFakeCloudProvider(project, zone, d) + if err != nil { + t.Fatalf("Failed to create fake cloud provider: %v", err) + } + for i := 0; i < tc.diskCount; i++ { + // Create diskCount dummy instances, each with a dynamically attached disk + instanceName := fmt.Sprintf("instance-%v", i) + diskName := fmt.Sprintf("pvc-%v", i) + instance := compute.Instance{ + Name: name, + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s", project, zone, instanceName), + Disks: []*compute.AttachedDisk{ + { + DeviceName: diskName, + Source: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, diskName), + }, + }, + } + fakeCloudProvider.InsertInstance(&instance, zone, instanceName) + } + gceDriver := initGCEDriverWithCloudProvider(t, fakeCloudProvider) + // Use attached disks (instances.list) API + gceDriver.cs.listVolumesConfig.UseInstancesAPIForPublishedNodes = true + + tok := "" + for i, expectedEntry := range tc.expectedEntries { + lvr := &csi.ListVolumesRequest{ + MaxEntries: tc.maxEntries, + StartingToken: tok, + } + resp, err := gceDriver.cs.ListVolumes(context.TODO(), lvr) + if err != nil { + t.Fatalf("Got error %v", err) + return + } + + if len(resp.Entries) != expectedEntry { + t.Fatalf("Got %v entries, expected %v on call # %d", len(resp.Entries), expectedEntry, i+1) + } + + tok = resp.NextToken + } + if len(tok) != 0 { + t.Fatalf("Expected no more entries, but got NextToken in response: %s", tok) + } + }) + } +} + func TestListVolumeArgs(t *testing.T) { diskCount := 500 testCases := []struct { @@ -1016,13 +1927,211 @@ func TestListVolumeArgs(t *testing.T) { return } - if len(resp.Entries) != tc.expectedEntries { - t.Fatalf("Got %v entries, expected %v", len(resp.Entries), tc.expectedEntries) + if len(resp.Entries) != tc.expectedEntries { + t.Fatalf("Got %v entries, expected %v", len(resp.Entries), tc.expectedEntries) + } + }) + } +} + +func TestListVolumeResponse(t *testing.T) { + zone1 := "us-central1-a" + zone2 := "us-central1-b" + testCases := []struct { + name string + disks []compute.Disk + instances []compute.Instance + expectedEntries []*csi.ListVolumesResponse_Entry + }{ + { + name: "unattached disk", + disks: []compute.Disk{ + { + Name: "pv-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, "pv-1"), + }, + }, + expectedEntries: []*csi.ListVolumesResponse_Entry{ + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{}, + }, + }, + }, + }, + { + name: "attached disk", + disks: []compute.Disk{ + { + Name: "pv-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, "pv-1"), + }, + }, + instances: []compute.Instance{ + { + Name: "node-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s", project, zone, "node-1"), + Disks: []*compute.AttachedDisk{ + { + DeviceName: "pv-1", + Source: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, "pv-1"), + }, + }, + }, + }, + expectedEntries: []*csi.ListVolumesResponse_Entry{ + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone, "node-1")}, + }, + }, + }, + }, + { + name: "attached regional disk", + disks: []compute.Disk{ + { + Name: "pv-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/disks/%s", project, region, "pv-1"), + }, + }, + instances: []compute.Instance{ + { + Name: "node-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s", project, zone, "node-1"), + Disks: []*compute.AttachedDisk{ + { + DeviceName: "pv-1", + Source: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/disks/%s", project, region, "pv-1"), + }, + }, + }, + }, + expectedEntries: []*csi.ListVolumesResponse_Entry{ + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/regions/%s/disks/%s", project, region, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone, "node-1")}, + }, + }, + }, + }, + { + name: "multi-zone attached disk", + disks: []compute.Disk{ + { + Name: fmt.Sprintf("%s-pv-1", zone1), + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone1, "pv-1"), + Labels: map[string]string{common.MultiZoneLabel: "true"}, + }, + { + Name: fmt.Sprintf("%s-pv-1", zone2), + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone2, "pv-1"), + Labels: map[string]string{common.MultiZoneLabel: "true"}, + }, + }, + instances: []compute.Instance{ + { + Name: "node-1", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s", project, zone1, "node-1"), + Disks: []*compute.AttachedDisk{ + { + DeviceName: "pv-1", + Source: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone1, "pv-1"), + }, + }, + Zone: zone1, + }, + { + Name: "node-2", + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s", project, zone2, "node-2"), + Disks: []*compute.AttachedDisk{ + { + DeviceName: "pv-1", + Source: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone2, "pv-1"), + }, + }, + Zone: zone2, + }, + }, + expectedEntries: []*csi.ListVolumesResponse_Entry{ + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone1, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone1, "node-1")}, + }, + }, + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone2, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone2, "node-2")}, + }, + }, + { + Volume: &csi.Volume{ + VolumeId: fmt.Sprintf("projects/%s/zones/multi-zone/disks/%s", project, "pv-1"), + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{ + fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone1, "node-1"), + fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone2, "node-2"), + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var d []*gce.CloudDisk + for _, disk := range tc.disks { + d = append(d, gce.CloudDiskFromV1(&disk)) + } + fakeCloudProvider, err := gce.CreateFakeCloudProvider(project, zone, d) + if err != nil { + } + for _, instance := range tc.instances { + fakeCloudProvider.InsertInstance(&instance, instance.Zone, instance.Name) + } + // Setup new driver each time so no interference + gceDriver := initGCEDriverWithCloudProvider(t, fakeCloudProvider) + gceDriver.cs.multiZoneVolumeHandleConfig = MultiZoneVolumeHandleConfig{ + Enable: true, + } + + resp, err := gceDriver.cs.ListVolumes(context.TODO(), &csi.ListVolumesRequest{}) + if err != nil { + t.Fatalf("ListVolumes unexpected error: %v", err) + } + + if diff := cmp.Diff(tc.expectedEntries, resp.Entries, cmp.Transformer("EntryToVolumeId", entryToVolumeId), cmpopts.SortSlices(less)); diff != "" { + t.Errorf("ListVolumes: -want err, +got err\n%s", diff) } }) } } +func entryToVolumeId(e csi.ListVolumesResponse_Entry) string { + return e.Volume.VolumeId +} + +func less(a, b fmt.Stringer) bool { + return a.String() < b.String() +} + func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { testCases := []struct { name string @@ -1070,7 +2179,7 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { // Setup new driver each time so no interference gceDriver := initGCEDriver(t, nil) - snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil, gceDriver.name) + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil, gceDriver.name, nil) if err != nil { t.Errorf("Got error extracting snapshot parameters: %v", err) } @@ -1280,6 +2389,7 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { name string volumeOnCloud bool expErrCode codes.Code + expErrMsg string sourceVolumeID string reqParameters map[string]string sourceReqParameters map[string]string @@ -1287,6 +2397,7 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { requestCapacityRange *csi.CapacityRange sourceTopology *csi.TopologyRequirement requestTopology *csi.TopologyRequirement + enableStoragePools bool expCloneKey *meta.Key // Accessible topologies validates that the replica zones are valid for regional disk clones. expAccessibleTop []*csi.Topology @@ -1368,6 +2479,31 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { }, }, }, + { + name: "fail cloning with storage pools", + volumeOnCloud: true, + sourceVolumeID: testZonalVolumeSourceID, + requestCapacityRange: stdCapRange, + sourceCapacityRange: stdCapRange, + enableStoragePools: true, + reqParameters: map[string]string{ + common.ParameterKeyType: "test-type", + common.ParameterKeyReplicationType: replicationTypeNone, + common.ParameterKeyDiskEncryptionKmsKey: "encryption-key", + common.ParameterKeyStoragePools: "projects/test-project/zones/country-region-zone/storagePools/storagePool-1", + }, + sourceReqParameters: zonalParams, + sourceTopology: &csi.TopologyRequirement{ + Requisite: requisiteTopology, + Preferred: prefTopology, + }, + requestTopology: &csi.TopologyRequirement{ + Requisite: requisiteTopology, + Preferred: prefTopology, + }, + expErrCode: codes.InvalidArgument, + expErrMsg: "storage pools do not support disk clones", + }, { name: "success zonal -> zonal cloning, req = all zones in region, pref = req w/ src zone as first element: delayed binding without allowedTopologies", volumeOnCloud: true, @@ -1769,6 +2905,7 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { for _, tc := range testCases { t.Logf("test case: %s", tc.name) gceDriver := initGCEDriver(t, nil) + gceDriver.cs.enableStoragePools = tc.enableStoragePools req := &csi.CreateVolumeRequest{ Name: testCloneVolumeName, @@ -1819,6 +2956,9 @@ func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { if tc.expErrCode != codes.OK { t.Fatalf("Expected error: %v, got no error", tc.expErrCode) } + if tc.expErrMsg != "" && tc.expErrMsg != err.Error() { + t.Fatalf("Got error: %v, expected error: %v", err.Error(), tc.expErrMsg) + } // Make sure the response has the source volume. respVolume := resp.GetVolume() @@ -1903,6 +3043,14 @@ func createZonalCloudDisk(name string) *gce.CloudDisk { }) } +func createZonalCloudDiskWithZone(name, zone string) *gce.CloudDisk { + return gce.CloudDiskFromV1(&compute.Disk{ + Name: name, + SelfLink: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/project/zones/zone/name/%s", name), + Zone: zone, + }) +} + func TestDeleteVolume(t *testing.T) { testCases := []struct { name string @@ -1967,6 +3115,63 @@ func TestDeleteVolume(t *testing.T) { } } +func TestMultiZoneDeleteVolume(t *testing.T) { + testCases := []struct { + name string + seedDisks []*gce.CloudDisk + req *csi.DeleteVolumeRequest + expErr bool + }{ + { + name: "single-zone", + seedDisks: []*gce.CloudDisk{ + createZonalCloudDiskWithZone(name, zone), + }, + req: &csi.DeleteVolumeRequest{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, common.MultiZoneValue, name), + }, + }, + { + name: "multi-zone", + seedDisks: []*gce.CloudDisk{ + createZonalCloudDiskWithZone(name, zone), + createZonalCloudDiskWithZone(name, secondZone), + }, + req: &csi.DeleteVolumeRequest{ + VolumeId: fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, common.MultiZoneValue, name), + }, + }, + } + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + // Setup new driver each time so no interference + fcp, err := gce.CreateFakeCloudProvider(project, zone, tc.seedDisks) + if err != nil { + t.Fatalf("Failed to create fake cloud provider: %v", err) + } + // Setup new driver each time so no interference + gceDriver := initGCEDriverWithCloudProvider(t, fcp) + gceDriver.cs.multiZoneVolumeHandleConfig.DiskTypes = []string{"hyperdisk-ml"} + gceDriver.cs.multiZoneVolumeHandleConfig.Enable = true + _, err = gceDriver.cs.DeleteVolume(context.Background(), tc.req) + if err == nil && tc.expErr { + t.Errorf("Expected error but got none") + } + if err != nil && !tc.expErr { + t.Errorf("Did not expect error but got: %v", err) + } + + disks, _, _ := fcp.ListDisks(context.TODO(), []googleapi.Field{}) + if len(disks) > 0 { + t.Errorf("Expected all disks to be deleted. Got: %v", disks) + } + + if err != nil { + continue + } + } +} + func TestGetRequestCapacity(t *testing.T) { testCases := []struct { name string @@ -2369,12 +3574,13 @@ func TestPrependZone(t *testing.T) { func TestPickZonesFromTopology(t *testing.T) { testCases := []struct { - name string - top *csi.TopologyRequirement - locReq *locationRequirements - numZones int - expZones []string - expErr bool + name string + top *csi.TopologyRequirement + locReq *locationRequirements + numZones int + fallbackRequisiteZones []string + expZones []string + expErr bool }{ { name: "success: preferred", @@ -2435,6 +3641,29 @@ func TestPickZonesFromTopology(t *testing.T) { numZones: 1, expZones: []string{"us-central1-a"}, }, + { + name: "success: requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd]", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-f"}, + }, + }, + Preferred: []*csi.Topology{}, + }, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-c", srcReplicationType: replicationTypeNone, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 2, + expZones: []string{"us-central1-c", "us-central1-f"}, + }, { name: "success: preferred and requisite", top: &csi.TopologyRequirement{ @@ -2501,7 +3730,7 @@ func TestPickZonesFromTopology(t *testing.T) { }, locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-a", srcReplicationType: replicationTypeRegionalPD, cloneReplicationType: replicationTypeRegionalPD}, numZones: 5, - expZones: []string{"us-central1-b", "us-central1-c", "us-central1-a", "us-central1-d", "us-central1-f"}, + expZones: []string{"us-central1-b", "us-central1-a", "us-central1-c", "us-central1-d", "us-central1-f"}, }, { name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd]", @@ -2536,6 +3765,149 @@ func TestPickZonesFromTopology(t *testing.T) { numZones: 5, expZones: []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-d", "us-central1-f"}, }, + { + name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd], 3 zones {a, b, c}", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-f"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + }, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-a", srcReplicationType: replicationTypeNone, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 3, + expZones: []string{"us-central1-a", "us-central1-c", "us-central1-b"}, + }, + { + name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd], 3 zones {b, c, f}", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-f"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + }, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-b", srcReplicationType: replicationTypeNone, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 3, + expZones: []string{"us-central1-b", "us-central1-c", "us-central1-f"}, + }, + { + name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd], fallback topologies specified but unused", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-c"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + fallbackRequisiteZones: []string{"us-central1-a", "us-central1-f", "us-central1-g"}, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-a", srcReplicationType: replicationTypeNone, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 2, + expZones: []string{"us-central1-a", "us-central1-b"}, + }, + { + name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:none, cloneReplicationType:regional-pd], fallback topologies specified", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-west1-b"}, + }, + }, + }, + fallbackRequisiteZones: []string{"us-central1-a", "us-central1-b", "us-central1-c"}, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-b", srcReplicationType: replicationTypeNone, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 2, + expZones: []string{"us-central1-b", "us-central1-c"}, + }, + { + name: "success: preferred and requisite, locationRequirements[region:us-central1, zone:us-central1-a, srcReplicationType:regional-pd, cloneReplicationType:regional-pd], fallback topologies specified", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{}, + Preferred: []*csi.Topology{ + // This is a bit contrived, a real regional PD should have two zones + // This only has one, so we can test that a second is pulled from + // fallbackRequisiteZones. + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + fallbackRequisiteZones: []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"}, + locReq: &locationRequirements{srcVolRegion: "us-central1", srcVolZone: "us-central1-b", srcReplicationType: replicationTypeRegionalPD, cloneReplicationType: replicationTypeRegionalPD}, + numZones: 2, + expZones: []string{"us-central1-b", "us-central1-c"}, + }, + { + name: "success: preferred and requisite, fallback topologies specified", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + fallbackRequisiteZones: []string{"us-central1-a", "us-central1-b", "us-central1-c"}, + numZones: 2, + expZones: []string{"us-central1-b", "us-central1-c"}, + }, { name: "fail: not enough topologies", top: &csi.TopologyRequirement{ @@ -2565,6 +3937,24 @@ func TestPickZonesFromTopology(t *testing.T) { numZones: 4, expErr: true, }, + { + name: "fail: not enough topologies, fallback topologies specified", + top: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + fallbackRequisiteZones: []string{"us-central1-a", "us-central1-b", "us-central1-c"}, + numZones: 4, + expErr: true, + }, { name: "fail: no topologies that match locationRequirment, locationRequirements[region:us-east1, zone:us-east1-a, replicationType:none]", top: &csi.TopologyRequirement{ @@ -2656,7 +4046,7 @@ func TestPickZonesFromTopology(t *testing.T) { expErr: true, }, { - name: "success: only requisite, locationRequirements[region:us-central1, zone:us-central1-a, replicationType:regional-pd", + name: "success: only requisite, all zones", top: &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -2693,15 +4083,17 @@ func TestPickZonesFromTopology(t *testing.T) { }, } for _, tc := range testCases { + // Apply a deterministic seed to make the test that calls rand.Intn stable. + rand.Seed(8) t.Logf("test case: %s", tc.name) - gotZones, err := pickZonesFromTopology(tc.top, tc.numZones, tc.locReq) + gotZones, err := pickZonesFromTopology(tc.top, tc.numZones, tc.locReq, tc.fallbackRequisiteZones) if err != nil && !tc.expErr { t.Errorf("got error: %v, but did not expect error", err) } if err == nil && tc.expErr { t.Errorf("got no error, but expected error") } - if !sets.NewString(gotZones...).Equal(sets.NewString(tc.expZones...)) { + if !slices.Equal(gotZones, tc.expZones) { t.Errorf("Expected zones: %v, but got: %v", tc.expZones, gotZones) } } @@ -2719,80 +4111,6 @@ func zonesEqual(gotZones, expectedZones []string) bool { return true } -func TestPickRandAndConsecutive(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - testCases := []struct { - name string - slice []string - n int - expErr bool - }{ - { - name: "success: normal", - slice: []string{"test", "second", "third"}, - n: 2, - }, - { - name: "success: full", - slice: []string{"test", "second", "third"}, - n: 3, - }, - { - name: "success: large", - slice: []string{"test", "second", "third", "fourth", "fifth", "sixth"}, - n: 2, - }, - { - name: "fail: n too large", - slice: []string{}, - n: 2, - expErr: true, - }, - } - for _, tc := range testCases { - t.Logf("test case: %s", tc.name) - tot := sets.String{} - sort.Strings(tc.slice) - for i := 0; i < 25; i++ { - theslice, err := pickRandAndConsecutive(tc.slice, tc.n) - if err != nil && !tc.expErr { - t.Errorf("Did not expect error but got: %v", err) - } - if err == nil && tc.expErr { - t.Errorf("Expected error but got none") - } - if err != nil { - break - } - if len(theslice) != tc.n { - t.Errorf("expected the resulting slice to be length %v, but got %v instead", tc.n, theslice) - } - // Find where it is in the slice - var idx = -1 - for j, elem := range tc.slice { - if elem == theslice[0] { - idx = j - break - } - } - if idx == -1 { - t.Errorf("could not find %v in the original slice %v", theslice[0], tc.slice) - } - for j := 0; j < tc.n; j++ { - if theslice[j] != tc.slice[(idx+j)%len(tc.slice)] { - t.Errorf("did not pick sorted consecutive values from the slice") - } - } - - tot.Insert(theslice...) - } - if !tot.Equal(sets.NewString(tc.slice...)) { - t.Errorf("randomly picking n from slice did not get all %v, instead got only %v", tc.slice, tot) - } - - } -} - func TestVolumeOperationConcurrency(t *testing.T) { readyToExecute := make(chan chan gce.Signal, 1) gceDriver := initBlockingGCEDriver(t, []*gce.CloudDisk{ @@ -3470,7 +4788,7 @@ func TestCreateConfidentialVolume(t *testing.T) { gceDriver := initGCEDriverWithCloudProvider(t, fcp) if tc.req.VolumeContentSource.GetType() != nil { - snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil, gceDriver.name) + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil, gceDriver.name, nil) if err != nil { t.Errorf("Got error extracting snapshot parameters: %v", err) } @@ -3525,10 +4843,10 @@ func backoffDriver(t *testing.T, config *backoffDriverConfig) *GCEDriver { driver := GetGCEDriver() driver.cs = &GCEControllerServer{ - Driver: driver, - seen: map[string]int{}, - volumeLocks: common.NewVolumeLocks(), - errorBackoff: newFakeCSIErrorBackoff(config.clock), + Driver: driver, + volumeEntriesSeen: map[string]int{}, + volumeLocks: common.NewVolumeLocks(), + errorBackoff: newFakeCSIErrorBackoff(config.clock), } driver.cs.CloudProvider = fcp diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index 6a5ff3f2..c40bd696 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -54,6 +54,7 @@ type GCEDriver struct { name string vendorVersion string extraVolumeLabels map[string]string + extraTags map[string]string ids *GCEIdentityServer ns *GCENodeServer @@ -68,7 +69,7 @@ func GetGCEDriver() *GCEDriver { return &GCEDriver{} } -func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, extraVolumeLabels map[string]string, identityServer *GCEIdentityServer, controllerServer *GCEControllerServer, nodeServer *GCENodeServer) error { +func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, extraVolumeLabels map[string]string, extraTags map[string]string, identityServer *GCEIdentityServer, controllerServer *GCEControllerServer, nodeServer *GCENodeServer) error { if name == "" { return fmt.Errorf("Driver name missing") } @@ -102,6 +103,7 @@ func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, extraVolu gceDriver.name = name gceDriver.vendorVersion = vendorVersion gceDriver.extraVolumeLabels = extraVolumeLabels + gceDriver.extraTags = extraTags gceDriver.ids = identityServer gceDriver.cs = controllerServer gceDriver.ns = nodeServer @@ -171,22 +173,26 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi } } -func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration) *GCEControllerServer { +func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, errorBackoffInitialDuration, errorBackoffMaxDuration time.Duration, fallbackRequisiteZones []string, enableStoragePools bool, multiZoneVolumeHandleConfig MultiZoneVolumeHandleConfig, listVolumesConfig ListVolumesConfig) *GCEControllerServer { return &GCEControllerServer{ - Driver: gceDriver, - CloudProvider: cloudProvider, - seen: map[string]int{}, - volumeLocks: common.NewVolumeLocks(), - errorBackoff: newCsiErrorBackoff(errorBackoffInitialDuration, errorBackoffMaxDuration), + Driver: gceDriver, + CloudProvider: cloudProvider, + volumeEntriesSeen: map[string]int{}, + volumeLocks: common.NewVolumeLocks(), + errorBackoff: newCsiErrorBackoff(errorBackoffInitialDuration, errorBackoffMaxDuration), + fallbackRequisiteZones: fallbackRequisiteZones, + enableStoragePools: enableStoragePools, + multiZoneVolumeHandleConfig: multiZoneVolumeHandleConfig, + listVolumesConfig: listVolumesConfig, } } -func (gceDriver *GCEDriver) Run(endpoint string, grpcLogCharCap int) { +func (gceDriver *GCEDriver) Run(endpoint string, grpcLogCharCap int, enableOtelTracing bool) { maxLogChar = grpcLogCharCap klog.V(4).Infof("Driver: %v", gceDriver.name) - // Start the nonblocking GRPC - s := NewNonBlockingGRPCServer() + //Start the nonblocking GRPC + s := NewNonBlockingGRPCServer(enableOtelTracing) // TODO(#34): Only start specific servers based on a flag. // In the future have this only run specific combinations of servers depending on which version this is. // The schema for that was in util. basically it was just s.start but with some nil servers. diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go index b32bc64b..15306c14 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver_test.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver_test.go @@ -46,9 +46,13 @@ func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) gceDriver := GetGCEDriver() errorBackoffInitialDuration := 200 * time.Millisecond errorBackoffMaxDuration := 5 * time.Minute + fallbackRequisiteZones := []string{} + enableStoragePools := false + multiZoneVolumeHandleConfig := MultiZoneVolumeHandleConfig{} + listVolumesConfig := ListVolumesConfig{} - controllerServer := NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration) - err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, controllerServer, nil) + controllerServer := NewControllerServer(gceDriver, cloudProvider, errorBackoffInitialDuration, errorBackoffMaxDuration, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, nil, controllerServer, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } diff --git a/pkg/gce-pd-csi-driver/identity.go b/pkg/gce-pd-csi-driver/identity.go index 898122f8..7583f6c0 100644 --- a/pkg/gce-pd-csi-driver/identity.go +++ b/pkg/gce-pd-csi-driver/identity.go @@ -29,7 +29,7 @@ type GCEIdentityServer struct { // GetPluginInfo(context.Context, *GetPluginInfoRequest) (*GetPluginInfoResponse, error) func (gceIdentity *GCEIdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { if gceIdentity.Driver.name == "" { - return nil, status.Error(codes.Unavailable, "Driver name not configured") + return nil, status.Error(codes.Internal, "Driver name not configured") } return &csi.GetPluginInfoResponse{ diff --git a/pkg/gce-pd-csi-driver/identity_test.go b/pkg/gce-pd-csi-driver/identity_test.go index 14f6b131..c4cda1a4 100644 --- a/pkg/gce-pd-csi-driver/identity_test.go +++ b/pkg/gce-pd-csi-driver/identity_test.go @@ -26,7 +26,7 @@ func TestGetPluginInfo(t *testing.T) { vendorVersion := "test-vendor" gceDriver := GetGCEDriver() identityServer := NewIdentityServer(gceDriver) - err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, identityServer, nil, nil) + err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, nil, identityServer, nil, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } @@ -49,7 +49,7 @@ func TestGetPluginInfo(t *testing.T) { func TestGetPluginCapabilities(t *testing.T) { gceDriver := GetGCEDriver() identityServer := NewIdentityServer(gceDriver) - err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, identityServer, nil, nil) + err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, identityServer, nil, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } @@ -82,7 +82,7 @@ func TestGetPluginCapabilities(t *testing.T) { func TestProbe(t *testing.T) { gceDriver := GetGCEDriver() identityServer := NewIdentityServer(gceDriver) - err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, identityServer, nil, nil) + err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, identityServer, nil, nil) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go index e4107f9d..39891f24 100644 --- a/pkg/gce-pd-csi-driver/node.go +++ b/pkg/gce-pd-csi-driver/node.go @@ -34,10 +34,13 @@ package gceGCEDriver import ( "context" + "errors" "fmt" "os" "path/filepath" + "regexp" "runtime" + "strconv" "time" "google.golang.org/grpc/codes" @@ -99,8 +102,12 @@ const ( defaultLinuxFsType = "ext4" defaultWindowsFsType = "ntfs" fsTypeExt3 = "ext3" + + readAheadKBMountFlagRegexPattern = "^read_ahead_kb=(.+)$" ) +var readAheadKBMountFlagRegex = regexp.MustCompile(readAheadKBMountFlagRegexPattern) + func getDefaultFsType() string { if runtime.GOOS == "windows" { return defaultWindowsFsType @@ -111,7 +118,7 @@ func getDefaultFsType() string { func (ns *GCENodeServer) isVolumePathMounted(path string) bool { notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(path) - klog.V(4).Infof("NodePublishVolume check volume path %s is mounted %t: error %v", path, !notMnt, err) + klog.V(4).Infof("Checking volume path %s is mounted %t: error %v", path, !notMnt, err) if err == nil && !notMnt { // TODO(#95): check if mount is compatible. Return OK if it is, or appropriate error. /* @@ -343,12 +350,18 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage // Get FS type fstype := getDefaultFsType() + shouldUpdateReadAhead := false + var readAheadKB int64 options := []string{} if mnt := volumeCapability.GetMount(); mnt != nil { if mnt.FsType != "" { fstype = mnt.FsType } options = collectMountOptions(fstype, mnt.MountFlags) + readAheadKB, shouldUpdateReadAhead, err = extractReadAheadKBMountFlag(mnt.MountFlags) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failure parsing mount flags: %v", err.Error()) + } } // [Edgeless] Part 2.5: Map the device as a crypt device, creating a new LUKS partition if needed @@ -409,10 +422,53 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage } } + // Part 5: Update read_ahead + if shouldUpdateReadAhead { + if err := ns.updateReadAhead(devicePath, readAheadKB); err != nil { + return nil, status.Errorf(codes.Internal, "failure updating readahead for %s to %dKB: %v", devicePath, readAheadKB, err.Error()) + } + } + klog.V(4).Infof("NodeStageVolume succeeded on %v to %s", volumeID, stagingTargetPath) return &csi.NodeStageVolumeResponse{}, nil } +func (ns *GCENodeServer) updateReadAhead(devicePath string, readAheadKB int64) error { + isBlock, err := ns.VolumeStatter.IsBlockDevice(devicePath) + if err != nil { + return fmt.Errorf("failed to determine whether %s is a block device: %v", devicePath, err) + } + if !isBlock { + return nil + } + + if err := setReadAheadKB(devicePath, readAheadKB, ns.Mounter); err != nil { + return fmt.Errorf("failed to set readahead: %v", err) + } + + return nil +} + +func extractReadAheadKBMountFlag(mountFlags []string) (int64, bool, error) { + for _, mountFlag := range mountFlags { + if readAheadKB := readAheadKBMountFlagRegex.FindStringSubmatch(mountFlag); len(readAheadKB) == 2 { + // There is only one matching pattern in readAheadKBMountFlagRegex + // If found, it will be at index 1 + readAheadKBInt, err := strconv.ParseInt(readAheadKB[1], 10, 0) + if err != nil { + return -1, false, fmt.Errorf("invalid read_ahead_kb mount flag %q: %v", mountFlag, err) + } + if readAheadKBInt < 0 { + // Negative values can result in unintuitive values when setting read ahead + // (due to blockdev intepreting negative integers as large positive integers). + return -1, false, fmt.Errorf("invalid negative value for read_ahead_kb mount flag: %q", mountFlag) + } + return readAheadKBInt, true, nil + } + } + return -1, false, nil +} + func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { // Validate arguments volumeID := req.GetVolumeId() @@ -447,10 +503,42 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns return nil, status.Errorf(codes.Internal, "NodeUnstageVolume failed to close mapped crypt device for disk %s: %s", stagingTargetPath, err.Error()) } + if err := ns.confirmDeviceUnused(volumeID); err != nil { + var targetErr *ignoreableError + if errors.As(err, &targetErr) { + klog.Warningf("Unabled to check if device for %s is unused. Device has been unmounted successfully. Ignoring and continuing with unstaging. (%v)", volumeID, err) + } else { + return nil, status.Errorf(codes.Internal, "NodeUnstageVolume for volume %s failed: %v", volumeID, err) + } + } + klog.V(4).Infof("NodeUnstageVolume succeeded on %v from %s", volumeID, stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } +// A private error wrapper used to pass control flow decisions up to the caller +type ignoreableError struct{ error } + +func (ns *GCENodeServer) confirmDeviceUnused(volumeID string) error { + devicePath, err := getDevicePath(ns, volumeID, "" /* partition, which is unused */) + if err != nil { + return &ignoreableError{fmt.Errorf("failed to find device path for volume %s: %v", volumeID, err.Error())} + } + + devFsPath, err := filepath.EvalSymlinks(devicePath) + if err != nil { + return &ignoreableError{fmt.Errorf("filepath.EvalSymlinks(%q) failed: %v", devicePath, err)} + } + + if inUse, err := ns.DeviceUtils.IsDeviceFilesystemInUse(ns.Mounter, devicePath, devFsPath); err != nil { + return &ignoreableError{fmt.Errorf("failed to check if device %s (aka %s) is in use: %v", devicePath, devFsPath, err)} + } else if inUse { + return fmt.Errorf("device %s (aka %s) is still in use", devicePath, devFsPath) + } + + return nil +} + func (ns *GCENodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { return &csi.NodeGetCapabilitiesResponse{ Capabilities: ns.Driver.nscap, diff --git a/pkg/gce-pd-csi-driver/node_test.go b/pkg/gce-pd-csi-driver/node_test.go index 459134c6..ef4e1b6c 100644 --- a/pkg/gce-pd-csi-driver/node_test.go +++ b/pkg/gce-pd-csi-driver/node_test.go @@ -35,7 +35,6 @@ package gceGCEDriver import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -46,6 +45,7 @@ import ( testingexec "k8s.io/utils/exec/testing" csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/mount-utils" @@ -91,7 +91,7 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils deviceutils.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver { gceDriver := GetGCEDriver() nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}) - err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nodeServer) + err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } @@ -102,7 +102,7 @@ func getTestBlockingMountGCEDriver(t *testing.T, readyToExecute chan chan struct gceDriver := GetGCEDriver() mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}) - err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nodeServer) + err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } @@ -114,7 +114,7 @@ func getTestBlockingFormatAndMountGCEDriver(t *testing.T, readyToExecute chan ch mounter := mountmanager.NewFakeSafeBlockingMounter(readyToExecute) nodeServer := NewNodeServer(gceDriver, mounter, deviceutils.NewFakeDeviceUtils(false), metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}).WithSerializedFormatAndMount(5*time.Second, 1) - err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nodeServer) + err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nil, nil, nodeServer) if err != nil { t.Fatalf("Failed to setup GCE Driver: %v", err) } @@ -134,7 +134,7 @@ func TestNodeGetVolumeStats(t *testing.T) { gceDriver := getTestGCEDriver(t) ns := gceDriver.ns - tempDir, err := ioutil.TempDir("", "ngvs") + tempDir, err := os.MkdirTemp("", "ngvs") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } @@ -149,21 +149,33 @@ func TestNodeGetVolumeStats(t *testing.T) { Readonly: false, VolumeCapability: stdVolCap, } + _, err = ns.NodePublishVolume(context.Background(), req) if err != nil { t.Fatalf("Failed to set up test by publishing default vol: %v", err) } testCases := []struct { - name string - volumeID string - volumePath string - expectErr bool + name string + volumeID string + volumePath string + expectedResp *csi.NodeGetVolumeStatsResponse + deviceCapacity int + expectErr bool }{ { - name: "normal", - volumeID: defaultVolumeID, - volumePath: targetPath, + name: "normal", + volumeID: defaultVolumeID, + volumePath: targetPath, + deviceCapacity: 300 * 1024 * 1024 * 1024, // 300 GB + expectedResp: &csi.NodeGetVolumeStatsResponse{ + Usage: []*csi.VolumeUsage{ + { + Unit: csi.VolumeUsage_BYTES, + Total: 300 * 1024 * 1024 * 1024, // 300 GB, + }, + }, + }, }, { name: "no vol id", @@ -185,17 +197,38 @@ func TestNodeGetVolumeStats(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + actionList := []testingexec.FakeCommandAction{ + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte(fmt.Sprintf("%d", tc.deviceCapacity)), nil, nil + }, + }, + }, + "blockdev", + strings.Split("--getsize64 /dev/disk/fake-path", " ")..., + ), + } + + mounter := mountmanager.NewFakeSafeMounterWithCustomExec(&testingexec.FakeExec{CommandScript: actionList}) + gceDriver := getTestGCEDriverWithCustomMounter(t, mounter) + ns := gceDriver.ns + req := &csi.NodeGetVolumeStatsRequest{ VolumeId: tc.volumeID, VolumePath: tc.volumePath, } - _, err := ns.NodeGetVolumeStats(context.Background(), req) + resp, err := ns.NodeGetVolumeStats(context.Background(), req) if err != nil && !tc.expectErr { t.Fatalf("Got unexpected err: %v", err) } if err == nil && tc.expectErr { t.Fatal("Did not get error but expected one") } + if diff := cmp.Diff(tc.expectedResp, resp); diff != "" { + t.Errorf("NodeGetVolumeStats(%s): -want, +got \n%s", req, diff) + } }) } } @@ -263,7 +296,7 @@ func TestNodePublishVolume(t *testing.T) { gceDriver := getTestGCEDriver(t) ns := gceDriver.ns - tempDir, err := ioutil.TempDir("", "npv") + tempDir, err := os.MkdirTemp("", "npv") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } @@ -362,7 +395,7 @@ func TestNodeUnpublishVolume(t *testing.T) { gceDriver := getTestGCEDriver(t) ns := gceDriver.ns - tempDir, err := ioutil.TempDir("", "nupv") + tempDir, err := os.MkdirTemp("", "nupv") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } @@ -424,7 +457,7 @@ func TestNodeStageVolume(t *testing.T) { AccessType: blockCap, } - tempDir, err := ioutil.TempDir("", "nsv") + tempDir, err := os.MkdirTemp("", "nsv") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } @@ -432,13 +465,17 @@ func TestNodeStageVolume(t *testing.T) { stagingPath := filepath.Join(tempDir, defaultStagingPath) testCases := []struct { - name string - req *csi.NodeStageVolumeRequest - deviceSize int - blockExtSize int - readonlyBit string - expResize bool - expErrCode codes.Code + name string + req *csi.NodeStageVolumeRequest + deviceSize int + blockExtSize int + readonlyBit string + expResize bool + expReadAheadUpdate bool + expReadAheadKB string + readAheadSectors string + sectorSizeInBytes int + expErrCode codes.Code }{ { name: "Valid request, no resize because block and filesystem sizes match", @@ -488,6 +525,54 @@ func TestNodeStageVolume(t *testing.T) { readonlyBit: "0", expResize: false, }, + { + name: "Valid request, update readahead", + req: &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{"read_ahead_kb=4096"}, + }, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + deviceSize: 5, + blockExtSize: 1, + readonlyBit: "0", + expResize: true, + expReadAheadUpdate: true, + readAheadSectors: "8192", + sectorSizeInBytes: 512, + }, + { + name: "Valid request, update readahead (different sectorsize)", + req: &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{"read_ahead_kb=4096"}, + }, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + deviceSize: 5, + blockExtSize: 1, + readonlyBit: "0", + expResize: true, + expReadAheadUpdate: true, + readAheadSectors: "4194304", + sectorSizeInBytes: 1, + }, { name: "Invalid request (Bad Access Mode)", req: &csi.NodeStageVolumeRequest{ @@ -531,10 +616,47 @@ func TestNodeStageVolume(t *testing.T) { }, expErrCode: codes.InvalidArgument, }, + { + name: "Invalid request (Invalid read_ahead_kb)", + req: &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{"read_ahead_kb=not_a_number"}, + }, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, + { + name: "Invalid request (negative read_ahead_kb)", + req: &csi.NodeStageVolumeRequest{ + VolumeId: volumeID, + StagingTargetPath: stagingPath, + VolumeCapability: &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + MountFlags: []string{"read_ahead_kb=-4096"}, + }, + }, + AccessMode: &csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + expErrCode: codes.InvalidArgument, + }, } for _, tc := range testCases { t.Logf("Test case: %s", tc.name) resizeCalled := false + readAheadUpdateCalled := false actionList := []testingexec.FakeCommandAction{ makeFakeCmd( &testingexec.FakeCmd{ @@ -631,6 +753,33 @@ func TestNodeStageVolume(t *testing.T) { ), }...) } + if tc.expReadAheadUpdate { + actionList = append(actionList, []testingexec.FakeCommandAction{ + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() ([]byte, []byte, error) { + return []byte(fmt.Sprintf("%d", tc.sectorSizeInBytes)), nil, nil + }, + }, + }, + "blockdev", + []string{"--getss", "/dev/disk/fake-path"}..., + ), + makeFakeCmd( + &testingexec.FakeCmd{ + CombinedOutputScript: []testingexec.FakeAction{ + func() (_ []byte, args []byte, _ error) { + readAheadUpdateCalled = true + return []byte{}, nil, nil + }, + }, + }, + "blockdev", + []string{"--setra", tc.readAheadSectors, "/dev/disk/fake-path"}..., + ), + }...) + } mounter := mountmanager.NewFakeSafeMounterWithCustomExec(&testingexec.FakeExec{CommandScript: actionList}) gceDriver := getTestGCEDriverWithCustomMounter(t, mounter) ns := gceDriver.ns @@ -654,6 +803,12 @@ func TestNodeStageVolume(t *testing.T) { if tc.expResize == false && resizeCalled == true { t.Fatalf("Test called resize, but it was not expected.") } + if tc.expReadAheadUpdate == true && readAheadUpdateCalled == false { + t.Fatalf("Test did not update read ahead, but it was expected.") + } + if tc.expReadAheadUpdate == false && readAheadUpdateCalled == true { + t.Fatalf("Test updated read ahead, but it was not expected.") + } } } @@ -835,7 +990,7 @@ func makeFakeCmd(fakeCmd *testingexec.FakeCmd, cmd string, args ...string) testi func TestNodeUnstageVolume(t *testing.T) { gceDriver := getTestGCEDriver(t) ns := gceDriver.ns - tempDir, err := ioutil.TempDir("", "nusv") + tempDir, err := os.MkdirTemp("", "nusv") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } @@ -901,7 +1056,7 @@ func TestNodeGetCapabilities(t *testing.T) { func runBlockingFormatAndMount(t *testing.T, gceDriver *GCEDriver, readyToExecute chan chan struct{}) { ns := gceDriver.ns - tempDir, err := ioutil.TempDir("", "cno") + tempDir, err := os.MkdirTemp("", "cno") if err != nil { t.Fatalf("Failed to set up temp dir: %v", err) } diff --git a/pkg/gce-pd-csi-driver/server.go b/pkg/gce-pd-csi-driver/server.go index 48f3af69..8dd212e7 100644 --- a/pkg/gce-pd-csi-driver/server.go +++ b/pkg/gce-pd-csi-driver/server.go @@ -21,6 +21,7 @@ import ( "path/filepath" "sync" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "k8s.io/klog/v2" @@ -39,14 +40,15 @@ type NonBlockingGRPCServer interface { ForceStop() } -func NewNonBlockingGRPCServer() NonBlockingGRPCServer { - return &nonBlockingGRPCServer{} +func NewNonBlockingGRPCServer(enableOtelTracing bool) NonBlockingGRPCServer { + return &nonBlockingGRPCServer{otelTracing: enableOtelTracing} } // NonBlocking server type nonBlockingGRPCServer struct { - wg sync.WaitGroup - server *grpc.Server + wg sync.WaitGroup + server *grpc.Server + otelTracing bool } func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { @@ -71,8 +73,13 @@ func (s *nonBlockingGRPCServer) ForceStop() { } func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + grpcInterceptor := grpc.UnaryInterceptor(logGRPC) + if s.otelTracing { + grpcInterceptor = grpc.ChainUnaryInterceptor(logGRPC, otelgrpc.UnaryServerInterceptor()) + } + opts := []grpc.ServerOption{ - grpc.UnaryInterceptor(logGRPC), + grpcInterceptor, } u, err := url.Parse(endpoint) diff --git a/pkg/gce-pd-csi-driver/trace.go b/pkg/gce-pd-csi-driver/trace.go new file mode 100644 index 00000000..2c4b9dd1 --- /dev/null +++ b/pkg/gce-pd-csi-driver/trace.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gceGCEDriver + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + "k8s.io/klog/v2" +) + +func InitOtelTracing() (*otlptrace.Exporter, error) { + // Setup OTLP exporter + ctx := context.Background() + exporter, err := otlptracegrpc.New(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create the OTLP exporter: %w", err) + } + + // Resource will auto populate spans with common attributes + resource, err := resource.New(ctx, + resource.WithFromEnv(), // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables + resource.WithProcess(), + resource.WithOS(), + resource.WithContainer(), + resource.WithHost(), + ) + if err != nil { + klog.Errorf("Failed to create the OTLP resource, spans will lack some metadata: %v", err) + } + + // Create a trace provider with the exporter. + // Use propagator and sampler defined in environment variables. + traceProvider := trace.NewTracerProvider(trace.WithBatcher(exporter), trace.WithResource(resource)) + + // Register the trace provider as global. + otel.SetTracerProvider(traceProvider) + + return exporter, nil +} diff --git a/pkg/gce-pd-csi-driver/utils.go b/pkg/gce-pd-csi-driver/utils.go index d8ef8ce0..8e14b272 100644 --- a/pkg/gce-pd-csi-driver/utils.go +++ b/pkg/gce-pd-csi-driver/utils.go @@ -23,7 +23,10 @@ import ( csi "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" ) const ( @@ -139,6 +142,68 @@ func validateAccessMode(am *csi.VolumeCapability_AccessMode) error { return nil } +func validateStoragePools(req *csi.CreateVolumeRequest, params common.DiskParameters, project string) error { + storagePoolsEnabled := params.StoragePools != nil + if !storagePoolsEnabled || req == nil { + return nil + } + + if params.EnableConfidentialCompute { + return fmt.Errorf("storage pools do not support confidential storage") + } + + if !(params.DiskType == "hyperdisk-balanced" || params.DiskType == "hyperdisk-throughput") { + return fmt.Errorf("invalid disk-type: %q. storage pools only support hyperdisk-balanced or hyperdisk-throughput", params.DiskType) + } + + if params.ReplicationType == replicationTypeRegionalPD { + return fmt.Errorf("storage pools do not support regional PD") + } + + if useVolumeCloning(req) { + return fmt.Errorf("storage pools do not support disk clones") + } + + // Check that requisite zones matches the storage pools zones. + // This means allowedTopologies was set properly by GCW. + if err := validateStoragePoolZones(req, params.StoragePools); err != nil { + return fmt.Errorf("failed to validate storage pools zones: %v", err) + } + + // Check that Storage Pools are in same project as the GCEClient that creates the volume. + if err := validateStoragePoolProjects(project, params.StoragePools); err != nil { + return fmt.Errorf("failed to validate storage pools projects: %v", err) + } + + return nil +} + +func validateStoragePoolZones(req *csi.CreateVolumeRequest, storagePools []common.StoragePool) error { + storagePoolZones, err := common.StoragePoolZones(storagePools) + if err != nil { + return err + } + reqZones, err := getZonesFromTopology(req.GetAccessibilityRequirements().GetRequisite()) + if err != nil { + return err + } + if !common.UnorderedSlicesEqual(storagePoolZones, reqZones) { + return fmt.Errorf("requisite topologies must match storage pools zones. requisite zones: %v, storage pools zones: %v", reqZones, storagePoolZones) + } + return nil +} + +func validateStoragePoolProjects(project string, storagePools []common.StoragePool) error { + spProjects := sets.String{} + for _, sp := range storagePools { + if sp.Project != project { + spProjects.Insert(sp.Project) + return fmt.Errorf("cross-project storage pools usage is not supported. Trying to CreateVolume in project %q with storage pools in projects %v", project, spProjects.UnsortedList()) + } + } + return nil +} + func getMultiWriterFromCapability(vc *csi.VolumeCapability) (bool, error) { if vc.GetAccessMode() == nil { return false, errors.New("access mode is nil") @@ -192,6 +257,11 @@ func collectMountOptions(fsType string, mntFlags []string) []string { var options []string for _, opt := range mntFlags { + if readAheadKBMountFlagRegex.FindString(opt) != "" { + // The read_ahead_kb flag is a special flag that isn't + // passed directly as an option to the mount command. + continue + } options = append(options, opt) } diff --git a/pkg/gce-pd-csi-driver/utils_linux.go b/pkg/gce-pd-csi-driver/utils_linux.go index e686323c..e0e670d8 100644 --- a/pkg/gce-pd-csi-driver/utils_linux.go +++ b/pkg/gce-pd-csi-driver/utils_linux.go @@ -112,3 +112,24 @@ func getBlockSizeBytes(devicePath string, m *mount.SafeFormatAndMount) (int64, e } return gotSizeBytes, nil } + +func setReadAheadKB(devicePath string, readAheadKB int64, m *mount.SafeFormatAndMount) error { + output, err := m.Exec.Command("blockdev", "--getss", devicePath).CombinedOutput() + if err != nil { + return fmt.Errorf("error when reading sector size at path %s: output: %s, err: %w", devicePath, string(output), err) + } + strOut := strings.TrimSpace(string(output)) + sectorSizeBytes, err := strconv.ParseInt(strOut, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse %q into an int size", strOut) + } + readAheadInSectors := readAheadKB * 1024 / sectorSizeBytes + readAheadInSectorsStr := strconv.FormatInt(readAheadInSectors, 10) + // Empirical testing indicates that the actual read_ahead_kb size that is set is rounded to the + // nearest 4KB. + output, err = m.Exec.Command("blockdev", "--setra", readAheadInSectorsStr, devicePath).CombinedOutput() + if err != nil { + return fmt.Errorf("error when setting readahead at path %s: output: %s, err: %w", devicePath, string(output), err) + } + return nil +} diff --git a/pkg/gce-pd-csi-driver/utils_test.go b/pkg/gce-pd-csi-driver/utils_test.go index 2a0fd5f4..fd114fc8 100644 --- a/pkg/gce-pd-csi-driver/utils_test.go +++ b/pkg/gce-pd-csi-driver/utils_test.go @@ -18,9 +18,12 @@ limitations under the License. package gceGCEDriver import ( + "fmt" "testing" csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" ) var ( @@ -291,3 +294,432 @@ func TestGetReadOnlyFromCapabilities(t *testing.T) { } } } + +func TestValidateStoragePools(t *testing.T) { + testCases := []struct { + name string + req *csi.CreateVolumeRequest + params common.DiskParameters + project string + expErr error + }{ + { + name: "success with storage pools not enabled", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + }, + expErr: nil, + }, + { + name: "success with nil CreateVolumeReq", + req: nil, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + }, + expErr: nil, + }, + { + name: "fail storage pools with confidential storage", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + EnableConfidentialCompute: true, + }, + project: "test-project", + expErr: fmt.Errorf("storage pools do not support confidential storage"), + }, + { + name: "fail storage pools with disk type other than HdB/HdT", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + }, + params: common.DiskParameters{ + DiskType: "pd-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "test-project", + expErr: fmt.Errorf("invalid disk-type: \"pd-balanced\". storage pools only support hyperdisk-balanced or hyperdisk-throughput"), + }, + { + name: "fail storage pools with regional PD", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + ReplicationType: "regional-pd", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "test-project", + expErr: fmt.Errorf("storage pools do not support regional PD"), + }, + { + name: "fail storage pools with disk clones", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: "projects/test-project/zones/us-central1-a/disks/disk-1", + }, + }, + }, + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "test-project", + expErr: fmt.Errorf("storage pools do not support disk clones"), + }, + { + name: "fail storage pools zones, requisite zones mismatch", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "test-project", + expErr: fmt.Errorf("failed to validate storage pools zones: requisite topologies must match storage pools zones. requisite zones: [us-central1-a], storage pools zones: [us-central1-a us-central1-b]"), + }, + { + name: "fail storage pools cross-project usage", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "other-project", + expErr: fmt.Errorf("failed to validate storage pools projects: cross-project storage pools usage is not supported. Trying to CreateVolume in project \"other-project\" with storage pools in projects [test-project]"), + }, + { + name: "success validateStoragePools", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + }, + params: common.DiskParameters{ + DiskType: "hyperdisk-balanced", + StoragePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + }, + project: "test-project", + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + input := "validateStoragePools()" + err := validateStoragePools(tc.req, tc.params, tc.project) + if tc.expErr != nil && err == nil { + t.Fatalf("%s didn't get any error, but expected error %v", input, tc.expErr) + } + if tc.expErr == nil && err != nil { + t.Fatalf("%s got error %v, but didn't expect any error", input, err) + } + if err != nil && tc.expErr != nil { + if diff := cmp.Diff(err.Error(), tc.expErr.Error()); diff != "" { + t.Errorf("%s: -want, +got \n%s", input, diff) + } + } + } +} + +func TestValidateStoragePoolZones(t *testing.T) { + testCases := []struct { + name string + req *csi.CreateVolumeRequest + storagePools []common.StoragePool + expErr error + }{ + { + name: "fail with 2 storage pools in 1 zone", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + storagePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + expErr: fmt.Errorf("found multiple storage pools in zone us-central1-a. Only one storage pool per zone is allowed"), + }, + { + name: "fail with requisite topology with no segments", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{{}}, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + storagePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + expErr: fmt.Errorf("topologies specified but no segments"), + }, + { + name: "fail with requisite zones does not match storage pools zones", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + }, + }, + }, + storagePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-b/storagePools/storagePool-1", + }, + }, + expErr: fmt.Errorf("requisite topologies must match storage pools zones. requisite zones: [us-central1-a], storage pools zones: [us-central1-b]"), + }, + { + name: "success validateStoragePoolZones", + req: &csi.CreateVolumeRequest{ + Name: "test-name", + AccessibilityRequirements: &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + Preferred: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: "us-central1-b"}, + }, + }, + }, + }, + storagePools: []common.StoragePool{ + { + Project: "test-project", + Zone: "us-central1-a", + Name: "storagePool-1", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + { + Project: "test-project", + Zone: "us-central1-b", + Name: "storagePool-2", + ResourceName: "projects/test-project/zones/us-central1-a/storagePools/storagePool-1", + }, + }, + expErr: nil, + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + err := validateStoragePoolZones(tc.req, tc.storagePools) + input := "validateStoragePoolZones()" + if tc.expErr != nil && err == nil { + t.Fatalf("%s didn't get any error, but expected error %v", input, tc.expErr) + } + if tc.expErr == nil && err != nil { + t.Fatalf("%s got error %v, but didn't expect any error", input, err) + } + if err != nil && tc.expErr != nil { + if diff := cmp.Diff(err.Error(), tc.expErr.Error()); diff != "" { + t.Errorf("%s: -want, +got \n%s", input, diff) + } + } + } +} diff --git a/pkg/gce-pd-csi-driver/utils_windows.go b/pkg/gce-pd-csi-driver/utils_windows.go index b12f4f82..f40639d4 100644 --- a/pkg/gce-pd-csi-driver/utils_windows.go +++ b/pkg/gce-pd-csi-driver/utils_windows.go @@ -105,3 +105,8 @@ func getBlockSizeBytes(devicePath string, m *mount.SafeFormatAndMount) (int64, e } return proxy.GetDiskTotalBytes(devicePath) } + +func setReadAheadKB(devicePath string, readAheadKB int64, m *mount.SafeFormatAndMount) error { + // This is a no-op on windows. + return nil +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a08f5074..8a998f08 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -17,6 +17,7 @@ limitations under the License. package metrics import ( + "errors" "fmt" "net/http" "os" @@ -36,6 +37,7 @@ const ( pdcsiDriverName = "pd.csi.storage.gke.io" DefaultDiskTypeForMetric = "unknownDiskType" DefaultEnableConfidentialCompute = "unknownConfidentialMode" + DefaultEnableStoragePools = "unknownStoragePools" ) var ( @@ -52,7 +54,7 @@ var ( Help: "CSI server side error metrics", StabilityLevel: metrics.ALPHA, }, - []string{"driver_name", "method_name", "grpc_status_code", "disk_type", "enable_confidential_storage"}) + []string{"driver_name", "method_name", "grpc_status_code", "disk_type", "enable_confidential_storage", "enable_storage_pools"}) ) type MetricsManager struct { @@ -94,12 +96,11 @@ func (mm *MetricsManager) RecordOperationErrorMetrics( operationName string, operationErr error, diskType string, - enableConfidentialStorage string) { - err := codes.OK.String() - if operationErr != nil { - err = common.CodeForError(operationErr).String() - } - pdcsiOperationErrorsMetric.WithLabelValues(pdcsiDriverName, "/csi.v1.Controller/"+operationName, err, diskType, enableConfidentialStorage).Inc() + enableConfidentialStorage string, + enableStoragePools string) { + errCode := errorCodeLabelValue(operationErr) + pdcsiOperationErrorsMetric.WithLabelValues(pdcsiDriverName, "/csi.v1.Controller/"+operationName, errCode, diskType, enableConfidentialStorage, enableStoragePools).Inc() + klog.Infof("Recorded PDCSI operation error code: %q", errCode) } func (mm *MetricsManager) EmitGKEComponentVersion() error { @@ -156,12 +157,29 @@ func IsGKEComponentVersionAvailable() bool { return true } -func GetMetricParameters(disk *gce.CloudDisk) (string, string) { +func GetMetricParameters(disk *gce.CloudDisk) (string, string, string) { diskType := DefaultDiskTypeForMetric enableConfidentialStorage := DefaultEnableConfidentialCompute + enableStoragePools := DefaultEnableStoragePools if disk != nil { diskType = disk.GetPDType() enableConfidentialStorage = strconv.FormatBool(disk.GetEnableConfidentialCompute()) + enableStoragePools = strconv.FormatBool(disk.GetEnableStoragePools()) + } + return diskType, enableConfidentialStorage, enableStoragePools +} + +// errorCodeLabelValue returns the label value for the given operation error. +// This was separated into a helper function for unit testing purposes. +func errorCodeLabelValue(operationErr error) string { + err := codes.OK.String() + if operationErr != nil { + // If the operationErr is a TemporaryError, unwrap the temporary error before passing it to CodeForError. + var tempErr *common.TemporaryError + if errors.As(operationErr, &tempErr) { + operationErr = tempErr.Unwrap() + } + err = common.CodeForError(operationErr).String() } - return diskType, enableConfidentialStorage + return err } diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index dcf49092..62499a57 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -18,10 +18,19 @@ limitations under the License. package metrics import ( + "context" + "errors" + "fmt" + "net/http" "testing" - computebeta "google.golang.org/api/compute/v0.beta" + "github.com/google/go-cmp/cmp" "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" ) @@ -29,53 +38,148 @@ const ( hyperdiskBalanced = "hyperdisk-balanced" ) -func CreateDiskWithConfidentialCompute(betaVersion bool, confidentialCompute bool, diskType string) *gce.CloudDisk { - if betaVersion { - return gce.CloudDiskFromBeta(&computebeta.Disk{ - EnableConfidentialCompute: confidentialCompute, - Type: diskType, - }) - } +func CreateDiskWithConfidentialCompute(confidentialCompute bool, diskType string) *gce.CloudDisk { + return gce.CloudDiskFromV1(&compute.Disk{ + EnableConfidentialCompute: confidentialCompute, + Type: diskType, + }) +} + +func CreateDiskWithStoragePool(storagePool string, diskType string) *gce.CloudDisk { return gce.CloudDiskFromV1(&compute.Disk{ - Type: diskType, + StoragePool: storagePool, + Type: diskType, }) } -func TestGetEnableConfidentialCompute(t *testing.T) { +func TestGetMetricParameters(t *testing.T) { testCases := []struct { name string disk *gce.CloudDisk expectedEnableConfidentialCompute string expectedDiskType string + expectedEnableStoragePools string }{ { - name: "test betaDisk with enableConfidentialCompute=false", - disk: CreateDiskWithConfidentialCompute(true, false, hyperdiskBalanced), + name: "test disk with enableConfidentialCompute=false", + disk: CreateDiskWithConfidentialCompute(false, hyperdiskBalanced), expectedEnableConfidentialCompute: "false", expectedDiskType: hyperdiskBalanced, + expectedEnableStoragePools: "false", }, { - name: "test betaDisk with enableConfidentialCompute=true", - disk: CreateDiskWithConfidentialCompute(true, true, hyperdiskBalanced), + name: "test disk with enableConfidentialCompute=true", + disk: CreateDiskWithConfidentialCompute(true, hyperdiskBalanced), expectedEnableConfidentialCompute: "true", expectedDiskType: hyperdiskBalanced, + expectedEnableStoragePools: "false", }, { - name: "test disk withpit enableConfidentialCompute", - disk: CreateDiskWithConfidentialCompute(false, false, hyperdiskBalanced), + name: "test disk with storage pool projects/my-project/zone/us-central1-a/storagePools/sp1", + disk: CreateDiskWithStoragePool("projects/my-project/zone/us-central1-a/storagePools/sp1", hyperdiskBalanced), expectedEnableConfidentialCompute: "false", expectedDiskType: hyperdiskBalanced, + expectedEnableStoragePools: "true", + }, + { + name: "test disk with no storage pool", + disk: CreateDiskWithStoragePool("", hyperdiskBalanced), + expectedEnableConfidentialCompute: "false", + expectedDiskType: hyperdiskBalanced, + expectedEnableStoragePools: "false", + }, + { + name: "test nil disk", + disk: nil, + expectedEnableConfidentialCompute: DefaultEnableConfidentialCompute, + expectedDiskType: DefaultDiskTypeForMetric, + expectedEnableStoragePools: DefaultEnableStoragePools, }, } for _, tc := range testCases { t.Logf("Running test: %v", tc.name) - diskType, confidentialCompute := GetMetricParameters(tc.disk) + diskType, confidentialCompute, enableStoragePools := GetMetricParameters(tc.disk) if confidentialCompute != tc.expectedEnableConfidentialCompute { - t.Fatalf("Got confidentialCompute value %v expected %v", confidentialCompute, tc.expectedEnableConfidentialCompute) + t.Fatalf("Got confidentialCompute value %q expected %q", confidentialCompute, tc.expectedEnableConfidentialCompute) } if diskType != tc.expectedDiskType { - t.Fatalf("Got confidentialCompute value %v expected %v", diskType, tc.expectedDiskType) + t.Fatalf("Got diskType value %q expected %q", diskType, tc.expectedDiskType) + } + if enableStoragePools != tc.expectedEnableStoragePools { + t.Fatalf("Got enableStoragePools value %q expected %q", enableStoragePools, tc.expectedEnableStoragePools) + } + } +} + +func TestErrorCodeLabelValue(t *testing.T) { + testCases := []struct { + name string + operationErr error + wantErrorCode string + }{ + { + name: "Not googleapi.Error", + operationErr: errors.New("I am not a googleapi.Error"), + wantErrorCode: "Internal", + }, + { + name: "User error", + operationErr: &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"}, + wantErrorCode: "InvalidArgument", + }, + { + name: "googleapi.Error but not a user error", + operationErr: &googleapi.Error{Code: http.StatusInternalServerError, Message: "Internal error"}, + wantErrorCode: "Internal", + }, + { + name: "context canceled error", + operationErr: context.Canceled, + wantErrorCode: "Canceled", + }, + { + name: "context deadline exceeded error", + operationErr: context.DeadlineExceeded, + wantErrorCode: "DeadlineExceeded", + }, + { + name: "status error with Aborted error code", + operationErr: status.Error(codes.Aborted, "aborted error"), + wantErrorCode: "Aborted", + }, + { + name: "user multiattach error", + operationErr: fmt.Errorf("The disk resource 'projects/foo/disk/bar' is already being used by 'projects/foo/instances/1'"), + wantErrorCode: "InvalidArgument", + }, + { + name: "TemporaryError that wraps googleapi error", + operationErr: common.NewTemporaryError(codes.Unavailable, &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"}), + wantErrorCode: "InvalidArgument", + }, + { + name: "TemporaryError that wraps fmt.Errorf, which wraps googleapi error", + operationErr: common.NewTemporaryError(codes.Aborted, fmt.Errorf("got error: %w", &googleapi.Error{Code: http.StatusBadRequest, Message: "User error with bad request"})), + wantErrorCode: "InvalidArgument", + }, + { + name: "TemporaryError that wraps status error", + operationErr: common.NewTemporaryError(codes.Aborted, status.Error(codes.InvalidArgument, "User error with bad request")), + wantErrorCode: "InvalidArgument", + }, + { + name: "TemporaryError that wraps multiattach error", + operationErr: common.NewTemporaryError(codes.Unavailable, fmt.Errorf("The disk resource 'projects/foo/disk/bar' is already being used by 'projects/foo/instances/1'")), + wantErrorCode: "InvalidArgument", + }, + } + + for _, tc := range testCases { + t.Logf("Running test: %v", tc.name) + errCode := errorCodeLabelValue(tc.operationErr) + if diff := cmp.Diff(tc.wantErrorCode, errCode); diff != "" { + t.Errorf("%s: -want err, +got err\n%s", tc.name, diff) } } } diff --git a/pkg/mount-manager/statter_linux.go b/pkg/mount-manager/statter_linux.go index d75c39b3..921ffdb9 100644 --- a/pkg/mount-manager/statter_linux.go +++ b/pkg/mount-manager/statter_linux.go @@ -61,10 +61,26 @@ func (*realStatter) StatFS(path string) (available, capacity, used, inodesFree, return } -type fakeStatter struct{} +type fakeStatter struct { + options FakeStatterOptions +} + +type FakeStatterOptions struct { + IsBlock bool +} func NewFakeStatter(mounter *mount.SafeFormatAndMount) *fakeStatter { - return &fakeStatter{} + return &fakeStatter{ + options: FakeStatterOptions{ + IsBlock: true, + }, + } +} + +func NewFakeStatterWithOptions(mounter *mount.SafeFormatAndMount, options FakeStatterOptions) *fakeStatter { + return &fakeStatter{ + options: options, + } } func (*fakeStatter) StatFS(path string) (available, capacity, used, inodesFree, inodes, inodesUsed int64, err error) { @@ -72,6 +88,6 @@ func (*fakeStatter) StatFS(path string) (available, capacity, used, inodesFree, return 1, 1, 1, 1, 1, 1, nil } -func (*fakeStatter) IsBlockDevice(fullPath string) (bool, error) { - return false, nil +func (fs *fakeStatter) IsBlockDevice(fullPath string) (bool, error) { + return fs.options.IsBlock, nil } diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index bf310083..107508ed 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -17,7 +17,7 @@ The release manager must: Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs must be updated. -[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +[Our CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) have the naming convention `-on-`. 1. Jobs should be actively monitored to find and fix failures in sidecars and @@ -90,7 +90,7 @@ naming convention `-on-`. 1. Submit a PR for README changes, in particular, Compatibility, Feature status, and any other sections that may need updating. 1. Check that all [canary CI - jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. @@ -99,7 +99,7 @@ naming convention `-on-`. [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) 1. If release was a new major/minor version, create a new `release-` branch at that commit. -1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). +1. Check [image build status](https://testgrid.k8s.io/sig-storage-image-build). 1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From the [k8s image repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), @@ -118,7 +118,7 @@ naming convention `-on-`. The following jobs are triggered after tagging to produce the corresponding image(s): -https://k8s-testgrid.appspot.com/sig-storage-image-build +https://testgrid.k8s.io/sig-storage-image-build Clicking on a failed build job opens that job in https://prow.k8s.io. Next to the job title is a rerun icon (circle with arrow). Clicking it opens a popup diff --git a/release-tools/filter-junit.go b/release-tools/filter-junit.go index 5454092b..25aae6e2 100644 --- a/release-tools/filter-junit.go +++ b/release-tools/filter-junit.go @@ -24,7 +24,6 @@ package main import ( "encoding/xml" "flag" - "io/ioutil" "os" "regexp" ) @@ -95,7 +94,7 @@ func main() { } } else { var err error - data, err = ioutil.ReadFile(input) + data, err = os.ReadFile(input) if err != nil { panic(err) } @@ -142,7 +141,7 @@ func main() { panic(err) } } else { - if err := ioutil.WriteFile(*output, data, 0644); err != nil { + if err := os.WriteFile(*output, data, 0644); err != nil { panic(err) } } diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go deleted file mode 100644 index bb54356c..00000000 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ /dev/null @@ -1,410 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tests - -import ( - "fmt" - "path/filepath" - "strings" - - csi "github.com/container-storage-interface/spec/lib/go/csi" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" - remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" -) - -type verifyArgs struct { - publishDir, stageDir string -} - -type verifyFunc func(*verifyArgs) error - -type detacherFunc func() - -var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { - BeforeEach(func() { - Expect(len(testContexts)).To(BeNumerically(">", 1)) - }) - - It("Should get reasonable topology from nodes with NodeGetInfo", func() { - for _, testContext := range testContexts { - resp, err := testContext.Client.NodeGetInfo() - Expect(err).To(BeNil()) - - // Get Cloud Instance - p, z, n := testContext.Instance.GetIdentity() - cloudInstance, err := computeService.Instances.Get(p, z, n).Do() - Expect(err).To(BeNil()) - Expect(cloudInstance).ToNot(BeNil()) - - // Check topology matches - segments := resp.GetAccessibleTopology().GetSegments() - Expect(segments).ToNot(BeNil()) - - Expect(segments[common.TopologyKeyZone]).To(Equal(z)) - Expect(len(segments)).To(Equal(1)) - } - - }) - - It("Should successfully run through entire lifecycle of an RePD volume on instances in 2 zones", func() { - // Create new driver and client - - Expect(testContexts).NotTo(BeEmpty()) - - zoneToContext := map[string]*remote.TestContext{} - zones := []string{} - - for _, tc := range testContexts { - _, z, _ := tc.Instance.GetIdentity() - // Zone hasn't been seen before - if _, ok := zoneToContext[z]; !ok { - zoneToContext[z] = tc - zones = append(zones, z) - } - if len(zoneToContext) == 2 { - break - } - } - - Expect(len(zoneToContext)).To(Equal(2), "Must have instances in 2 zones") - - controllerContext := zoneToContext[zones[0]] - controllerClient := controllerContext.Client - controllerInstance := controllerContext.Instance - - p, _, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones(zones) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: zones[0]}, - }, - { - Segments: map[string]string{common.TopologyKeyZone: zones[1]}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - zonesSet := sets.NewString(zones...) - for _, replicaZone := range cloudDisk.ReplicaZones { - tokens := strings.Split(replicaZone, "/") - actualZone := tokens[len(tokens)-1] - Expect(zonesSet.Has(actualZone)).To(BeTrue(), "Expected zone %v to exist in zone set %v", actualZone, zones) - } - - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // For each of the two instances - i := 0 - for _, testContext := range zoneToContext { - readOnly := false - if i >= 1 { - readOnly = true - } - err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly) - Expect(err).To(BeNil(), "failed volume lifecycle checks") - i = i + 1 - } - }) - - It("Should create a RePD instance, write to it, force-attach it to another instance, and read the same data", func() { - Expect(testContexts).NotTo(BeEmpty()) - - zoneToContext := map[string]*remote.TestContext{} - zones := []string{} - - for _, tc := range testContexts { - _, z, _ := tc.Instance.GetIdentity() - // Zone hasn't been seen before - if _, ok := zoneToContext[z]; !ok { - zoneToContext[z] = tc - zones = append(zones, z) - } - if len(zoneToContext) == 2 { - break - } - } - - Expect(len(zoneToContext)).To(Equal(2), "Must have instances in 2 zones") - - controllerContext := zoneToContext[zones[0]] - controllerClient := controllerContext.Client - controllerInstance := controllerContext.Instance - - p, _, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones(zones) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - common.ParameterAvailabilityClass: common.ParameterRegionalHardFailoverClass, - }, defaultRepdSizeGb, &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: zones[0]}, - }, - { - Segments: map[string]string{common.TopologyKeyZone: zones[1]}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - zonesSet := sets.NewString(zones...) - for _, replicaZone := range cloudDisk.ReplicaZones { - tokens := strings.Split(replicaZone, "/") - actualZone := tokens[len(tokens)-1] - Expect(zonesSet.Has(actualZone)).To(BeTrue(), "Expected zone %v to exist in zone set %v", actualZone, zones) - } - Expect(volume.VolumeContext).To(HaveKeyWithValue("force-attach", "true")) - - detachers := []detacherFunc{} - - defer func() { - // Perform any detaches - for _, fn := range detachers { - fn() - } - - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach disk to instance in the first zone. - tc0 := zoneToContext[zones[0]] - err, detacher, args := testAttachAndMount(volume.VolumeId, volName, tc0.Instance, tc0.Client, false /* useBlock */, false /* forceAttach */) - detachers = append(detachers, detacher) - Expect(err).To(BeNil(), "failed attach in zone 0") - testFileName := filepath.Join(args.publishDir, "force-attach-test") - testFileContents := "force attach test" - err = testutils.WriteFile(tc0.Instance, testFileName, testFileContents) - Expect(err).To(BeNil(), "failed write in zone 0") - _, err = tc0.Instance.SSH("sync") // Sync so force detach doesn't lose data. - Expect(err).To(BeNil(), "failed sync") - - readContents, err := testutils.ReadFile(tc0.Instance, testFileName) - Expect(err).To(BeNil(), "failed read in zone 0") - Expect(strings.TrimSpace(string(readContents))).To(BeIdenticalTo(testFileContents), "content mismatch in zone 0") - - // Now force attach to the second instance without detaching. - tc1 := zoneToContext[zones[1]] - err, detacher, args = testAttachAndMount(volume.VolumeId, volName, tc1.Instance, tc1.Client, false /* useBlock */, true /* forceAttach */) - detachers = append(detachers, detacher) - Expect(err).To(BeNil(), "failed force attach in zone 1") - readContents, err = testutils.ReadFile(tc1.Instance, testFileName) - Expect(err).To(BeNil(), "failed read in zone 1") - Expect(strings.TrimSpace(string(readContents))).To(BeIdenticalTo(testFileContents), "content mismatch in zone 1") - }) -}) - -func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error { - var testFileContents = "test" - writeFile := func(a *verifyArgs) error { - if !readOnly { - // Write a file - testFile := filepath.Join(a.publishDir, "testfile") - err := testutils.WriteFile(instance, testFile, testFileContents) - if err != nil { - return fmt.Errorf("Failed to write file: %v", err.Error()) - } - } - return nil - } - - verifyReadFile := func(a *verifyArgs) error { - // Read File - secondTestFile := filepath.Join(a.publishDir, "testfile") - readContents, err := testutils.ReadFile(instance, secondTestFile) - if err != nil { - return fmt.Errorf("ReadFile failed with error: %v", err.Error()) - } - if strings.TrimSpace(string(readContents)) != testFileContents { - return fmt.Errorf("wanted test file content: %s, got content: %s", testFileContents, readContents) - } - return nil - } - return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile) -} - -func testAttachAndMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, useBlock, forceAttach bool) (error, func(), *verifyArgs) { - // Attach Disk - err := client.ControllerPublishVolume(volID, instance.GetNodeID(), forceAttach) - if err != nil { - return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, nil - } - - detach := func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - } - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - if useBlock { - err = client.NodeStageBlockVolume(volID, stageDir) - } else { - err = client.NodeStageExt4Volume(volID, stageDir) - } - - if err != nil { - detach() - return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err), nil, nil - } - - unstageAndDetach := func() { - // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - - detach() - } - - // Mount Disk - publishDir := filepath.Join("/tmp/", volName, "mount") - - if useBlock { - err = client.NodePublishBlockVolume(volID, stageDir, publishDir) - } else { - err = client.NodePublishVolume(volID, stageDir, publishDir) - } - - if err != nil { - unstageAndDetach() - return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()), nil, nil - } - err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") - if err != nil { - unstageAndDetach() - return fmt.Errorf("Chmod failed with error: %v", err.Error()), nil, nil - } - - args := &verifyArgs{ - publishDir: publishDir, - stageDir: stageDir, - } - - return nil, unstageAndDetach, args -} - -func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error { - klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) - err, detacher, args := testAttachAndMount(volID, volName, instance, client, useBlock, false /* forceAttach */) - if err != nil { - return fmt.Errorf("failed to attach and mount: %w", err) - } - defer detacher() - - err = firstMountVerify(args) - if err != nil { - return fmt.Errorf("failed to verify after first mount to %s: %w", args.publishDir, err) - } - - // Unmount Disk - err = client.NodeUnpublishVolume(volID, args.publishDir) - if err != nil { - return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) - } - - if secondMountVerify != nil { - // Mount disk somewhere else - secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") - if useBlock { - err = client.NodePublishBlockVolume(volID, args.stageDir, secondPublishDir) - } else { - err = client.NodePublishVolume(volID, args.stageDir, secondPublishDir) - } - if err != nil { - return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) - } - err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") - if err != nil { - return fmt.Errorf("Chmod failed with error: %v", err) - } - - b := verifyArgs{ - publishDir: secondPublishDir, - } - err = secondMountVerify(&b) - if err != nil { - return fmt.Errorf("failed to verify after second mount to %s: %v", args.publishDir, err.Error()) - } - - // Unmount Disk - err = client.NodeUnpublishVolume(volID, secondPublishDir) - if err != nil { - return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) - } - } - - klog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID()) - return nil -} diff --git a/test/e2e/tests/resize_e2e_test.go b/test/e2e/tests/resize_e2e_test.go deleted file mode 100644 index 2b04047d..00000000 --- a/test/e2e/tests/resize_e2e_test.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tests - -import ( - "path/filepath" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - csi "github.com/container-storage-interface/spec/lib/go/csi" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" -) - -var _ = Describe("GCE PD CSI Driver", func() { - It("Should online resize controller and node for an ext4 volume", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := client.CreateVolume(volName, nil, defaultSizeGb, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: z}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - - defer func() { - // Delete Disk - client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "Controller publish volume failed") - - defer func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - }() - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) - Expect(err).To(BeNil(), "Node Stage volume failed") - - defer func() { - // Unstage Disk - err = client.NodeUnstageVolume(volume.VolumeId, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - }() - - // Mount Disk - publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishVolume(volume.VolumeId, stageDir, publishDir) - Expect(err).To(BeNil(), "Node publish volume failed") - - defer func() { - // Unmount Disk - err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) - if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %w", err) - } - }() - - // Verify pre-resize fs size - sizeGb, err := testutils.GetFSSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get FSSize in GB") - Expect(sizeGb).To(Equal(defaultSizeGb)) - - // Resize controller - var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) - - Expect(err).To(BeNil(), "Controller expand volume failed") - - // Verify cloud size - cloudDisk, err = computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Get cloud disk failed") - Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) - - // Resize node - _, err = client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) - Expect(err).To(BeNil(), "Node expand volume failed") - - // Verify disk size - sizeGb, err = testutils.GetFSSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get FSSize in GB") - Expect(sizeGb).To(Equal(newSizeGb)) - - }) - - It("Should offline resize controller and node for an ext4 volume", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := client.CreateVolume(volName, nil, defaultSizeGb, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: z}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created & size - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - - defer func() { - // Delete Disk - client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Volume should be attached/formatted/mounted/unmounted/detached - err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - - // Resize controller - var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) - - Expect(err).To(BeNil(), "Controller expand volume failed") - - // Verify cloud size - cloudDisk, err = computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Get cloud disk failed") - Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) - - // Attach and mount again - err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "Controller publish volume failed") - - defer func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - - }() - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) - Expect(err).To(BeNil(), "Node Stage volume failed") - - defer func() { - // Unstage Disk - err = client.NodeUnstageVolume(volume.VolumeId, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - }() - - // Mount Disk - publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishVolume(volume.VolumeId, stageDir, publishDir) - Expect(err).To(BeNil(), "Node publish volume failed") - - defer func() { - // Unmount Disk - err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) - if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %w", err) - } - }() - - // Resize node - _, err = client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) - Expect(err).To(BeNil(), "Node expand volume failed") - - // Verify disk size - sizeGb, err := testutils.GetFSSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get FSSize in GB") - Expect(sizeGb).To(Equal(newSizeGb)) - - }) - - It("Should resize controller and node for an block volume", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := client.CreateVolume(volName, nil, defaultSizeGb, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: z}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - - defer func() { - // Delete Disk - client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "Controller publish volume failed") - - defer func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - - }() - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageBlockVolume(volume.VolumeId, stageDir) - Expect(err).To(BeNil(), "Node Stage volume failed") - - defer func() { - // Unstage Disk - err = client.NodeUnstageVolume(volume.VolumeId, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - }() - - // Mount Disk - publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishBlockVolume(volume.VolumeId, stageDir, publishDir) - Expect(err).To(BeNil(), "Node publish volume failed") - - defer func() { - // Unmount Disk - err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) - if err != nil { - klog.Errorf("NodeUnpublishVolume failed with error: %w", err) - } - }() - - // Verify pre-resize fs size - sizeGb, err := testutils.GetBlockSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get block device size in GB") - Expect(sizeGb).To(Equal(defaultSizeGb), "Old size should be equal") - - // Resize controller - var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) - - Expect(err).To(BeNil(), "Controller expand volume failed") - - // Verify cloud size - cloudDisk, err = computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Get cloud disk failed") - Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) - - // Resize node - resp, err := client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) - Expect(err).To(BeNil(), "Node expand volume failed") - Expect(resp.CapacityBytes).To(Equal(common.GbToBytes(newSizeGb)), "Node expand should not do anything") - - // Verify disk size - sizeGb, err = testutils.GetBlockSizeInGb(instance, publishDir) - Expect(err).To(BeNil(), "Failed to get block device size in GB") - Expect(sizeGb).To(Equal(newSizeGb), "New size should be equal") - - }) -}) diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go deleted file mode 100644 index 0b11bf15..00000000 --- a/test/e2e/tests/setup_e2e_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tests - -import ( - "context" - "flag" - "fmt" - "math/rand" - "strings" - "testing" - "time" - - cloudkms "cloud.google.com/go/kms/apiv1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - computealpha "google.golang.org/api/compute/v0.alpha" - computebeta "google.golang.org/api/compute/v0.beta" - compute "google.golang.org/api/compute/v1" - "k8s.io/klog/v2" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" - remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" -) - -var ( - project = flag.String("project", "", "Project to run tests in") - serviceAccount = flag.String("service-account", "", "Service account to bring up instance with") - architecture = flag.String("arch", "amd64", "Architecture pd csi driver build on") - zones = flag.String("zones", "us-east4-a,us-east4-c", "Zones to run tests in. If there are multiple zones, separate each by comma") - machineType = flag.String("machine-type", "n2-standard-2", "Type of machine to provision instance on") - imageURL = flag.String("image-url", "projects/debian-cloud/global/images/family/debian-11", "OS image url to get image from") - runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") - deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") - - testContexts = []*remote.TestContext{} - computeService *compute.Service - computeAlphaService *computealpha.Service - computeBetaService *computebeta.Service - kmsClient *cloudkms.KeyManagementClient -) - -func init() { - klog.InitFlags(flag.CommandLine) -} - -func TestE2E(t *testing.T) { - flag.Parse() - RegisterFailHandler(Fail) - RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests") -} - -var _ = BeforeSuite(func() { - var err error - tcc := make(chan *remote.TestContext) - defer close(tcc) - - zones := strings.Split(*zones, ",") - - rand.Seed(time.Now().UnixNano()) - - computeService, err = remote.GetComputeClient() - Expect(err).To(BeNil()) - - computeAlphaService, err = remote.GetComputeAlphaClient() - Expect(err).To(BeNil()) - - computeBetaService, err = remote.GetComputeBetaClient() - Expect(err).To(BeNil()) - - // Create the KMS client. - kmsClient, err = cloudkms.NewKeyManagementClient(context.Background()) - Expect(err).To(BeNil()) - - if *runInProw { - *project, *serviceAccount = testutils.SetupProwConfig("gce-project") - } - - Expect(*project).ToNot(BeEmpty(), "Project should not be empty") - Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty") - - klog.Infof("Running in project %v with service account %v", *project, *serviceAccount) - - for _, zone := range zones { - go func(curZone string) { - defer GinkgoRecover() - tcc <- NewTestContext(curZone) - }(zone) - } - - for i := 0; i < len(zones); i++ { - tc := <-tcc - testContexts = append(testContexts, tc) - klog.Infof("Added TestContext for node %s", tc.Instance.GetName()) - } -}) - -var _ = AfterSuite(func() { - for _, tc := range testContexts { - err := remote.TeardownDriverAndClient(tc) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - if *deleteInstances { - tc.Instance.DeleteInstance() - } - } -}) - -func NewTestContext(zone string) *remote.TestContext { - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) - klog.Infof("Setting up node %s", nodeID) - - i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService) - if err != nil { - klog.Fatalf("Failed to setup instance %v: %w", nodeID, err) - } - - err = testutils.MkdirAll(i, "/lib/udev_containerized") - if err != nil { - klog.Fatalf("Failed to make scsi_id containerized directory: %w", err) - } - - err = testutils.CopyFile(i, "/lib/udev/scsi_id", "/lib/udev_containerized/scsi_id") - if err != nil { - klog.Fatalf("Failed to copy scsi_id to containerized directory: %w", err) - } - - err = testutils.CopyFile(i, "/lib/udev/google_nvme_id", "/lib/udev_containerized/google_nvme_id") - if err != nil { - klog.Fatalf("Failed to copy google_nvme_id to containerized directory: %w", err) - } - - klog.Infof("Creating new driver and client for node %s", i.GetName()) - tc, err := testutils.GCEClientAndDriverSetup(i, "") - if err != nil { - klog.Fatalf("Failed to set up TestContext for instance %v: %w", i.GetName(), err) - } - - klog.Infof("Finished creating TestContext for node %s", tc.Instance.GetName()) - return tc -} - -func getRandomTestContext() *remote.TestContext { - Expect(testContexts).ToNot(BeEmpty()) - rn := rand.Intn(len(testContexts)) - return testContexts[rn] -} diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go deleted file mode 100644 index e337f117..00000000 --- a/test/e2e/tests/single_zone_e2e_test.go +++ /dev/null @@ -1,1519 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tests - -import ( - "context" - "fmt" - "path/filepath" - "regexp" - "strings" - "time" - - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" - - csi "github.com/container-storage-interface/spec/lib/go/csi" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/iterator" - kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1" - fieldmask "google.golang.org/genproto/protobuf/field_mask" -) - -const ( - testNamePrefix = "gcepd-csi-e2e-" - - defaultSizeGb int64 = 5 - defaultExtremeSizeGb int64 = 500 - defaultHdTSizeGb int64 = 2048 - defaultRepdSizeGb int64 = 200 - defaultMwSizeGb int64 = 200 - defaultVolumeLimit int64 = 127 - readyState = "READY" - standardDiskType = "pd-standard" - extremeDiskType = "pd-extreme" - hdtDiskType = "hyperdisk-throughput" - provisionedIOPSOnCreate = "12345" - provisionedIOPSOnCreateInt = int64(12345) - provisionedIOPSOnCreateDefaultInt = int64(100000) - provisionedThroughputOnCreate = "66Mi" - provisionedThroughputOnCreateInt = int64(66) - defaultEpsilon = 500000000 // 500M -) - -var _ = Describe("GCE PD CSI Driver", func() { - - It("Should get reasonable volume limits from nodes with NodeGetInfo", func() { - testContext := getRandomTestContext() - resp, err := testContext.Client.NodeGetInfo() - Expect(err).To(BeNil()) - volumeLimit := resp.GetMaxVolumesPerNode() - Expect(volumeLimit).To(Equal(defaultVolumeLimit)) - }) - - It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - }) - - It("Should automatically fix the symlink between /dev/* and /dev/by-id if the disk does not match", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err := client.ControllerPublishVolume(volID, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID()) - - defer func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - - }() - - // MESS UP THE symlink - devicePaths := deviceutils.NewDeviceUtils().GetDiskByIdPaths(volName, "") - for _, devicePath := range devicePaths { - err = testutils.RmAll(instance, devicePath) - Expect(err).To(BeNil(), "failed to remove /dev/by-id folder") - err = testutils.Symlink(instance, "/dev/null", devicePath) - Expect(err).To(BeNil(), "failed to add invalid symlink /dev/by-id folder") - } - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) - Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") - - // Validate that the link is correct - var validated bool - for _, devicePath := range devicePaths { - validated, err = testutils.ValidateLogicalLinkIsDisk(instance, devicePath, volName) - Expect(err).To(BeNil(), "failed to validate link %s is disk %s: %v", stageDir, volName, err) - if validated { - break - } - } - Expect(validated).To(BeTrue(), "could not find device in %v that links to volume %s", devicePaths, volName) - - defer func() { - // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - }() - }) - - It("Should automatically add a symlink between /dev/* and /dev/by-id if disk is not found", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err := client.ControllerPublishVolume(volID, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID()) - - defer func() { - // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) - if err != nil { - klog.Errorf("Failed to detach disk: %w", err) - } - - }() - - // DELETE THE symlink - devicePaths := deviceutils.NewDeviceUtils().GetDiskByIdPaths(volName, "") - for _, devicePath := range devicePaths { - err = testutils.RmAll(instance, devicePath) - Expect(err).To(BeNil(), "failed to remove /dev/by-id folder") - } - - // Stage Disk - stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) - Expect(err).To(BeNil(), "failed to repair /dev/by-id symlink and stage volume") - - // Validate that the link is correct - var validated bool - for _, devicePath := range devicePaths { - validated, err = testutils.ValidateLogicalLinkIsDisk(instance, devicePath, volName) - Expect(err).To(BeNil(), "failed to validate link %s is disk %s: %v", stageDir, volName, err) - if validated { - break - } - } - Expect(validated).To(BeTrue(), "could not find device in %v that links to volume %s", devicePaths, volName) - - defer func() { - // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) - if err != nil { - klog.Errorf("Failed to unstage volume: %w", err) - } - fp := filepath.Join("/tmp/", volName) - err = testutils.RmAll(instance, fp) - if err != nil { - klog.Errorf("Failed to rm file path %s: %w", fp, err) - } - }() - }) - - It("Should create disks in correct zones when topology is specified", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, _, _ := testContext.Instance.GetIdentity() - - zones := []string{"us-central1-c", "us-central1-b", "us-central1-a"} - - for _, zone := range zones { - volName := testNamePrefix + string(uuid.NewUUID()) - topReq := &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: zone}, - }, - }, - } - volume, err := testContext.Client.CreateVolume(volName, nil, defaultSizeGb, topReq, nil) - Expect(err).To(BeNil(), "Failed to create volume") - defer func() { - err = testContext.Client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "Failed to delete volume") - }() - - _, err = computeService.Disks.Get(p, zone, volName).Do() - Expect(err).To(BeNil(), "Could not find disk in correct zone") - } - }) - - DescribeTable("Should complete entire disk lifecycle with underspecified volume ID", - func(diskType string) { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - volName, _ := createAndValidateUniqueZonalDisk(client, p, z, diskType) - - underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) - - defer func() { - // Delete Disk - err := client.DeleteVolume(underSpecifiedID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - Entry("on hyperdisk-throughput", hdtDiskType), - ) - - DescribeTable("Should complete publish/unpublish lifecycle with underspecified volume ID and missing volume", - func(diskType string) { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create Disk - volName, _ := createAndValidateUniqueZonalDisk(client, p, z, diskType) - underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) - - defer func() { - // Detach Disk - err := instance.DetachDisk(volName) - Expect(err).To(BeNil(), "DetachDisk failed") - - // Delete Disk - err = client.DeleteVolume(underSpecifiedID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - - // Unpublish Disk - err = client.ControllerUnpublishVolume(underSpecifiedID, instance.GetNodeID()) - Expect(err).To(BeNil(), "ControllerUnpublishVolume failed") - }() - - // Attach Disk - err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID(), false /* forceAttach */) - Expect(err).To(BeNil(), "ControllerPublishVolume failed") - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - ) - - It("Should successfully create RePD in two zones in the drivers region when none are specified", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones([]string{z}) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, nil, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - for _, replicaZone := range cloudDisk.ReplicaZones { - actualZone := zoneFromURL(replicaZone) - gotRegion, err := common.GetRegionFromZones([]string{actualZone}) - Expect(err).To(BeNil(), "failed to get region from actual zone %v", actualZone) - Expect(gotRegion).To(Equal(region), "Got region from replica zone that did not match supplied region") - } - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) - - DescribeTable("Should create and delete disk with default zone", - func(diskType string) { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - - diskSize := defaultSizeGb - if diskType == extremeDiskType { - diskSize = defaultExtremeSizeGb - } - - volume, err := client.CreateVolume(volName, disk.params, diskSize, nil, nil) - - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(diskSize)) - Expect(cloudDisk.Name).To(Equal(volName)) - disk.validate(cloudDisk) - - defer func() { - // Delete Disk - client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - ) - - DescribeTable("Should create and delete pd-extreme disk with default iops", - func(diskType string) { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - diskParams := map[string]string{ - common.ParameterKeyType: diskType, - } - volName := testNamePrefix + string(uuid.NewUUID()) - - diskSize := defaultExtremeSizeGb - - volume, err := client.CreateVolume(volName, diskParams, diskSize, nil, nil) - - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultExtremeSizeGb)) - Expect(cloudDisk.Type).To(ContainSubstring(extremeDiskType)) - Expect(cloudDisk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateDefaultInt)) - Expect(cloudDisk.Name).To(Equal(volName)) - - defer func() { - // Delete Disk - client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }, - Entry("on pd-extreme", extremeDiskType), - ) - - DescribeTable("Should create and delete disk with labels", - func(diskType string) { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - params := merge(disk.params, map[string]string{ - common.ParameterKeyLabels: "key1=value1,key2=value2", - }) - - diskSize := defaultSizeGb - if diskType == extremeDiskType { - diskSize = defaultExtremeSizeGb - } - volume, err := client.CreateVolume(volName, params, diskSize, nil, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(diskSize)) - Expect(cloudDisk.Labels).To(Equal(map[string]string{ - "key1": "value1", - "key2": "value2", - // The label below is added as an --extra-label driver command line argument. - testutils.DiskLabelKey: testutils.DiskLabelValue, - })) - Expect(cloudDisk.Name).To(Equal(volName)) - disk.validate(cloudDisk) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - ) - - It("Should create and delete snapshot for the volume with default zone", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - // Create Snapshot - snapshotName := testNamePrefix + string(uuid.NewUUID()) - snapshotID, err := client.CreateSnapshot(snapshotName, volID, nil) - Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) - - // Validate Snapshot Created - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - Expect(snapshot.Name).To(Equal(snapshotName)) - - err = wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) { - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - if snapshot.Status == "READY" { - return true, nil - } - return false, nil - }) - Expect(err).To(BeNil(), "Could not wait for snapshot be ready") - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - - // Delete Snapshot - err = client.DeleteSnapshot(snapshotID) - Expect(err).To(BeNil(), "DeleteSnapshot failed") - - // Validate Snapshot Deleted - _, err = computeService.Snapshots.Get(p, snapshotName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") - }() - }) - - DescribeTable("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", - func(diskType string) { - ctx := context.Background() - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - locationID := "global" - - // The resource name of the key rings. - parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID) - keyRingId := "gce-pd-csi-test-ring" - - key, keyVersions := setupKeyRing(ctx, parentName, keyRingId) - - // Defer deletion of all key versions - // https://cloud.google.com/kms/docs/destroy-restore - defer func() { - for _, keyVersion := range keyVersions { - destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{ - Name: keyVersion, - } - _, err := kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) - Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion) - } - }() - - // Go through volume lifecycle using CMEK-ed PD Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - params := merge(disk.params, map[string]string{ - common.ParameterKeyDiskEncryptionKmsKey: key.Name, - }) - topology := &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: z}, - }, - }, - } - - diskSize := defaultSizeGb - if diskType == extremeDiskType { - diskSize = defaultExtremeSizeGb - } - volume, err := controllerClient.CreateVolume(volName, params, diskSize, topology, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(diskSize)) - Expect(cloudDisk.Name).To(Equal(volName)) - disk.validate(cloudDisk) - - defer func() { - // Delete Disk - err = controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Test disk works - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") - - // Revoke CMEK key - // https://cloud.google.com/kms/docs/enable-disable - - for _, keyVersion := range keyVersions { - disableReq := &kmspb.UpdateCryptoKeyVersionRequest{ - CryptoKeyVersion: &kmspb.CryptoKeyVersion{ - Name: keyVersion, - State: kmspb.CryptoKeyVersion_DISABLED, - }, - UpdateMask: &fieldmask.FieldMask{ - Paths: []string{"state"}, - }, - } - _, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq) - Expect(err).To(BeNil(), "Failed to disable crypto key") - } - - // Make sure attach of PD fails - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") - - // Restore CMEK key - for _, keyVersion := range keyVersions { - enableReq := &kmspb.UpdateCryptoKeyVersionRequest{ - CryptoKeyVersion: &kmspb.CryptoKeyVersion{ - Name: keyVersion, - State: kmspb.CryptoKeyVersion_ENABLED, - }, - UpdateMask: &fieldmask.FieldMask{ - Paths: []string{"state"}, - }, - } - _, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq) - Expect(err).To(BeNil(), "Failed to enable crypto key") - } - - // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. - time.Sleep(time.Second) - // Make sure attach of PD succeeds - err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - ) - - It("Should create disks, attach them places, and verify List returns correct results", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - nodeID := testContext.Instance.GetNodeID() - - _, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - defer deleteVolumeOrError(client, volID) - - _, secondVolID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - defer deleteVolumeOrError(client, secondVolID) - - // Attach volID to current instance - err := client.ControllerPublishVolume(volID, nodeID, false /* forceAttach */) - Expect(err).To(BeNil(), "Failed ControllerPublishVolume") - defer client.ControllerUnpublishVolume(volID, nodeID) - - // List Volumes - volsToNodes, err := client.ListVolumes() - Expect(err).To(BeNil(), "Failed ListVolumes") - - // Verify - Expect(volsToNodes[volID]).ToNot(BeNil(), "Couldn't find attached nodes for vol") - Expect(volsToNodes[volID]).To(ContainElement(nodeID), "Couldn't find node in attached nodes for vol") - Expect(volsToNodes[secondVolID]).To(BeNil(), "Second vol ID attached nodes not nil") - }) - - It("Should create and delete snapshot for RePD in two zones ", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones([]string{z}) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, nil, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - for _, replicaZone := range cloudDisk.ReplicaZones { - actualZone := zoneFromURL(replicaZone) - gotRegion, err := common.GetRegionFromZones([]string{actualZone}) - Expect(err).To(BeNil(), "failed to get region from actual zone %v", actualZone) - Expect(gotRegion).To(Equal(region), "Got region from replica zone that did not match supplied region") - } - - // Create Snapshot - snapshotName := testNamePrefix + string(uuid.NewUUID()) - snapshotID, err := controllerClient.CreateSnapshot(snapshotName, volume.VolumeId, nil) - Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) - - // Validate Snapshot Created - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - Expect(snapshot.Name).To(Equal(snapshotName)) - - err = wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) { - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - if snapshot.Status == "READY" { - return true, nil - } - return false, nil - }) - Expect(err).To(BeNil(), "Could not wait for snapshot be ready") - - defer func() { - // Delete Disk - err := controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - - // Delete Snapshot - err = controllerClient.DeleteSnapshot(snapshotID) - Expect(err).To(BeNil(), "DeleteSnapshot failed") - - // Validate Snapshot Deleted - _, err = computeService.Snapshots.Get(p, snapshotName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") - }() - }) - - It("Should get correct VolumeStats for Block", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - verifyVolumeStats := func(a *verifyArgs) error { - available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) - if err != nil { - return fmt.Errorf("failed to get node volume stats: %v", err.Error()) - } - if available != 0 || capacity != common.GbToBytes(defaultSizeGb) || used != 0 || - inodesFree != 0 || inodes != 0 || inodesUsed != 0 { - return fmt.Errorf("got: available %v, capacity %v, used %v, inodesFree %v, inodes %v, inodesUsed %v -- expected: capacity = %v, available = 0, used = 0, inodesFree = 0, inodes = 0 , inodesUsed = 0", - available, capacity, used, inodesFree, inodes, inodesUsed, common.GbToBytes(defaultSizeGb)) - } - return nil - } - - // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - }) - - It("Should get correct VolumeStats", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - verifyVolumeStats := func(a *verifyArgs) error { - available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) - if err != nil { - return fmt.Errorf("failed to get node volume stats: %v", err.Error()) - } - if !equalWithinEpsilon(available, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(capacity, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(used, 0, defaultEpsilon) || - inodesFree == 0 || inodes == 0 || inodesUsed == 0 { - return fmt.Errorf("got: available %v, capacity %v, used %v, inodesFree %v, inodes %v, inodesUsed %v -- expected: available ~= %v, capacity ~= %v, used = 0, inodesFree != 0, inodes != 0 , inodesUsed != 0", - available, capacity, used, inodesFree, inodes, inodesUsed, common.GbToBytes(defaultSizeGb), common.GbToBytes(defaultSizeGb)) - } - return nil - } - - // Attach Disk - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - }) - - // Pending while multi-writer feature is in Alpha - PIt("Should create and delete multi-writer disk", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - p, _, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Hardcode to us-east1-a while feature is in alpha - zone := "us-east1-a" - - // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, zone, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeAlphaService.Disks.Get(p, zone, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) - - // Pending while multi-writer feature is in Alpha - PIt("Should complete entire disk lifecycle with multi-writer disk", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - instance := testContext.Instance - - // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, z, standardDiskType) - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - - // Attach Disk - testFileContents := "test" - writeFunc := func(a *verifyArgs) error { - err := testutils.WriteBlock(instance, a.publishDir, testFileContents) - if err != nil { - return fmt.Errorf("Failed to write file: %v", err.Error()) - } - return nil - } - verifyReadFunc := func(a *verifyArgs) error { - readContents, err := testutils.ReadBlock(instance, a.publishDir, len(testFileContents)) - if err != nil { - return fmt.Errorf("ReadFile failed with error: %v", err.Error()) - } - if strings.TrimSpace(string(readContents)) != testFileContents { - return fmt.Errorf("wanted test file content: %s, got content: %s", testFileContents, readContents) - } - return nil - } - err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, writeFunc, verifyReadFunc) - Expect(err).To(BeNil(), "Failed to go through volume lifecycle") - }) - - DescribeTable("Should successfully create disk with PVC/PV tags", - func(diskType string) { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - diskSize := defaultSizeGb - if diskType == extremeDiskType { - diskSize = defaultExtremeSizeGb - } - - p, z, _ := controllerInstance.GetIdentity() - - // Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - params := merge(disk.params, map[string]string{ - common.ParameterKeyPVCName: "test-pvc", - common.ParameterKeyPVCNamespace: "test-pvc-namespace", - common.ParameterKeyPVName: "test-pv-name", - }) - volume, err := controllerClient.CreateVolume(volName, params, diskSize, nil /* topReq */, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(diskSize)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(cloudDisk.Description).To(Equal("{\"kubernetes.io/created-for/pv/name\":\"test-pv-name\",\"kubernetes.io/created-for/pvc/name\":\"test-pvc\",\"kubernetes.io/created-for/pvc/namespace\":\"test-pvc-namespace\",\"storage.gke.io/created-by\":\"pd.csi.storage.gke.io\"}")) - disk.validate(cloudDisk) - - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }, - Entry("on pd-standard", standardDiskType), - Entry("on pd-extreme", extremeDiskType), - ) - - // Use the region of the test location. - It("Should successfully create snapshot with storage locations", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - // Create Snapshot - snapshotName := testNamePrefix + string(uuid.NewUUID()) - - // Convert GCP zone to region, e.g. us-central1-a => us-central1 - // This is safe because we hardcode the zones. - snapshotLocation := z[:len(z)-2] - - snapshotParams := map[string]string{ - common.ParameterKeyStorageLocations: snapshotLocation, - common.ParameterKeyVolumeSnapshotName: "test-volumesnapshot-name", - common.ParameterKeyVolumeSnapshotNamespace: "test-volumesnapshot-namespace", - common.ParameterKeyVolumeSnapshotContentName: "test-volumesnapshotcontent-name", - } - snapshotID, err := client.CreateSnapshot(snapshotName, volID, snapshotParams) - Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) - - // Validate Snapshot Created - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - Expect(snapshot.Name).To(Equal(snapshotName)) - Expect(snapshot.Description).To(Equal("{\"kubernetes.io/created-for/volumesnapshot/name\":\"test-volumesnapshot-name\",\"kubernetes.io/created-for/volumesnapshot/namespace\":\"test-volumesnapshot-namespace\",\"kubernetes.io/created-for/volumesnapshotcontent/name\":\"test-volumesnapshotcontent-name\",\"storage.gke.io/created-by\":\"pd.csi.storage.gke.io\"}")) - - err = wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) { - snapshot, err := computeService.Snapshots.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - if snapshot.Status == "READY" { - return true, nil - } - return false, nil - }) - Expect(err).To(BeNil(), "Could not wait for snapshot be ready") - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - - // Delete Snapshot - err = client.DeleteSnapshot(snapshotID) - Expect(err).To(BeNil(), "DeleteSnapshot failed") - - // Validate Snapshot Deleted - _, err = computeService.Snapshots.Get(p, snapshotName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") - }() - }) - - // Use the region of the test location. - It("Should successfully create snapshot backed by disk image", func() { - testContext := getRandomTestContext() - - p, z, _ := testContext.Instance.GetIdentity() - client := testContext.Client - - // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) - - // Create Snapshot - snapshotName := testNamePrefix + string(uuid.NewUUID()) - testImageFamily := "test-family" - - snapshotParams := map[string]string{common.ParameterKeySnapshotType: common.DiskImageType, common.ParameterKeyImageFamily: testImageFamily} - snapshotID, err := client.CreateSnapshot(snapshotName, volID, snapshotParams) - Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) - - // Validate Snapshot Created - snapshot, err := computeService.Images.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - Expect(snapshot.Name).To(Equal(snapshotName)) - - err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { - snapshot, err := computeService.Images.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - if snapshot.Status == "READY" { - return true, nil - } - return false, nil - }) - Expect(err).To(BeNil(), "Could not wait for snapshot be ready") - - // Check Snapshot Type - snapshot, err = computeService.Images.Get(p, snapshotName).Do() - Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") - _, snapshotType, _, err := common.SnapshotIDToProjectKey(cleanSelfLink(snapshot.SelfLink)) - Expect(err).To(BeNil(), "Failed to parse snapshot ID") - Expect(snapshotType).To(Equal(common.DiskImageType), "Expected images type in snapshot ID") - - defer func() { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - - // Delete Snapshot - err = client.DeleteSnapshot(snapshotID) - Expect(err).To(BeNil(), "DeleteSnapshot failed") - - // Validate Snapshot Deleted - _, err = computeService.Images.Get(p, snapshotName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") - }() - }) - - It("Should successfully create zonal PD from a zonal PD VolumeContentSource", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - - // Create Source Disk - _, srcVolID := createAndValidateUniqueZonalDisk(controllerClient, p, z, standardDiskType) - - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "none", - }, defaultSizeGb, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: z}, - }, - }, - }, - &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: srcVolID, - }, - }, - }) - - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - // Validate the the clone disk zone matches the source disk zone. - _, srcKey, err := common.VolumeIDToKey(srcVolID) - Expect(err).To(BeNil(), "Could not get source volume key from id") - Expect(zoneFromURL(cloudDisk.Zone)).To(Equal(srcKey.Zone)) - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.Disks.Get(p, z, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) - - It("Should successfully create RePD from a zonal PD VolumeContentSource", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones([]string{z}) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Source Disk - srcVolName := testNamePrefix + string(uuid.NewUUID()) - srcVolume, err := controllerClient.CreateVolume(srcVolName, map[string]string{ - common.ParameterKeyReplicationType: "none", - }, defaultRepdSizeGb, nil, nil) - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, nil, - &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: srcVolume.VolumeId, - }, - }, - }) - - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - replicaZonesCompatible := false - _, srcKey, err := common.VolumeIDToKey(srcVolume.VolumeId) - Expect(err).To(BeNil(), "Could not get source volume key from id") - for _, replicaZone := range cloudDisk.ReplicaZones { - actualZone := zoneFromURL(replicaZone) - if actualZone == srcKey.Zone { - replicaZonesCompatible = true - } - gotRegion, err := common.GetRegionFromZones([]string{actualZone}) - Expect(err).To(BeNil(), "failed to get region from actual zone %v", actualZone) - Expect(gotRegion).To(Equal(region), "Got region from replica zone that did not match supplied region") - } - // Validate that one of the replicaZones of the clone matches the zone of the source disk. - Expect(replicaZonesCompatible).To(Equal(true)) - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) - - It("Should successfully create RePD from a RePD VolumeContentSource", func() { - Expect(testContexts).ToNot(BeEmpty()) - testContext := getRandomTestContext() - - controllerInstance := testContext.Instance - controllerClient := testContext.Client - - p, z, _ := controllerInstance.GetIdentity() - - region, err := common.GetRegionFromZones([]string{z}) - Expect(err).To(BeNil(), "Failed to get region from zones") - - // Create Source Disk - srcVolName := testNamePrefix + string(uuid.NewUUID()) - srcVolume, err := controllerClient.CreateVolume(srcVolName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, nil, nil) - // Create Disk - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := controllerClient.CreateVolume(volName, map[string]string{ - common.ParameterKeyReplicationType: "regional-pd", - }, defaultRepdSizeGb, nil, - &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: srcVolume.VolumeId, - }, - }, - }) - - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) - // Validate that the replicaZones of the clone match the replicaZones of the source disk. - srcCloudDisk, err := computeService.RegionDisks.Get(p, region, srcVolName).Do() - Expect(err).To(BeNil(), "Could not get source disk from cloud directly") - Expect(srcCloudDisk.ReplicaZones).To(Equal(cloudDisk.ReplicaZones)) - for _, replicaZone := range cloudDisk.ReplicaZones { - actualZone := zoneFromURL(replicaZone) - gotRegion, err := common.GetRegionFromZones([]string{actualZone}) - Expect(err).To(BeNil(), "failed to get region from actual zone %v", actualZone) - Expect(gotRegion).To(Equal(region), "Got region from replica zone that did not match supplied region") - } - defer func() { - // Delete Disk - controllerClient.DeleteVolume(volume.VolumeId) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - _, err = computeService.RegionDisks.Get(p, region, volName).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") - }() - }) - - It("Should pass/fail if valid/invalid compute endpoint is passed in", func() { - // gets instance set up w/o compute-endpoint set from test setup - _, err := getRandomTestContext().Client.ListVolumes() - Expect(err).To(BeNil(), "no error expected when passed valid compute url") - - zone := "us-central1-c" - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) - i, err := remote.SetupInstance(*project, *architecture, zone, nodeID, *machineType, *serviceAccount, *imageURL, computeService) - - if err != nil { - klog.Fatalf("Failed to setup instance %v: %w", nodeID, err) - } - - klog.Infof("Creating new driver and client for node %s\n", i.GetName()) - - // Create new driver and client w/ invalid endpoint - tcInvalid, err := testutils.GCEClientAndDriverSetup(i, "invalid-string") - if err != nil { - klog.Fatalf("Failed to set up Test Context for instance %v: %w", i.GetName(), err) - } - - _, err = tcInvalid.Client.ListVolumes() - Expect(err.Error()).To(ContainSubstring("no such host"), "expected error when passed invalid compute url") - - // Create new driver and client w/ valid, passed-in endpoint - tcValid, err := testutils.GCEClientAndDriverSetup(i, "https://compute.googleapis.com") - if err != nil { - klog.Fatalf("Failed to set up Test Context for instance %v: %w", i.GetName(), err) - } - _, err = tcValid.Client.ListVolumes() - - Expect(err).To(BeNil(), "no error expected when passed valid compute url") - }) -}) - -func equalWithinEpsilon(a, b, epsiolon int64) bool { - if a > b { - return a-b < epsiolon - } - return b-a < epsiolon -} - -func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) { - // Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - - diskSize := defaultSizeGb - switch diskType { - case extremeDiskType: - diskSize = defaultExtremeSizeGb - case hdtDiskType: - diskSize = defaultHdTSizeGb - } - volume, err := client.CreateVolume(volName, disk.params, diskSize, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: zone}, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do() - Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(diskSize)) - Expect(cloudDisk.Name).To(Equal(volName)) - disk.validate(cloudDisk) - - return volName, volume.VolumeId -} - -func deleteVolumeOrError(client *remote.CsiClient, volID string) { - // Delete Disk - err := client.DeleteVolume(volID) - Expect(err).To(BeNil(), "DeleteVolume failed") - - // Validate Disk Deleted - project, key, err := common.VolumeIDToKey(volID) - Expect(err).To(BeNil(), "Failed to conver volume ID To key") - _, err = computeService.Disks.Get(project, key.Zone, key.Name).Do() - Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") -} - -func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) { - // Create Disk - disk := typeToDisk[diskType] - volName := testNamePrefix + string(uuid.NewUUID()) - volume, err := client.CreateVolumeWithCaps(volName, disk.params, defaultMwSizeGb, - &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{common.TopologyKeyZone: zone}, - }, - }, - }, - []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Block{ - Block: &csi.VolumeCapability_BlockVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, - }, - }, - }, nil) - Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) - - // Validate Disk Created - cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do() - Expect(err).To(BeNil(), "Failed to get cloud disk") - Expect(cloudDisk.Status).To(Equal(readyState)) - Expect(cloudDisk.SizeGb).To(Equal(defaultMwSizeGb)) - Expect(cloudDisk.Name).To(Equal(volName)) - disk.validate(cloudDisk) - - alphaDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do() - Expect(err).To(BeNil(), "Failed to get cloud disk using alpha API") - Expect(alphaDisk.MultiWriter).To(Equal(true)) - - return volName, volume.VolumeId -} - -func cleanSelfLink(selfLink string) string { - r, _ := regexp.Compile("https:\\/\\/www.*apis.com\\/.*(v1|beta|alpha)\\/") - return r.ReplaceAllString(selfLink, "") -} - -// Returns the zone from the URL with the format https://compute.googleapis.com/compute/v1/projects/{project}/zones/{zone}. -// Returns the empty string if the zone cannot be abstracted from the URL. -func zoneFromURL(url string) string { - tokens := strings.Split(url, "/") - if len(tokens) == 0 { - return "" - } - return tokens[len(tokens)-1] -} - -func setupKeyRing(ctx context.Context, parentName string, keyRingId string) (*kmspb.CryptoKey, []string) { - // Create KeyRing - ringReq := &kmspb.CreateKeyRingRequest{ - Parent: parentName, - KeyRingId: keyRingId, - } - keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq) - if !gce.IsGCEError(err, "alreadyExists") { - getKeyRingReq := &kmspb.GetKeyRingRequest{ - Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId), - } - keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq) - - } - Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId) - - // Create CryptoKey in KeyRing - keyId := "test-key-" + string(uuid.NewUUID()) - keyReq := &kmspb.CreateCryptoKeyRequest{ - Parent: keyRing.Name, - CryptoKeyId: keyId, - CryptoKey: &kmspb.CryptoKey{ - Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT, - VersionTemplate: &kmspb.CryptoKeyVersionTemplate{ - Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION, - }, - }, - } - key, err := kmsClient.CreateCryptoKey(ctx, keyReq) - Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name) - - keyVersions := []string{} - keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{ - Parent: key.Name, - } - - it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq) - - for { - keyVersion, err := it.Next() - if err == iterator.Done { - break - } - Expect(err).To(BeNil(), "Failed to list crypto key versions") - - keyVersions = append(keyVersions, keyVersion.Name) - } - return key, keyVersions -} - -type disk struct { - params map[string]string - validate func(disk *compute.Disk) -} - -var typeToDisk = map[string]*disk{ - standardDiskType: { - params: map[string]string{ - common.ParameterKeyType: standardDiskType, - }, - validate: func(disk *compute.Disk) { - Expect(disk.Type).To(ContainSubstring(standardDiskType)) - }, - }, - extremeDiskType: { - params: map[string]string{ - common.ParameterKeyType: extremeDiskType, - common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, - }, - validate: func(disk *compute.Disk) { - Expect(disk.Type).To(ContainSubstring(extremeDiskType)) - Expect(disk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) - }, - }, - hdtDiskType: { - params: map[string]string{ - common.ParameterKeyType: hdtDiskType, - common.ParameterKeyProvisionedThroughputOnCreate: provisionedThroughputOnCreate, - }, - validate: func(disk *compute.Disk) { - Expect(disk.Type).To(ContainSubstring(hdtDiskType)) - Expect(disk.ProvisionedThroughput).To(Equal(provisionedThroughputOnCreateInt)) - }, - }, -} - -func merge(a, b map[string]string) map[string]string { - res := map[string]string{} - for k, v := range a { - res[k] = v - } - for k, v := range b { - res[k] = v - } - return res -} diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go deleted file mode 100644 index 97364bed..00000000 --- a/test/e2e/utils/utils.go +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "context" - "fmt" - "math/rand" - "os" - "path" - "regexp" - "strconv" - "strings" - "time" - - "golang.org/x/oauth2/google" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - "k8s.io/klog/v2" - boskosclient "sigs.k8s.io/boskos/client" - "sigs.k8s.io/boskos/common" - utilcommon "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" -) - -const ( - DiskLabelKey = "csi" - DiskLabelValue = "e2e-test" -) - -var ( - boskos, _ = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos", "", "") -) - -func GCEClientAndDriverSetup(instance *remote.InstanceInfo, computeEndpoint string) (*remote.TestContext, error) { - port := fmt.Sprintf("%v", 1024+rand.Intn(10000)) - goPath, ok := os.LookupEnv("GOPATH") - if !ok { - return nil, fmt.Errorf("Could not find environment variable GOPATH") - } - pkgPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/") - binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver") - - endpoint := fmt.Sprintf("tcp://localhost:%s", port) - extra_flags := []string{ - fmt.Sprintf("--extra-labels=%s=%s", DiskLabelKey, DiskLabelValue), - "--max-concurrent-format-and-mount=20", // otherwise the serialization times out the e2e test. - } - if computeEndpoint != "" { - extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint %s", computeEndpoint)) - } - workspace := remote.NewWorkspaceDir("gce-pd-e2e-") - // Log at V(6) as the compute API calls are emitted at that level and it's - // useful to see what's happening when debugging tests. - driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver -v=6 --endpoint=%s %s 2> %s/prog.out < /dev/null > /dev/null &'", - workspace, endpoint, strings.Join(extra_flags, " "), workspace) - - config := &remote.ClientConfig{ - PkgPath: pkgPath, - BinPath: binPath, - WorkspaceDir: workspace, - RunDriverCmd: driverRunCmd, - Port: port, - } - - err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", "latest") - if err != nil { - return nil, err - } - - return remote.SetupNewDriverAndClient(instance, config) -} - -// getBoskosProject retries acquiring a boskos project until success or timeout -func getBoskosProject(resourceType string) *common.Resource { - timer := time.NewTimer(30 * time.Minute) - defer timer.Stop() - ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-timer.C: - klog.Fatalf("timed out trying to acquire boskos project") - case <-ticker.C: - p, err := boskos.Acquire(resourceType, "free", "busy") - if err != nil { - klog.Warningf("boskos failed to acquire project: %w", err) - } else if p == nil { - klog.Warningf("boskos does not have a free %s at the moment", resourceType) - } else { - return p - } - } - } - -} - -func SetupProwConfig(resourceType string) (project, serviceAccount string) { - // Try to get a Boskos project - klog.V(4).Infof("Running in PROW") - klog.V(4).Infof("Fetching a Boskos loaned project") - - p := getBoskosProject(resourceType) - project = p.Name - - go func(c *boskosclient.Client, proj string) { - for range time.Tick(time.Minute * 5) { - if err := c.UpdateOne(p.Name, "busy", nil); err != nil { - klog.Warningf("[Boskos] Update %s failed with %v", p.Name, err) - } - } - }(boskos, p.Name) - - // If we're on CI overwrite the service account - klog.V(4).Infof("Fetching the default compute service account") - - c, err := google.DefaultClient(context.Background(), cloudresourcemanager.CloudPlatformScope) - if err != nil { - klog.Fatalf("Failed to get Google Default Client: %w", err) - } - - cloudresourcemanagerService, err := cloudresourcemanager.New(c) - if err != nil { - klog.Fatalf("Failed to create new cloudresourcemanager: %w", err) - } - - resp, err := cloudresourcemanagerService.Projects.Get(project).Do() - if err != nil { - klog.Fatalf("Failed to get project %v from Cloud Resource Manager: %w", project, err) - } - - // Default Compute Engine service account - // [PROJECT_NUMBER]-compute@developer.gserviceaccount.com - serviceAccount = fmt.Sprintf("%v-compute@developer.gserviceaccount.com", resp.ProjectNumber) - klog.Infof("Using project %v and service account %v", project, serviceAccount) - return project, serviceAccount -} - -func ForceChmod(instance *remote.InstanceInfo, filePath string, perms string) error { - originalumask, err := instance.SSHNoSudo("umask") - if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", originalumask, err.Error()) - } - output, err := instance.SSHNoSudo("umask", "0000") - if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err.Error()) - } - output, err = instance.SSH("chmod", "-R", perms, filePath) - if err != nil { - return fmt.Errorf("failed to chmod file %s. Output: %v, errror: %v", filePath, output, err.Error()) - } - output, err = instance.SSHNoSudo("umask", originalumask) - if err != nil { - return fmt.Errorf("failed to umask. Output: %v, errror: %v", output, err.Error()) - } - return nil -} - -func WriteFile(instance *remote.InstanceInfo, filePath, fileContents string) error { - output, err := instance.SSHNoSudo("echo", fileContents, ">", filePath) - if err != nil { - return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", filePath, output, err.Error()) - } - return nil -} - -func ReadFile(instance *remote.InstanceInfo, filePath string) (string, error) { - output, err := instance.SSHNoSudo("cat", filePath) - if err != nil { - return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", filePath, output, err.Error()) - } - return output, nil -} - -func WriteBlock(instance *remote.InstanceInfo, path, fileContents string) error { - output, err := instance.SSHNoSudo("echo", fileContents, "|", "dd", "of="+path) - if err != nil { - return fmt.Errorf("failed to write test file %s. Output: %v, errror: %v", path, output, err.Error()) - } - return nil -} - -func ReadBlock(instance *remote.InstanceInfo, path string, length int) (string, error) { - lengthStr := strconv.Itoa(length) - output, err := instance.SSHNoSudo("dd", "if="+path, "bs="+lengthStr, "count=1", "2>", "/dev/null") - if err != nil { - return "", fmt.Errorf("failed to read test file %s. Output: %v, errror: %v", path, output, err.Error()) - } - return output, nil -} - -func GetFSSizeInGb(instance *remote.InstanceInfo, mountPath string) (int64, error) { - output, err := instance.SSH("df", "--output=size", "-BG", mountPath, "|", "awk", "'NR==2'") - if err != nil { - return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", mountPath, output, err.Error()) - } - output = strings.TrimSuffix(strings.TrimSpace(output), "G") - n, err := strconv.ParseInt(output, 10, 64) - if err != nil { - return -1, fmt.Errorf("failed to parse size %s into int", output) - } - return n, nil -} - -func GetBlockSizeInGb(instance *remote.InstanceInfo, devicePath string) (int64, error) { - output, err := instance.SSH("blockdev", "--getsize64", devicePath) - if err != nil { - return -1, fmt.Errorf("failed to get size of path %s. Output: %v, error: %v", devicePath, output, err.Error()) - } - n, err := strconv.ParseInt(strings.TrimSpace(output), 10, 64) - if err != nil { - return -1, fmt.Errorf("failed to parse size %s into int", output) - } - return utilcommon.BytesToGbRoundDown(n), nil -} - -func Symlink(instance *remote.InstanceInfo, src, dest string) error { - output, err := instance.SSH("ln", "-s", src, dest) - if err != nil { - return fmt.Errorf("failed to symlink from %s to %s. Output: %v, errror: %v", src, dest, output, err.Error()) - } - return nil -} - -func RmAll(instance *remote.InstanceInfo, filePath string) error { - output, err := instance.SSH("rm", "-rf", filePath) - if err != nil { - return fmt.Errorf("failed to delete all %s. Output: %v, errror: %v", filePath, output, err.Error()) - } - return nil -} - -func MkdirAll(instance *remote.InstanceInfo, dir string) error { - output, err := instance.SSH("mkdir", "-p", dir) - if err != nil { - return fmt.Errorf("failed to mkdir -p %s. Output: %v, errror: %v", dir, output, err.Error()) - } - return nil -} - -func CopyFile(instance *remote.InstanceInfo, src, dest string) error { - output, err := instance.SSH("cp", src, dest) - if err != nil { - return fmt.Errorf("failed to copy %s to %s. Output: %v, errror: %v", src, dest, output, err.Error()) - } - return nil -} - -// ValidateLogicalLinkIsDisk takes a symlink location at "link" and finds the -// link location - it then finds the backing PD using either scsi_id or -// google_nvme_id (depending on the /dev path) and validates that it is the -// same as diskName -func ValidateLogicalLinkIsDisk(instance *remote.InstanceInfo, link, diskName string) (bool, error) { - const ( - scsiPattern = `^0Google\s+PersistentDisk\s+([\S]+)\s*$` - sdPattern = `sd\w+` - nvmeSerialPattern = `ID_SERIAL_SHORT=([\S]+)\s*` - nvmeDevPattern = `nvme\w+` - ) - // regex to parse scsi_id output and extract the serial - scsiRegex := regexp.MustCompile(scsiPattern) - sdRegex := regexp.MustCompile(sdPattern) - nvmeSerialRegex := regexp.MustCompile(nvmeSerialPattern) - nvmeDevRegex := regexp.MustCompile(nvmeDevPattern) - - devFsPath, err := instance.SSH("find", link, "-printf", "'%l'") - if err != nil { - return false, fmt.Errorf("failed to find symbolic link for %s. Output: %v, errror: %v", link, devFsPath, err.Error()) - } - if len(devFsPath) == 0 { - return false, nil - } - if sdx := sdRegex.FindString(devFsPath); len(sdx) != 0 { - fullDevPath := path.Join("/dev/", string(sdx)) - scsiIDOut, err := instance.SSH("/lib/udev_containerized/scsi_id", "--page=0x83", "--whitelisted", fmt.Sprintf("--device=%v", fullDevPath)) - if err != nil { - return false, fmt.Errorf("failed to find %s's SCSI ID. Output: %v, errror: %v", devFsPath, scsiIDOut, err.Error()) - } - scsiID := scsiRegex.FindStringSubmatch(scsiIDOut) - if len(scsiID) == 0 { - return false, fmt.Errorf("scsi_id output cannot be parsed: %s. Output: %v", scsiID, scsiIDOut) - } - if scsiID[1] != diskName { - return false, fmt.Errorf("scsiID %s did not match expected diskName %s", scsiID, diskName) - } - return true, nil - } else if nvmex := nvmeDevRegex.FindString(devFsPath); len(nvmex) != 0 { - fullDevPath := path.Join("/dev/", string(nvmex)) - nvmeIDOut, err := instance.SSH("/lib/udev_containerized/google_nvme_id", fmt.Sprintf("-d%v", fullDevPath)) - if err != nil { - return false, fmt.Errorf("failed to find %s's NVME ID. Output: %v, errror: %v", devFsPath, nvmeIDOut, err.Error()) - } - nvmeID := nvmeSerialRegex.FindStringSubmatch(nvmeIDOut) - if len(nvmeID) == 0 { - return false, fmt.Errorf("google_nvme_id output cannot be parsed: %s. Output: %v", nvmeID, nvmeIDOut) - } - if nvmeID[1] != diskName { - return false, fmt.Errorf("nvmeID %s did not match expected diskName %s", nvmeID, diskName) - } - return true, nil - } - return false, fmt.Errorf("symlinked disk %s for diskName %s does not match a supported /dev/sd* or /dev/nvme* path", devFsPath, diskName) -} diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go deleted file mode 100644 index 5f63cbaa..00000000 --- a/test/k8s-integration/cluster.go +++ /dev/null @@ -1,493 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - apimachineryversion "k8s.io/apimachinery/pkg/version" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" -) - -func gkeLocationArgs(gceZone, gceRegion string) (locationArg, locationVal string, err error) { - switch { - case len(gceZone) > 0: - locationArg = "--zone" - locationVal = gceZone - case len(gceRegion) > 0: - locationArg = "--region" - locationVal = gceRegion - default: - return "", "", fmt.Errorf("zone and region unspecified") - } - return -} - -func isRegionalGKECluster(gceZone, gceRegion string) bool { - return len(gceRegion) > 0 -} - -func clusterDownGCE(k8sDir string) error { - cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) - cmd.Env = os.Environ() - err := runCommand("Bringing Down E2E Cluster on GCE", cmd) - if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err) - } - return nil -} - -func clusterDownGKE(gceZone, gceRegion string) error { - locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion) - if err != nil { - return err - } - - cmd := exec.Command("gcloud", "container", "clusters", "delete", *gkeTestClusterName, - locationArg, locationVal, "--quiet") - err = runCommand("Bringing Down E2E Cluster on GKE", cmd) - if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err.Error()) - } - return nil -} - -func buildKubernetes(k8sDir, command string) error { - cmd := exec.Command("make", "-C", k8sDir, command) - cmd.Env = os.Environ() - err := runCommand(fmt.Sprintf("Running command in kubernetes/kubernetes path=%s", k8sDir), cmd) - if err != nil { - return fmt.Errorf("failed to build Kubernetes: %w", err) - } - return nil -} - -func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, imageType string) error { - kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh") - _, err := os.Stat(kshPath) - if err == nil { - // Set kubectl to the one bundled in the k8s tar for versioning - err = os.Setenv("GCE_PD_KUBECTL", kshPath) - if err != nil { - return fmt.Errorf("failed to set cluster specific kubectl: %w", err) - } - } else { - klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath) - } - - if len(*kubeFeatureGates) != 0 { - err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates) - if err != nil { - return fmt.Errorf("failed to set kubernetes feature gates: %w", err) - } - klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) - } - - err = setImageTypeEnvs(imageType) - if err != nil { - return fmt.Errorf("failed to set image type environment variables: %w", err) - } - - err = os.Setenv("NUM_NODES", strconv.Itoa(numNodes)) - if err != nil { - return err - } - - // the chain is NUM_WINDOWS_NODES -> --num-windows-nodes -> NUM_WINDOWS_NODES - // runCommand runs e2e-up.sh inheriting env vars so the `--num-windows-nodes` - // flags might not be needed, added to be similar to the setup of NUM_NODES - err = os.Setenv("NUM_WINDOWS_NODES", strconv.Itoa(numWindowsNodes)) - if err != nil { - return err - } - - // The default master size with few nodes is too small; the tests must hit the API server - // more than usual. The main issue seems to be memory, to reduce GC times that stall the - // api server. For defaults, get-master-size in k/k/cluster/gce/config-common.sh. - if numNodes < 20 { - err = os.Setenv("MASTER_SIZE", "n1-standard-4") - if err != nil { - return err - } - } - - err = os.Setenv("KUBE_GCE_ZONE", gceZone) - if err != nil { - return err - } - cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh")) - cmd.Env = os.Environ() - err = runCommand("Starting E2E Cluster on GCE", cmd) - if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err.Error()) - } - - return nil -} - -func setImageTypeEnvs(imageType string) error { - switch strings.ToLower(imageType) { - case "cos": - case "cos_containerd": - case "gci": // GCI/COS is default type and does not need env vars set - case "ubuntu", "ubuntu_containerd": - return errors.New("setting environment vars for bringing up *ubuntu* cluster on GCE is unimplemented") - /* TODO(dyzz) figure out how to bring up a Ubuntu cluster on GCE. The below doesn't work. - err := os.Setenv("KUBE_OS_DISTRIBUTION", "ubuntu") - if err != nil { - return err - } - err = os.Setenv("KUBE_GCE_NODE_IMAGE", image) - if err != nil { - return err - } - err = os.Setenv("KUBE_GCE_NODE_PROJECT", imageProject) - if err != nil { - return err - } - */ - default: - return fmt.Errorf("could not set env for image type %s, only gci, cos, ubuntu supported", imageType) - } - return nil -} - -func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, imageType string, useManagedDriver bool) error { - locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion) - if err != nil { - return err - } - - out, err := exec.Command("gcloud", "container", "clusters", "list", - locationArg, locationVal, "--verbosity", "none", "--filter", - fmt.Sprintf("name=%s", *gkeTestClusterName)).CombinedOutput() - - if err != nil { - return fmt.Errorf("failed to check for previous test cluster: %v %s", err.Error(), out) - } - if len(out) > 0 { - klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", *gkeTestClusterName) - err = clusterDownGKE(gceZone, gceRegion) - if err != nil { - return err - } - } - - var cmd *exec.Cmd - cmdParams := []string{"container", "clusters", "create", *gkeTestClusterName, - locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes), - "--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--no-enable-autoupgrade"} - if isVariableSet(gkeClusterVer) { - cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer) - } else { - cmdParams = append(cmdParams, "--release-channel", *gkeReleaseChannel) - // release channel based GKE clusters require autorepair to be enabled. - cmdParams = append(cmdParams, "--enable-autorepair") - } - - if isVariableSet(gkeNodeVersion) { - cmdParams = append(cmdParams, "--node-version", *gkeNodeVersion) - } - - if useManagedDriver { - cmdParams = append(cmdParams, "--addons", "GcePersistentDiskCsiDriver") - } - - cmd = exec.Command("gcloud", cmdParams...) - err = runCommand("Starting E2E Cluster on GKE", cmd) - if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err.Error()) - } - - // Because gcloud cannot disable addons on cluster create, the deployment has - // to be disabled on update. - clusterVersion := mustGetKubeClusterVersion() - if !useManagedDriver && isGKEDeploymentInstalledByDefault(clusterVersion) { - cmd = exec.Command( - "gcloud", "beta", "container", "clusters", "update", - *gkeTestClusterName, locationArg, locationVal, "--quiet", - "--update-addons", "GcePersistentDiskCsiDriver=DISABLED") - err = runCommand("Updating E2E Cluster on GKE to disable driver deployment", cmd) - if err != nil { - return fmt.Errorf("failed to update kubernetes e2e cluster on gke: %v", err.Error()) - } - } - - return nil -} - -func downloadTarball(k8sDir, releaseVersion, subDir, tarballName string) error { - tarballPath := filepath.Join(k8sDir, subDir) - if err := os.MkdirAll(tarballPath, 0777); err != nil { - return err - } - tarballOutput := filepath.Join(tarballPath, tarballName) - downloadUrl := fmt.Sprintf("https://dl.k8s.io/release/%s/%s", releaseVersion, tarballName) - klog.Infof("Downloading tarball %s to path=%s from url=%s", tarballName, tarballPath, downloadUrl) - _, err := exec.Command("curl", "-Lsf", "--output", tarballOutput, downloadUrl).CombinedOutput() - if err != nil { - return err - } - - return nil -} - -func downloadAndExtractSrcTarball(k8sDir, releaseVersion, tarballName string) error { - tarballPath := filepath.Join(k8sDir, tarballName) - if err := downloadTarball(k8sDir, releaseVersion, "", tarballName); err != nil { - return err - } - out, err := exec.Command("tar", "xf", tarballPath, "-C", k8sDir).CombinedOutput() - if err != nil { - return fmt.Errorf("Failed to extract src tarball %s: %s: %s", tarballName, out, err) - } - return nil -} - -func getReleaseVersionFromKubeVersion(kubeVersion string) (string, error) { - releaseVersion := fmt.Sprintf("v%s", kubeVersion) - if kubeVersion == "stable" || kubeVersion == "latest" { - // See https://kubernetes.io/releases/download/ - out, err := exec.Command("curl", "-Lsf", "https://dl.k8s.io/release/stable.txt").CombinedOutput() - if err != nil { - return "", err - } - releaseVersion = string(out) - } - return releaseVersion, nil -} - -func downloadKubernetesRelease(k8sDir, kubeVersion, platform, arch string) error { - releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion) - if err != nil { - return err - } - - // Download precompiled tarballs - serverTarballName := fmt.Sprintf("kubernetes-server-%s-%s.tar.gz", platform, arch) - if err := downloadTarball(k8sDir, releaseVersion, "server", serverTarballName); err != nil { - return err - } - - manifestsTarballName := fmt.Sprintf("kubernetes-manifests.tar.gz") - if err := downloadTarball(k8sDir, releaseVersion, "server", manifestsTarballName); err != nil { - return err - } - - nodeTarballName := fmt.Sprintf("kubernetes-node-%s-%s.tar.gz", platform, arch) - if err := downloadTarball(k8sDir, releaseVersion, "node", nodeTarballName); err != nil { - return err - } - - clientTarballName := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", platform, arch) - if err := downloadTarball(k8sDir, releaseVersion, "client", clientTarballName); err != nil { - return err - } - - return nil -} - -func downloadKubernetesSource(k8sDir, kubeVersion string) error { - klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sDir) - - if err := os.MkdirAll(k8sDir, 0777); err != nil { - return err - } - if err := os.RemoveAll(k8sDir); err != nil { - return err - } - - // We clone rather than download from release archives, because the file naming has not been - // stable. For example, in late 2021 it appears that archives of minor versions (eg v1.21.tgz) - // stopped and was replaced with just patch version. - if kubeVersion == "master" { - // Clone of master. We cannot use a shallow clone, because the k8s version is not set, and - // in order to find the revision git searches through the tags, and tags are not fetched in - // a shallow clone. Not using a shallow clone adds about 700M to the ~5G archive directory, - // after make quick-release, so this is not disastrous. - klog.Info("cloning k8s master") - out, err := exec.Command("git", "clone", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error()) - } - } else { - releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion) - if err != nil { - return err - } - - // Download versioned source. - if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes-src.tar.gz"); err != nil { - return err - } - if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes.tar.gz"); err != nil { - return err - } - } - return nil -} - -func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) ([]string, error) { - var locationArg, locationVal, locationArgK2 string - switch { - case len(gceZone) > 0: - locationArg = "--gcp-zone" - locationArgK2 = "--zone" - locationVal = gceZone - case len(gceRegion) > 0: - locationArg = "--gcp-region" - locationArgK2 = "--region" - locationVal = gceRegion - } - - var gkeEnv string - switch gkeURL := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER"); gkeURL { - case "https://staging-container.sandbox.googleapis.com/": - gkeEnv = "staging" - case "https://test-container.sandbox.googleapis.com/": - gkeEnv = "test" - case "": - gkeEnv = "prod" - default: - // if the URL does not match to an option, assume it is a custom GKE backend - // URL and pass that to kubetest - gkeEnv = gkeURL - } - - cmd := exec.Command("gcloud", "config", "get-value", "project") - project, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to get current project: %v", err.Error()) - } - - // kubetest arguments - args := []string{ - "--up=false", - "--down=false", - "--provider=gke", - "--gcp-network=default", - "--check-version-skew=false", - "--deployment=gke", - fmt.Sprintf("--gcp-node-image=%s", imageType), - "--gcp-network=default", - fmt.Sprintf("--cluster=%s", *gkeTestClusterName), - fmt.Sprintf("--gke-environment=%s", gkeEnv), - fmt.Sprintf("%s=%s", locationArg, locationVal), - fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]), - } - - // kubetest2 arguments - argsK2 := []string{ - "--up=false", - "--down=false", - fmt.Sprintf("--cluster-name=%s", *gkeTestClusterName), - fmt.Sprintf("--environment=%s", gkeEnv), - fmt.Sprintf("%s=%s", locationArgK2, locationVal), - fmt.Sprintf("--project=%s", project[:len(project)-1]), - } - - if useKubetest2 { - return argsK2, nil - } else { - return args, nil - } -} - -func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) { - if kubeVersion != "" && gkeVersion != "" { - return "", fmt.Errorf("both kube version (%s) and gke version (%s) specified", kubeVersion, gkeVersion) - } - if kubeVersion == "" && gkeVersion == "" { - return "", errors.New("neither kube version nor gke version specified") - } - var v string - if kubeVersion != "" { - v = kubeVersion - } else if gkeVersion != "" { - v = gkeVersion - } - if v == "master" || v == "latest" { - // Ugh - return v, nil - } - toks := strings.Split(v, ".") - if len(toks) < 2 || len(toks) > 3 { - return "", fmt.Errorf("got unexpected number of tokens in version string %s - wanted 2 or 3", v) - } - return strings.Join(toks[:2], "."), nil - -} - -func getKubeClusterVersion() (string, error) { - out, err := exec.Command("kubectl", "version", "-o=json").Output() - if err != nil { - return "", fmt.Errorf("failed to obtain cluster version, error: %v; output was %s", err.Error(), out) - } - type version struct { - ClientVersion *apimachineryversion.Info `json:"clientVersion,omitempty" yaml:"clientVersion,omitempty"` - ServerVersion *apimachineryversion.Info `json:"serverVersion,omitempty" yaml:"serverVersion,omitempty"` - } - - var v version - err = json.Unmarshal(out, &v) - if err != nil { - return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err.Error()) - } - - return v.ServerVersion.GitVersion, nil -} - -func mustGetKubeClusterVersion() string { - ver, err := getKubeClusterVersion() - if err != nil { - klog.Fatalf("Error: %v", err.Error()) - } - return ver -} - -// getKubeConfig returns the full path to the -// kubeconfig file set in $KUBECONFIG env. -// If unset, then it defaults to $HOME/.kube/config -func getKubeConfig() (string, error) { - config, ok := os.LookupEnv("KUBECONFIG") - if ok { - return config, nil - } - homeDir, ok := os.LookupEnv("HOME") - if !ok { - return "", fmt.Errorf("HOME env not set") - } - return filepath.Join(homeDir, ".kube/config"), nil -} - -// getKubeClient returns a Kubernetes client interface -// for the test cluster -func getKubeClient() (kubernetes.Interface, error) { - kubeConfig, err := getKubeConfig() - if err != nil { - return nil, err - } - config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) - if err != nil { - return nil, fmt.Errorf("failed to create config: %v", err.Error()) - } - kubeClient, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("failed to create client: %v", err.Error()) - } - return kubeClient, nil -} - -func isGKEDeploymentInstalledByDefault(clusterVersion string) bool { - cv := mustParseVersion(clusterVersion) - return cv.atLeast(mustParseVersion("1.18.10-gke.2101")) && - cv.lessThan(mustParseVersion("1.19.0")) || - cv.atLeast(mustParseVersion("1.19.3-gke.2100")) -} diff --git a/test/k8s-integration/cluster_test.go b/test/k8s-integration/cluster_test.go deleted file mode 100644 index 5d32f484..00000000 --- a/test/k8s-integration/cluster_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import "testing" - -func TestGetNormalizedVersion(t *testing.T) { - tests := []struct { - name string - kubeV string - gkeV string - expectV string - expectErr bool - }{ - { - name: "kube", - kubeV: "1.13.5", - expectV: "1.13", - }, - { - name: "kube master", - kubeV: "master", - expectV: "master", - }, - { - name: "gke", - gkeV: "1.14.3", - expectV: "1.14", - }, - { - name: "gke minor", - gkeV: "1.14", - expectV: "1.14", - }, - { - name: "gke latest", - gkeV: "latest", - expectV: "latest", - }, - { - name: "kube minor", - kubeV: "1.13", - expectV: "1.13", - }, - { - name: "kube err", - kubeV: "1", - expectErr: true, - }, - { - name: "neither err", - expectErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - gotV, err := getNormalizedVersion(tc.kubeV, tc.gkeV) - if err != nil { - if !tc.expectErr { - t.Fatalf("Got unexpected err: %v", err) - } - return - } - if err == nil && tc.expectErr { - t.Fatal("Got no error but expected one") - } - if gotV != tc.expectV { - t.Errorf("Got version: %s, expected: %s", gotV, tc.expectV) - } - }) - } -} diff --git a/test/k8s-integration/config/pd-volumesnapshotclass.yaml b/test/k8s-integration/config/pd-volumesnapshotclass.yaml deleted file mode 100644 index bff9c3cd..00000000 --- a/test/k8s-integration/config/pd-volumesnapshotclass.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshotClass -metadata: - name: csi-gce-pd-snapshot-class -driver: pd.csi.storage.gke.io -deletionPolicy: Delete diff --git a/test/k8s-integration/config/sc-balanced.yaml b/test/k8s-integration/config/sc-balanced.yaml deleted file mode 100644 index 642c692f..00000000 --- a/test/k8s-integration/config/sc-balanced.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd-balanced -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-balanced -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-extreme.yaml b/test/k8s-integration/config/sc-extreme.yaml deleted file mode 100644 index 7f930c24..00000000 --- a/test/k8s-integration/config/sc-extreme.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-extreme - provisioned-iops-on-create: '10000' - # Add labels for testing. - labels: key1=value1,key2=value2 -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-regional.yaml b/test/k8s-integration/config/sc-regional.yaml deleted file mode 100644 index 4ad0b779..00000000 --- a/test/k8s-integration/config/sc-regional.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd-regional -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-standard - replication-type: regional-pd -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-standard-no-labels.yaml b/test/k8s-integration/config/sc-standard-no-labels.yaml deleted file mode 100644 index ab8d631f..00000000 --- a/test/k8s-integration/config/sc-standard-no-labels.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-standard -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-standard.yaml b/test/k8s-integration/config/sc-standard.yaml deleted file mode 100644 index 5a85b3fc..00000000 --- a/test/k8s-integration/config/sc-standard.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-standard - # Add labels for testing. - labels: key1=value1,key2=value2 -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-windows.yaml b/test/k8s-integration/config/sc-windows.yaml deleted file mode 100644 index 9a88a6ed..00000000 --- a/test/k8s-integration/config/sc-windows.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-standard - csi.storage.k8s.io/fstype: ntfs -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-xfs.yaml b/test/k8s-integration/config/sc-xfs.yaml deleted file mode 100644 index 2f286746..00000000 --- a/test/k8s-integration/config/sc-xfs.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-gcepd-xfs -provisioner: pd.csi.storage.gke.io -parameters: - type: pd-balanced - csi.storage.k8s.io/fstype: xfs -volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/test-config-template.in b/test/k8s-integration/config/test-config-template.in deleted file mode 100644 index a70686c5..00000000 --- a/test/k8s-integration/config/test-config-template.in +++ /dev/null @@ -1,31 +0,0 @@ -StorageClass: - FromFile: {{.StorageClassFile}} -{{if .SnapshotClassFile }} -SnapshotClass: - FromFile: {{ .SnapshotClassFile }} -{{end}} -{{if .Timeouts}} -Timeouts: - {{ range $key, $value := .Timeouts }}{{ $key }}: {{ $value }} - {{ end }} -{{end}} -DriverInfo: - Name: csi-gcepd-{{.StorageClass}}--{{.SnapshotClass}} - SupportedFsType: - {{range .SupportedFsType}} {{ . }}: - {{end}} - Capabilities: - {{range .Capabilities}} {{ . }}: true - {{end}} - StressTestOptions: - NumPods: 10 - NumRestarts: 10 - SupportedMountOption: - debug: - noatime: - SupportedSizeRange: - Min: {{.MinimumVolumeSize}} - Max: 64Ti - TopologyKeys: - - topology.gke.io/zone - NumAllowedTopologies: {{.NumAllowedTopologies}} diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go deleted file mode 100644 index 25cdef47..00000000 --- a/test/k8s-integration/driver-config.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strings" - "text/template" -) - -type driverConfig struct { - StorageClassFile string - StorageClass string - SnapshotClassFile string - SnapshotClass string - Capabilities []string - SupportedFsType []string - MinimumVolumeSize string - NumAllowedTopologies int - Timeouts map[string]string -} - -const ( - testConfigDir = "test/k8s-integration/config" - configTemplateFile = "test-config-template.in" - configFile = "test-config.yaml" - // configurable timeouts for the k8s e2e testsuites - dataSourceProvisionTimeout = "480s" - - // These are keys for the configurable timeout map. - dataSourceProvisionTimeoutKey = "DataSourceProvision" -) - -// generateDriverConfigFile loads a testdriver config template and creates a file -// with the test-specific configuration -func generateDriverConfigFile(testParams *testParameters) (string, error) { - // Load template - t, err := template.ParseFiles(filepath.Join(testParams.pkgDir, testConfigDir, configTemplateFile)) - if err != nil { - return "", err - } - - // Create destination - configFilePath := filepath.Join(testParams.pkgDir, testConfigDir, configFile) - f, err := os.Create(configFilePath) - if err != nil { - return "", err - } - defer f.Close() - - w := bufio.NewWriter(f) - defer w.Flush() - - // Fill in template parameters. Capabilities can be found here: - // https://github.com/kubernetes/kubernetes/blob/b717be8269a4f381ab6c23e711e8924bc1f64c93/test/e2e/storage/testsuites/testdriver.go#L136 - - caps := []string{ - "persistence", - "block", - "fsGroup", - "exec", - "multipods", - "topology", - "controllerExpansion", - "nodeExpansion", - } - var fsTypes []string - if testParams.platform == "windows" { - fsTypes = []string{"ntfs"} - caps = []string{ - "persistence", - "exec", - "multipods", - "topology", - } - } else { - fsTypes = []string{ - "ext2", - "ext3", - "ext4", - } - } - - /* Unsupported Capabilities: - RWX - volumeLimits # PD Supports volume limits but test is very slow - singleNodeVolume - dataSource - */ - - switch testParams.deploymentStrategy { - case "gke": - if testParams.imageType == "cos" { - var gkeVer *version - // The node version is what matters for XFS support. If the node version is not given, we assume - // it's the same as the cluster master version. - if testParams.nodeVersion != "" { - gkeVer = mustParseVersion(testParams.nodeVersion) - } else { - gkeVer = mustParseVersion(testParams.clusterVersion) - } - if gkeVer.lessThan(mustParseVersion("1.18.0")) { - // XFS is not supported on COS before 1.18.0 - } else { - fsTypes = append(fsTypes, "xfs") - } - } else { - // XFS is supported on all non-COS images. - fsTypes = append(fsTypes, "xfs") - } - case "gce": - fsTypes = append(fsTypes, "xfs") - default: - return "", fmt.Errorf("got unknown deployment strat %s, expected gce or gke", testParams.deploymentStrategy) - } - - var absSnapshotClassFilePath string - var snapshotClassName string - // If snapshot class is passed in as argument, include snapshot specific driver capabiltiites. - if testParams.snapshotClassFile != "" { - caps = append(caps, "snapshotDataSource") - // Update the absolute file path pointing to the snapshot class file, if it is provided as an argument. - absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile) - snapshotClassName = testParams.snapshotClassFile[:strings.LastIndex(testParams.snapshotClassFile, ".")] - } else { - snapshotClassName = "no-volumesnapshotclass" - } - - if !strings.Contains(testParams.storageClassFile, "sc-extreme") { - caps = append(caps, "pvcDataSource") - } - minimumVolumeSize := "5Gi" - numAllowedTopologies := 1 - if testParams.storageClassFile == regionalPDStorageClass { - minimumVolumeSize = "200Gi" - numAllowedTopologies = 2 - } - timeouts := map[string]string{ - dataSourceProvisionTimeoutKey: dataSourceProvisionTimeout, - } - extLoc := strings.LastIndex(testParams.storageClassFile, ".") - scName := testParams.storageClassFile[:extLoc] - params := driverConfig{ - StorageClassFile: filepath.Join(testParams.pkgDir, testConfigDir, testParams.storageClassFile), - StorageClass: scName, - SnapshotClassFile: absSnapshotClassFilePath, - SnapshotClass: snapshotClassName, - SupportedFsType: fsTypes, - Capabilities: caps, - MinimumVolumeSize: minimumVolumeSize, - NumAllowedTopologies: numAllowedTopologies, - Timeouts: timeouts, - } - - // Write config file - err = t.Execute(w, params) - if err != nil { - return "", err - } - - return configFilePath, nil -} diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go deleted file mode 100644 index 0481e8ad..00000000 --- a/test/k8s-integration/driver.go +++ /dev/null @@ -1,220 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/podlogs" -) - -func getOverlayDir(pkgDir, deployOverlayName string) string { - return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName) -} - -func installDriver(testParams *testParameters, stagingImage, deployOverlayName string, doDriverBuild bool) error { - if doDriverBuild { - // Install kustomize - klog.Infof("Installing kustomize") - out, err := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err.Error()) - } - - // Edit ci kustomization to use given image tag - overlayDir := getOverlayDir(testParams.pkgDir, deployOverlayName) - err = os.Chdir(overlayDir) - if err != nil { - return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err.Error()) - } - - // TODO (#138): in a local environment this is going to modify the actual kustomize files. - // maybe a copy should be made instead - out, err = exec.Command( - filepath.Join(testParams.pkgDir, "bin", "kustomize"), - "edit", - "set", - "image", - fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, testParams.stagingVersion)).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err.Error()) - } - } - - var deployEnv []string - if deployOverlayName != "noauth" { - // setup service account file for secret creation - tmpSaFile := filepath.Join(generateUniqueTmpDir(), "cloud-sa.json") - defer removeDir(filepath.Dir(tmpSaFile)) - - // Need to copy it to name the file "cloud-sa.json" - out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput() - if err != nil { - return fmt.Errorf("error copying service account key: %s, err: %v", out, err.Error()) - } - defer shredFile(tmpSaFile) - - deployEnv = append(deployEnv, fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile))) - } - - // deploy driver - deployCmd := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check") - deployEnv = append(deployEnv, - fmt.Sprintf("GOPATH=%s", testParams.goPath), - fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName)) - deployEnv = append(os.Environ(), deployEnv...) - deployCmd.Env = deployEnv - err := runCommand("Deploying driver", deployCmd) - if err != nil { - return fmt.Errorf("failed to deploy driver: %w", err) - } - - waitScript := filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "wait-for-driver.sh") - waitCmd := exec.Command(waitScript) - waitCmd.Env = deployEnv - err = runCommand("Waiting for driver to start", waitCmd) - if err != nil { - return fmt.Errorf("driver failed to come up: %w", err) - } - if testParams.platform == "windows" { - waitCmd = exec.Command(waitScript, "--windows") - waitCmd.Env = deployEnv - err = runCommand("Waiting for windows deployment to start", waitCmd) - if err != nil { - return fmt.Errorf("Windows deployment failed to come up: %w", err) - } - } - out, err := exec.Command("kubectl", "describe", "pods", "-n", getDriverNamespace()).CombinedOutput() - klog.Infof("describe pods \n %s", string(out)) - - if err != nil { - return fmt.Errorf("failed to describe pods: %w", err) - } - // Print out Windows pod logs for debugging - if testParams.platform == "windows" { - podsCmd := exec.Command("kubectl", "get", "pods", "-l", "app=gcp-compute-persistent-disk-csi-driver-win", "-o", "name", "-n", getDriverNamespace()) - out, err = podsCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to get pd csi pods: %v", err.Error()) - } - pods := strings.Fields(string(out)) - for _, pod := range pods { - logCmd := exec.Command("kubectl", "logs", pod, "-n", getDriverNamespace(), "--all-containers=true") - out, err := logCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to get logs from pod %s: %v", pod, err.Error()) - } - klog.Infof("kubectl logs pod: %s, output %s", pod, string(out)) - } - } - - return nil -} - -func deleteDriver(testParams *testParameters, deployOverlayName string) error { - deleteCmd := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "delete-driver.sh")) - deleteCmd.Env = append(os.Environ(), - fmt.Sprintf("GOPATH=%s", testParams.goPath), - fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName), - ) - err := runCommand("Deleting driver", deleteCmd) - if err != nil { - return fmt.Errorf("failed to delete driver: %v", err.Error()) - } - return nil -} - -func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { - err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion) - if err != nil { - return err - } - err = os.Setenv("GCE_PD_CSI_STAGING_IMAGE", stagingImage) - if err != nil { - return err - } - var cmd *exec.Cmd - - if platform == "windows" { - // build multi-arch image which can work for both Linux and Windows - cmd = exec.Command("make", "-C", pkgDir, "build-and-push-multi-arch", - fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), - fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) - err = runCommand("Building and Pushing GCP Container for Windows", cmd) - if err != nil { - return fmt.Errorf("failed to run make command for windows: err: %v", err.Error()) - } - } else { - cmd = exec.Command("make", "-C", pkgDir, "push-container", - fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), - fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) - err = runCommand("Pushing GCP Container for Linux", cmd) - if err != nil { - return fmt.Errorf("failed to run make command for linux: err: %v", err.Error()) - } - } - - return nil -} - -func deleteImage(stagingImage, stagingVersion string) error { - cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet") - err := runCommand("Deleting GCR Container", cmd) - if err != nil { - return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err.Error()) - } - return nil -} - -// dumpDriverLogs will watch all pods in the driver namespace -// and copy its logs to the test artifacts directory, if set. -// It returns a context.CancelFunc that needs to be invoked when -// the test is finished. -func dumpDriverLogs() (context.CancelFunc, error) { - // Dump all driver logs to the test artifacts - artifactsDir, ok := os.LookupEnv("ARTIFACTS") - if ok { - client, err := getKubeClient() - if err != nil { - return nil, fmt.Errorf("failed to get kubeclient: %v", err.Error()) - } - out := podlogs.LogOutput{ - StatusWriter: os.Stdout, - LogPathPrefix: filepath.Join(artifactsDir, "pd-csi-driver") + "/", - } - ctx, cancel := context.WithCancel(context.Background()) - if err = podlogs.CopyAllLogs(ctx, client, getDriverNamespace(), out); err != nil { - return cancel, fmt.Errorf("failed to start pod logger: %v", err.Error()) - } - return cancel, nil - } - return nil, nil -} - -// mergeArtifacts merges the results of doing multiple gingko runs, taking all junit files -// in the specified subdirectories of the artifacts directory and merging into a single -// file at the artifcats root. If artifacts are not saved (ie, ARTIFACTS is not set), -// this is a no-op. See kubernetes-csi/csi-release-tools/prow.sh for the inspiration. -func mergeArtifacts(subdirectories []string) error { - artifactsDir, ok := os.LookupEnv("ARTIFACTS") - if !ok { - // No artifacts, nothing to merge. - return nil - } - var sourceDirs []string - for _, subdir := range subdirectories { - sourceDirs = append(sourceDirs, filepath.Join(artifactsDir, subdir)) - } - return MergeJUnit("External Storage", sourceDirs, filepath.Join(artifactsDir, "junit_pdcsi.xml")) -} - -func getDriverNamespace() string { - if *useGKEManagedDriver { - return managedDriverNamespace - } - return externalDriverNamespace -} diff --git a/test/k8s-integration/filter-junit.go b/test/k8s-integration/filter-junit.go deleted file mode 100644 index 844df603..00000000 --- a/test/k8s-integration/filter-junit.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/xml" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - - "k8s.io/klog/v2" -) - -/* - * TestSuite represents a JUnit file. Due to how encoding/xml works, we have - * represent all fields that we want to be passed through. It's therefore - * not a complete solution, but good enough for Ginkgo + Spyglass. - */ -type TestSuite struct { - XMLName string `xml:"testsuite"` - TestCases []TestCase `xml:"testcase"` -} - -type TestCase struct { - Name string `xml:"name,attr"` - Time string `xml:"time,attr"` - SystemOut string `xml:"system-out,omitempty"` - Failure string `xml:"failure,omitempty"` - Skipped SkipReason `xml:"skipped,omitempty"` -} - -// SkipReason deals with the special : -// if present, we must re-encode it, even if empty. -type SkipReason string - -func (s *SkipReason) UnmarshalText(text []byte) error { - *s = SkipReason(text) - if *s == "" { - *s = " " - } - return nil -} - -func (s SkipReason) MarshalText() ([]byte, error) { - if s == " " { - return []byte{}, nil - } - return []byte(s), nil -} - -// MergeJUnit merges all junit xml files found in sourceDirectories into a single xml file at destination, using the filter. -// The merging removes duplicate skipped tests. The original files are deleted. -func MergeJUnit(testFilter string, sourceDirectories []string, destination string) error { - var junit TestSuite - var data []byte - - re := regexp.MustCompile(testFilter) - - var mergeErrors []string - var filesToDelete []string - for _, dir := range sourceDirectories { - files, err := ioutil.ReadDir(dir) - if err != nil { - klog.Errorf("Failed to read juint directory %s: %w", dir, err) - mergeErrors = append(mergeErrors, err.Error()) - continue - } - for _, file := range files { - if !strings.HasSuffix(file.Name(), ".xml") { - continue - } - fullFilename := filepath.Join(dir, file.Name()) - filesToDelete = append(filesToDelete, fullFilename) - data, err := ioutil.ReadFile(fullFilename) - if err != nil { - return err - } - if err = xml.Unmarshal(data, &junit); err != nil { - return err - } - } - } - - // Keep only matching testcases. Testcases skipped in all test runs are only stored once. - filtered := map[string]TestCase{} - for _, testcase := range junit.TestCases { - if !re.MatchString(testcase.Name) { - continue - } - entry, ok := filtered[testcase.Name] - if !ok || // not present yet - entry.Skipped != "" && testcase.Skipped == "" { // replaced skipped test with real test run - filtered[testcase.Name] = testcase - } - } - junit.TestCases = nil - for _, testcase := range filtered { - junit.TestCases = append(junit.TestCases, testcase) - } - - // Re-encode. - data, err := xml.MarshalIndent(junit, "", " ") - if err != nil { - return err - } - - if err = ioutil.WriteFile(destination, data, 0644); err != nil { - return err - } - - if mergeErrors != nil { - return fmt.Errorf("Problems reading junit files; partial merge has been performed: %s", strings.Join(mergeErrors, " ")) - } else { - // Only delete original files if everything went well. - var removeErrors []string - for _, filename := range filesToDelete { - if err := os.Remove(filename); err != nil { - removeErrors = append(removeErrors, err.Error()) - } - } - if removeErrors != nil { - return fmt.Errorf("Problem removing original junit results: %s", strings.Join(removeErrors, " ")) - } - } - return nil -} diff --git a/test/k8s-integration/fs.go b/test/k8s-integration/fs.go deleted file mode 100644 index 7d35576f..00000000 --- a/test/k8s-integration/fs.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "io" - "os" - "path/filepath" -) - -// CopyFile copies a file from src to dst -func CopyFile(src, dst string) (err error) { - // get source information - info, err := os.Stat(src) - if err != nil { - return err - } - return copyFile(src, dst, info) -} - -func copyFile(src, dst string, info os.FileInfo) error { - // open src for reading - in, err := os.Open(src) - if err != nil { - return err - } - defer func() { - closeErr := in.Close() - // if we weren't returning an error - if err == nil { - err = closeErr - } - }() - if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { - return err - } - // create dst file - // this is like f, err := os.Create(dst); os.Chmod(f.Name(), src.Mode()) - out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode()) - if err != nil { - return err - } - // make sure we close the file - defer func() { - closeErr := out.Close() - // if we weren't returning an error - if err == nil { - err = closeErr - } - }() - // actually copy - if _, err = io.Copy(out, in); err != nil { - return err - } - err = out.Sync() - return err -} diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go deleted file mode 100644 index 67aa5a46..00000000 --- a/test/k8s-integration/main.go +++ /dev/null @@ -1,892 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "syscall" - - "k8s.io/apimachinery/pkg/util/uuid" - apimachineryversion "k8s.io/apimachinery/pkg/util/version" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "k8s.io/klog/v2" - "k8s.io/utils/strings/slices" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" -) - -var ( - // Kubernetes cluster flags - teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test") - teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test") - bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster") - platform = flag.String("platform", "linux", "platform that the tests will be run on, either linux or windows") - arch = flag.String("arch", "amd64", "architecture that the tests will be run on (eg: amd64/arm64)") - gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in") - gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in") - kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster") - testVersion = flag.String("test-version", "", "version of Kubernetes to download and use for tests") - kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster") - localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries") - deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke") - gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke") - numNodes = flag.Int("num-nodes", 0, "the number of nodes in the test cluster") - numWindowsNodes = flag.Int("num-windows-nodes", 0, "the number of Windows nodes in the test cluster") - imageType = flag.String("image-type", "cos_containerd", "the image type to use for the cluster") - gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'") - gkeTestClusterPrefix = flag.String("gke-cluster-prefix", "pdcsi", "Prefix of GKE cluster names. A random suffix will be appended to form the full name.") - gkeTestClusterName = flag.String("gke-cluster-name", "", "Name of existing cluster") - gkeNodeVersion = flag.String("gke-node-version", "", "GKE cluster worker node version") - isRegionalCluster = flag.Bool("is-regional-cluster", false, "tell the test that a regional cluster is being used. Should be used for running on an existing regional cluster (ie, --bringup-cluster=false). The test will fail if a zonal GKE cluster is created when this flag is true") - - // Test infrastructure flags - boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") - storageClassFiles = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") - snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") - inProw = flag.Bool("run-in-prow", false, "is the test running in PROW") - - // Driver flags - stagingImage = flag.String("staging-image", "", "name of image to stage to") - saFile = flag.String("service-account-file", "", "path of service account file") - deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with") - doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source") - doK8sBuild = flag.Bool("do-k8s-build", true, "building the driver from source. If false, will fetch precompiled artifacts") - useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests") - - // Test flags - migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration") - testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e") - - useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests") - parallel = flag.Int("parallel", 4, "the number of parallel tests setting for ginkgo parallelism") - - allowedVersionTags = []string{"master", "stable", "latest"} -) - -const ( - pdImagePlaceholder = "gke.gcr.io/gcp-compute-persistent-disk-csi-driver" - k8sInDockerBuildBinDir = "_output/dockerized/bin/linux/amd64" - k8sOutOfDockerBuildBinDir = "_output/bin" - externalDriverNamespace = "gce-pd-csi-driver" - managedDriverNamespace = "kube-system" - regionalPDStorageClass = "sc-regional.yaml" -) - -type testParameters struct { - platform string - stagingVersion string - goPath string - pkgDir string - testParentDir string - k8sSourceDir string - testFocus string - testSkip string - storageClassFile string - snapshotClassFile string - cloudProviderArgs []string - deploymentStrategy string - outputDir string - allowedNotReadyNodes int - useGKEManagedDriver bool - clusterVersion string - nodeVersion string - imageType string - parallel int -} - -func init() { - flag.Set("logtostderr", "true") -} - -func main() { - klog.InitFlags(nil) - flag.Set("logtostderr", "true") - flag.Parse() - - if *useGKEManagedDriver { - *doDriverBuild = false - *teardownDriver = false - } - - if !*inProw && *doDriverBuild { - ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to") - } - - if *useGKEManagedDriver { - ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver") - ensureFlag(doDriverBuild, false, "'do-driver-build' must be false when using GKE managed driver") - ensureFlag(teardownDriver, false, "'teardown-driver' must be false when using GKE managed driver") - ensureVariable(stagingImage, false, "'staging-image' must not be set when using GKE managed driver") - ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver") - } - - if !*useGKEManagedDriver { - ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag") - if *deployOverlayName != "noauth" { - ensureVariable(saFile, true, "service-account-file is a required flag") - } - } - - ensureVariable(testFocus, true, "test-focus is a required flag") - - if len(*gceRegion) != 0 { - ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set") - } else { - ensureVariable(gceZone, true, "One of gce-zone or gce-region must be set") - } - - if *migrationTest { - ensureVariable(storageClassFiles, false, "storage-class-file and migration-test cannot both be set") - } else { - ensureVariable(storageClassFiles, true, "One of storageclass-file and migration-test must be set") - } - - if !*bringupCluster && *platform != "windows" { - ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster") - } else { - ensureVariable(imageType, true, "image type is a required flag. A good default is 'cos_containerd'") - if *isRegionalCluster { - klog.Error("is-regional-cluster can only be set when using an existing cluster") - } - } - - if *deploymentStrat == "gke" { - ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.") - ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel}, - "For GKE cluster deployment, exactly one of 'gke-cluster-version' or 'gke-release-channel' must be set") - ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.") - if len(*localK8sDir) == 0 { - ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.") - } - if len(*gkeTestClusterName) == 0 { - randSuffix := string(uuid.NewUUID())[0:4] - *gkeTestClusterName = *gkeTestClusterPrefix + randSuffix - } - } else if *deploymentStrat == "gce" { - ensureVariable(gceRegion, false, "regional clusters not supported for 'gce' deployment") - ensureVariable(gceZone, true, "gce-zone required for 'gce' deployment") - } - - if len(*localK8sDir) != 0 { - ensureVariable(kubeVersion, false, "Cannot set a kube version when using a local k8s dir.") - ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.") - } - - if *numNodes == 0 && *bringupCluster { - klog.Fatalf("num-nodes must be set to number of nodes in cluster") - } - if *numWindowsNodes == 0 && *bringupCluster && *platform == "windows" { - klog.Fatalf("num-windows-nodes must be set if the platform is windows") - } - - if *kubeVersion == "master" && !*doK8sBuild { - klog.Fatalf("must set do-k8s-build=true when kubeVersion=%s", *kubeVersion) - } - - if len(*kubeVersion) != 0 { - if !slices.Contains(allowedVersionTags, *kubeVersion) { - if _, err := parseVersion(*kubeVersion); err != nil { - klog.Fatalf("invalid kube-version %s", *kubeVersion) - } - } - } - - if len(*testVersion) != 0 { - if !slices.Contains(allowedVersionTags, *testVersion) { - if _, err := parseVersion(*testVersion); err != nil { - klog.Fatalf("invalid test-version %s", *testVersion) - } - } - } - - err := handle() - if err != nil { - klog.Fatalf("Failed to run integration test: %w", err) - } -} - -func buildTestingBinaries(k8sDir string) error { - if err := buildKubernetes(k8sDir, "WHAT=test/e2e/e2e.test"); err != nil { - return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error()) - } - - // kubetest relies on ginkgo and kubectl already built in the test k8s directory - if err := buildKubernetes(k8sDir, "ginkgo"); err != nil { - return fmt.Errorf("failed to build gingko: %v", err.Error()) - } - - if err := buildKubernetes(k8sDir, "kubectl"); err != nil { - return fmt.Errorf("failed to build kubectl: %v", err.Error()) - } - return nil -} - -func handle() error { - oldmask := syscall.Umask(0000) - defer syscall.Umask(oldmask) - - testParams := &testParameters{ - platform: *platform, - testFocus: *testFocus, - stagingVersion: string(uuid.NewUUID()), - deploymentStrategy: *deploymentStrat, - useGKEManagedDriver: *useGKEManagedDriver, - imageType: *imageType, - parallel: *parallel, - } - - goPath, ok := os.LookupEnv("GOPATH") - if !ok { - return fmt.Errorf("Could not find env variable GOPATH") - } - testParams.goPath = goPath - testParams.pkgDir = filepath.Join(goPath, "src", "sigs.k8s.io", "gcp-compute-persistent-disk-csi-driver") - - // If running in Prow, then acquire and set up a project through Boskos - if *inProw { - oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() - project := strings.TrimSpace(string(oldProject)) - if err != nil { - return fmt.Errorf("failed to get gcloud project: %s, err: %w", oldProject, err) - } - newproject, _ := testutils.SetupProwConfig(*boskosResourceType) - err = setEnvProject(newproject) - if err != nil { - return fmt.Errorf("failed to set project environment to %s: %w", newproject, err) - } - - defer func() { - err = setEnvProject(string(oldProject)) - if err != nil { - klog.Errorf("failed to set project environment to %s: %w", oldProject, err.Error()) - } - }() - project = newproject - if *doDriverBuild { - *stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project))) - } - if _, ok := os.LookupEnv("USER"); !ok { - err = os.Setenv("USER", "prow") - if err != nil { - return fmt.Errorf("failed to set user in prow to prow: %v", err.Error()) - } - } - } - - // Build and push the driver, if required. Defer the driver image deletion. - if *doDriverBuild { - klog.Infof("Building GCE PD CSI Driver") - err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform) - if err != nil { - return fmt.Errorf("failed pushing image: %v", err.Error()) - } - defer func() { - if *teardownCluster { - err := deleteImage(*stagingImage, testParams.stagingVersion) - if err != nil { - klog.Errorf("failed to delete image: %w", err) - } - } - }() - } - - // Create temporary directories for kubernetes source/build artifacts - testParams.testParentDir = generateUniqueTmpDir() - defer removeDir(testParams.testParentDir) - - // If kube version is set, then download and build Kubernetes for cluster creation - // Otherwise, either GKE or a prebuild local K8s dir is being used - if len(*kubeVersion) != 0 { - testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") - klog.Infof("Downloading Kubernetes source from: %s", *kubeVersion) - if err := downloadKubernetesSource(testParams.k8sSourceDir, *kubeVersion); err != nil { - return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) - } - - if *doK8sBuild { - klog.Info("Building Kubernetes source") - if err := buildKubernetes(testParams.k8sSourceDir, "quick-release"); err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err.Error()) - } - } else { - klog.Info("Fetching precompiled Kubernetes artifacts for %s/%s", *platform, *arch) - if err := downloadKubernetesRelease(testParams.k8sSourceDir, *kubeVersion, *platform, *arch); err != nil { - return fmt.Errorf("failed to download Kubernetes release: %v", err.Error()) - } - if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil { - return err - } - } - } else { - testParams.k8sSourceDir = *localK8sDir - } - - // If using kubetest and test version is set, then download and build Kubernetes to run K8s tests - // Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used. - if !*useKubeTest2 && len(*testVersion) != 0 && *testVersion != *kubeVersion { - testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") - // Overlay the Kubernetes source - if err := downloadKubernetesSource(testParams.k8sSourceDir, *testVersion); err != nil { - return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) - } - klog.Infof("Building Kubernetes Testing binaries: %s", *kubeVersion) - if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil { - return err - } - } - - if *deploymentStrat == "gke" { - gkeRegional := isRegionalGKECluster(*gceZone, *gceRegion) - if *isRegionalCluster && !gkeRegional { - return fmt.Errorf("--is-regional-cluster set but deployed GKE cluster would be zonal") - } - *isRegionalCluster = gkeRegional - } - - // Create a cluster either through GKE or GCE - if *bringupCluster { - var err error = nil - switch *deploymentStrat { - case "gce": - err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, *numWindowsNodes, testParams.imageType) - case "gke": - err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *numWindowsNodes, testParams.imageType, testParams.useGKEManagedDriver) - default: - err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy) - } - if err != nil { - return fmt.Errorf("failed to cluster up: %w", err) - } - } - - // Defer the tear down of the cluster through GKE or GCE - if *teardownCluster { - defer func() { - switch testParams.deploymentStrategy { - case "gce": - err := clusterDownGCE(testParams.k8sSourceDir) - if err != nil { - klog.Errorf("failed to cluster down: %w", err) - } - case "gke": - err := clusterDownGKE(*gceZone, *gceRegion) - if err != nil { - klog.Errorf("failed to cluster down: %w", err) - } - default: - klog.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy) - } - }() - } - - // For windows cluster, when cluster is up, all Windows nodes are tainted with NoSchedule to avoid linux pods - // being scheduled to Windows nodes. When running windows tests, we need to remove the taint. - if testParams.platform == "windows" { - klog.Infof("Removing taints from all windows nodes.") - - nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=windows", "-o", "name") - out, err := nodesCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to get windows nodes: %v", err.Error()) - } - nodes := strings.Fields(string(out)) - for _, node := range nodes { - taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os:NoSchedule-") - out, err := taintCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to untaint windows node %s: error %v. output %s", node, err.Error(), string(out)) - } - klog.Infof("untaint windows nodes: %s, output %s", node, string(out)) - } - - // It typically takes 5+ minutes to download Windows container image. To avoid tests being timed out, - // pre-pulling the test images as best effort. - klog.Infof("Prepulling test images.") - err = os.Setenv("PREPULL_YAML", filepath.Join(testParams.pkgDir, "test", "k8s-integration", "prepull.yaml")) - if err != nil { - return err - } - out, err = exec.Command(filepath.Join(testParams.pkgDir, "test", "k8s-integration", "prepull-image.sh")).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to prepull images: %s, err: %v", out, err.Error()) - } - out, err = exec.Command("kubectl", "describe", "pods", "-n", getDriverNamespace()).CombinedOutput() - klog.Infof("describe pods \n %s", string(out)) - - if err != nil { - return fmt.Errorf("failed to describe pods: %v", err.Error()) - } - - } - - if !testParams.useGKEManagedDriver { - // Install the driver and defer its teardown - err := installDriver(testParams, *stagingImage, *deployOverlayName, *doDriverBuild) - if *teardownDriver { - defer func() { - if teardownErr := deleteDriver(testParams, *deployOverlayName); teardownErr != nil { - klog.Errorf("failed to delete driver: %w", teardownErr) - } - }() - } - if err != nil { - return fmt.Errorf("failed to install CSI Driver: %w", err) - } - } - - // Dump all driver logs to the test artifacts - cancel, err := dumpDriverLogs() - if err != nil { - return fmt.Errorf("failed to start driver logging: %w", err) - } - defer func() { - if cancel != nil { - cancel() - } - }() - - // For windows cluster, it has both Windows nodes and Linux nodes. Before triggering the tests, taint Linux nodes - // with NoSchedule to avoid test pods being scheduled on Linux. Need to do this step after driver is deployed. - // Also the test framework will not proceed to run tests unless all nodes are ready - // AND schedulable. Allow not-ready nodes since we make Linux nodes - // unschedulable. - testParams.allowedNotReadyNodes = 0 - if *platform == "windows" { - klog.Infof("Tainting linux nodes") - nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=linux", "-o", "name") - out, err := nodesCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to get linux nodes: %v", err.Error()) - } - nodes := strings.Fields(string(out)) - testParams.allowedNotReadyNodes = len(nodes) - for _, node := range nodes { - taintCmd := exec.Command("kubectl", "taint", "node", node, "node.kubernetes.io/os=linux:NoSchedule", "--overwrite=true") - out, err := taintCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to untaint windows node %s, error %v, output %s", node, err.Error(), string(out)) - } - klog.Infof("taint linux nodes: %s, output %s", node, string(out)) - } - } - - switch testParams.deploymentStrategy { - case "gke": - testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, *useKubeTest2) - if err != nil { - return fmt.Errorf("failed to build GKE kubetest args: %v", err.Error()) - } - case "gce": - if *useKubeTest2 { - testParams.cloudProviderArgs = []string{ - // This flag tells kubetest2 what "repo-root" is. - // If --legacy-mode is set, kubernetes/kubernetes is used; - // otherwise kubernetes/cloud-provider-gcp is used. - "--legacy-mode", - fmt.Sprintf("--repo-root=%s", testParams.k8sSourceDir), - } - } - } - - // Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k, - // hence we use the main.Version utils to parse and compare GKE managed cluster versions. - // For clusters deployed on GCE, use the apimachinery version utils (which supports non-gke based semantic versioning). - testParams.clusterVersion = mustGetKubeClusterVersion() - klog.Infof("kubernetes cluster server version: %s", testParams.clusterVersion) - switch testParams.deploymentStrategy { - case "gce": - testParams.testSkip = generateGCETestSkip(testParams) - case "gke": - testParams.nodeVersion = *gkeNodeVersion - testParams.testSkip = generateGKETestSkip(testParams) - - default: - return fmt.Errorf("Unknown deployment strategy %s", testParams.deploymentStrategy) - } - - skipDiskImageSnapshots := false - if mustParseVersion(testParams.clusterVersion).lessThan(mustParseVersion("1.22.0")) { - // Disk image cloning in only supported from 1.22 on. - skipDiskImageSnapshots = true - } - - // Run the tests using the k8sSourceDir kubernetes - if len(*storageClassFiles) != 0 { - applicableStorageClassFiles := []string{} - applicableSnapshotClassFiles := []string{} - for _, rawScFile := range strings.Split(*storageClassFiles, ",") { - scFile := strings.TrimSpace(rawScFile) - if len(scFile) == 0 { - continue - } - if scFile == regionalPDStorageClass && !*isRegionalCluster { - klog.Warningf("Skipping regional StorageClass in zonal cluster") - continue - } - applicableStorageClassFiles = append(applicableStorageClassFiles, scFile) - } - if len(applicableStorageClassFiles) == 0 { - return fmt.Errorf("No applicable storage classes found") - } - for _, rawSnapshotClassFile := range strings.Split(*snapshotClassFiles, ",") { - snapshotClassFile := strings.TrimSpace(rawSnapshotClassFile) - if skipDiskImageSnapshots && strings.Contains(snapshotClassFile, "image-volumesnapshotclass") { - continue - } - if len(snapshotClassFile) != 0 { - applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, snapshotClassFile) - } - } - var ginkgoErrors []string - var testOutputDirs []string - - // Run non-snapshot tests. - testParams.snapshotClassFile = "" - for _, scFile := range applicableStorageClassFiles { - outputDir := strings.TrimSuffix(scFile, ".yaml") - testOutputDirs = append(testOutputDirs, outputDir) - testParams.storageClassFile = scFile - if err = runCSITests(testParams, outputDir); err != nil { - ginkgoErrors = append(ginkgoErrors, err.Error()) - } - } - // Run snapshot tests, if there are applicable files, using the first storage class. - if len(applicableStorageClassFiles) > 0 { - testParams.storageClassFile = applicableStorageClassFiles[0] - for _, snapshotClassFile := range applicableSnapshotClassFiles { - testParams.snapshotClassFile = snapshotClassFile - outputDir := strings.TrimSuffix(snapshotClassFile, ".yaml") - testOutputDirs = append(testOutputDirs, outputDir) - if err = runCSITests(testParams, outputDir); err != nil { - ginkgoErrors = append(ginkgoErrors, err.Error()) - } - } - } - if err = mergeArtifacts(testOutputDirs); err != nil { - return fmt.Errorf("artifact merging failed: %w", err) - } - if ginkgoErrors != nil { - return fmt.Errorf("runCSITests failed: %v", strings.Join(ginkgoErrors, " ")) - } - } else if *migrationTest { - err = runMigrationTests(testParams) - } else { - return fmt.Errorf("did not run either CSI or Migration test") - } - - if err != nil { - return fmt.Errorf("failed to run tests: %w", err) - } - - return nil -} - -func generateGCETestSkip(testParams *testParameters) string { - skipString := "\\[Disruptive\\]|\\[Serial\\]" - v := apimachineryversion.MustParseSemantic(testParams.clusterVersion) - - // "volumeMode should not mount / map unused volumes in a pod" tests a - // (https://github.com/kubernetes/kubernetes/pull/81163) - // bug-fix introduced in 1.16 - if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) { - skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod" - } - - // ExpandCSIVolumes feature is beta in k8s 1.16 - if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) { - skipString = skipString + "|allowExpansion" - } - - if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) { - skipString = skipString + "|VolumeSnapshotDataSource" - } - if v.LessThan(apimachineryversion.MustParseSemantic("1.20.0")) { - skipString = skipString + "|fsgroupchangepolicy" - } - if testParams.platform == "windows" { - skipString = skipString + "|\\[LinuxOnly\\]" - } - - return skipString -} - -func generateGKETestSkip(testParams *testParameters) string { - skipString := "\\[Disruptive\\]|\\[Serial\\]" - - curVer := mustParseVersion(testParams.clusterVersion) - var nodeVer *version - if testParams.nodeVersion != "" { - nodeVer = mustParseVersion(testParams.nodeVersion) - } - - // Cloning test fixes were introduced after 1.23. - if curVer.lessThan(mustParseVersion("1.24.0")) { - skipString = skipString + "|pvc.data.source" - } - - // Snapshot and restore test fixes were introduced after 1.26 in PR#972. - if curVer.lessThan(mustParseVersion("1.26.0")) { - skipString = skipString + "|should.provision.correct.filesystem.size.when.restoring.snapshot.to.larger.size.pvc" - } - - // "volumeMode should not mount / map unused volumes in a pod" tests a - // (https://github.com/kubernetes/kubernetes/pull/81163) - // bug-fix introduced in 1.16 - if curVer.lessThan(mustParseVersion("1.16.0")) || (nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.16.0"))) { - skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod" - } - - // Check master and node version to skip Pod FsgroupChangePolicy test suite. - if curVer.lessThan(mustParseVersion("1.20.0")) || (nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.20.0"))) { - skipString = skipString + "|fsgroupchangepolicy" - } - - // Generic Ephemeral volume is only enabled in version 1.21. - // If there's node skew Generic Ephemeral volume tests might not be skipped correctly - if curVer.lessThan(mustParseVersion("1.21.0")) || (nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.21.0"))) { - skipString = skipString + "|Generic\\sEphemeral-volume" - } - - // ExpandCSIVolumes feature is beta in k8s 1.16 - // For GKE deployed PD CSI driver, resizer sidecar is enabled in 1.16.8-gke.3 - if (testParams.useGKEManagedDriver && curVer.lessThan(mustParseVersion("1.16.8-gke.3"))) || - (!testParams.useGKEManagedDriver && curVer.lessThan(mustParseVersion("1.16.0")) || - (nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.16.0")))) { - skipString = skipString + "|allowExpansion" - } - - // For GKE deployed PD CSI snapshot is enabled in 1.17.6-gke.4(and higher), 1.18.3-gke.0(and higher). - if (testParams.useGKEManagedDriver && curVer.lessThan(mustParseVersion("1.17.6-gke.4"))) || - (!testParams.useGKEManagedDriver && (*curVer).lessThan(mustParseVersion("1.17.0"))) { - skipString = skipString + "|VolumeSnapshotDataSource" - } - - // Starting in 1.23, the storage framework uses ephemeral containers for - // testing data written to a pod. This is enabled by looking only at the - // control plane, so it breaks on node skew tests when ephemeral containers - // exist in the API but aren't supported on the node. - if nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.23.0")) && mustParseVersion("1.23.0").lessThan(curVer) { - skipString = skipString + "|volumes.should.store.data|provisioning.should.provision.storage.with.snapshot.data.source" - } - - // Starting in 1.24, the storage framework has a new test case: - // https://github.com/kubernetes/kubernetes/commit/4a076578451aa27e8ac60beec1fd3f23918c5331, - // which breaks the node skew tests when the node version - // is less than 1.24. - if nodeVer != nil && nodeVer.lessThan(mustParseVersion("1.24.0")) { - skipString = skipString + "|provisioning.should.mount.multiple.PV.pointing.to.the.same.storage.on.the.same.node" - } - - // Skip mount options test until we fix the invalid mount options for xfs. - skipString = skipString + "|csi-gcepd-sc-xfs.*provisioning.should.provision.storage.with.mount.options" - - return skipString -} - -func setEnvProject(project string) error { - out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to set gcloud project to %s: %s, err: %v", project, out, err.Error()) - } - - err = os.Setenv("PROJECT", project) - if err != nil { - return err - } - return nil -} - -func runMigrationTests(testParams *testParameters) error { - return runTestsWithConfig(testParams, "--storage.migratedPlugins=kubernetes.io/gce-pd", "") -} - -func runCSITests(testParams *testParameters, reportPrefix string) error { - testDriverConfigFile, err := generateDriverConfigFile(testParams) - if err != nil { - return fmt.Errorf("failed to generated driver config: %w", err) - } - testConfigArg := fmt.Sprintf("--storage.testdriver=%s", testDriverConfigFile) - return runTestsWithConfig(testParams, testConfigArg, reportPrefix) -} - -func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix string) error { - if !*useKubeTest2 && len(testParams.k8sSourceDir) > 0 { - err := os.Chdir(testParams.k8sSourceDir) - if err != nil { - return fmt.Errorf("failed to chdir to k8sSourceDir %s: %v", testParams.k8sSourceDir, err.Error()) - } - } - - kubeconfig, err := getKubeConfig() - if err != nil { - return fmt.Errorf("failed to get kubeconfig: %v", err.Error()) - } - os.Setenv("KUBECONFIG", kubeconfig) - - artifactsDir, ok := os.LookupEnv("ARTIFACTS") - kubetestDumpDir := "" - if ok { - if len(reportPrefix) > 0 { - kubetestDumpDir = filepath.Join(artifactsDir, reportPrefix) - if err := os.MkdirAll(kubetestDumpDir, 0755); err != nil { - return fmt.Errorf("failed to make dump dir %s: %v", kubetestDumpDir, err.Error()) - } - } else { - kubetestDumpDir = artifactsDir - } - } - - focus := testParams.testFocus - skip := testParams.testSkip - // If testParams.snapshotClassFile is empty, then snapshot tests will be automatically skipped. Otherwise confirm - // the right tests are run. - if testParams.snapshotClassFile != "" && strings.Contains(skip, "VolumeSnapshotDataSource") { - return fmt.Errorf("Snapshot class file %s specified, but snapshot tests are skipped: %s", testParams.snapshotClassFile, skip) - } - if testParams.snapshotClassFile != "" { - // Run exactly the snapshot tests, if there is a snapshot class file. - focus = "Driver:\\s*csi-gcepd.*Feature:VolumeSnapshotDataSource" - } - - ginkgoArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s", focus, skip) - - windowsArgs := "" - if testParams.platform == "windows" { - windowsArgs = fmt.Sprintf(" --node-os-distro=%s --allowed-not-ready-nodes=%d", testParams.platform, testParams.allowedNotReadyNodes) - } - ginkgoArgs = ginkgoArgs + windowsArgs - - testArgs := fmt.Sprintf("%s %s", ginkgoArgs, testConfigArg) - - // kubetest2 flags - var runID string - if uid, exists := os.LookupEnv("PROW_JOB_ID"); exists && uid != "" { - // reuse uid for CI use cases - runID = uid - } else { - runID = string(uuid.NewUUID()) - } - - // Usage: kubetest2 [Flags] [DeployerFlags] -- [TesterArgs] - // [Flags] - kubeTest2Args := []string{ - *deploymentStrat, - fmt.Sprintf("--run-id=%s", runID), - "--test=ginkgo", - } - - // [DeployerFlags] - kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgs...) - if kubetestDumpDir != "" { - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--artifacts=%s", kubetestDumpDir)) - } - - kubeTest2Args = append(kubeTest2Args, "--") - - // [TesterArgs] - if len(*testVersion) != 0 { - if *testVersion == "master" { - // the kubernetes binaries should've already been built above because of `--kube-version` - // or by the user if --local-k8s-dir was set, these binaries should be copied to the - // path sent to kubetest2 through its --artifacts path - - // pkg/_artifacts is the default value that kubetests uses for --artifacts - kubernetesTestBinariesPath := filepath.Join(testParams.pkgDir, "_artifacts") - if kubetestDumpDir != "" { - // a custom artifacts dir was set - kubernetesTestBinariesPath = kubetestDumpDir - } - kubernetesTestBinariesPath = filepath.Join(kubernetesTestBinariesPath, runID) - - klog.Infof("Copying kubernetes binaries to path=%s to run the tests", kubernetesTestBinariesPath) - err := copyKubernetesTestBinaries(testParams.k8sSourceDir, kubernetesTestBinariesPath) - if err != nil { - return fmt.Errorf("failed to copy the kubernetes test binaries, err=%v", err.Error()) - } - kubeTest2Args = append(kubeTest2Args, "--use-built-binaries") - } else { - testResourceVersion := *testVersion - if *testVersion != "stable" && *testVersion != "latest" { - // Find the minor version - v, err := parseVersion(*testVersion) - if err != nil { - // Note, this shouldn't happen, as we check flags in main(). - return fmt.Errorf("failed to parse --test-version") - } - testResourceVersion = fmt.Sprintf("stable-%s", v.minorVersion()) - } - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=%s.txt", testResourceVersion)) - } - } - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", focus)) - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", skip)) - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--parallel=%d", testParams.parallel)) - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, windowsArgs)) - - // kubetest flags - kubeTestArgs := []string{ - "--test", - "--ginkgo-parallel", - "--check-version-skew=false", - fmt.Sprintf("--test_args=%s", testArgs), - } - if kubetestDumpDir != "" { - kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir)) - } - kubeTestArgs = append(kubeTestArgs, testParams.cloudProviderArgs...) - - if *useKubeTest2 { - err = runCommand("Running Tests", exec.Command("kubetest2", kubeTest2Args...)) - } else { - err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...)) - } - if err != nil { - return fmt.Errorf("failed to run tests on e2e cluster: %v", err.Error()) - } - - return nil -} - -var ( - kubernetesTestBinaries = []string{ - "kubectl", - "e2e.test", - "ginkgo", - } -) - -// copyKubernetesBinariesForTest copies the common test binaries to the output directory -func copyKubernetesTestBinaries(kuberoot string, outroot string) error { - const dockerizedOutput = "_output/dockerized" - root := filepath.Join(kuberoot, dockerizedOutput, "bin", runtime.GOOS, runtime.GOARCH) - for _, binary := range kubernetesTestBinaries { - source := filepath.Join(root, binary) - dest := filepath.Join(outroot, binary) - if _, err := os.Stat(source); err == nil { - klog.Infof("copying %s to %s", source, dest) - if err := CopyFile(source, dest); err != nil { - return fmt.Errorf("failed to copy %s to %s: %v", source, dest, err.Error()) - } - } else { - return fmt.Errorf("could not find %s: %v", source, err.Error()) - } - } - return nil -} diff --git a/test/k8s-integration/podlogs/podlogs.go b/test/k8s-integration/podlogs/podlogs.go deleted file mode 100644 index 2d97d271..00000000 --- a/test/k8s-integration/podlogs/podlogs.go +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package podlogs enables live capturing of all events and log -// messages for some or all pods in a namespace as they get generated. -// This helps debugging both a running test (what is currently going -// on?) and the output of a CI run (events appear in chronological -// order and output that normally isn't available like the command -// stdout messages are available). -package podlogs - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "path" - "regexp" - "strings" - "sync" - - v1 "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" -) - -// LogOutput determines where output from CopyAllLogs goes. -type LogOutput struct { - // If not nil, errors will be logged here. - StatusWriter io.Writer - - // If not nil, all output goes to this writer with "/:" as prefix. - LogWriter io.Writer - - // Base directory for one log file per container. - // The full path of each log file will be -.log. - LogPathPrefix string -} - -// Matches harmless errors from pkg/kubelet/kubelet_pods.go. -var expectedErrors = regexp.MustCompile(`container .* in pod .* is (terminated|waiting to start|not available)|the server could not find the requested resource`) - -// CopyPodLogs is basically CopyPodLogs for all current or future pods in the given namespace ns -func CopyAllLogs(ctx context.Context, cs clientset.Interface, ns string, to LogOutput) error { - return CopyPodLogs(ctx, cs, ns, "", to) -} - -// CopyPodLogs follows the logs of all containers in pod with the given podName, -// including those that get created in the future, and writes each log -// line as configured in the output options. It does that until the -// context is done or until an error occurs. -// -// If podName is empty, it will follow all pods in the given namespace ns. -// -// Beware that there is currently no way to force log collection -// before removing pods, which means that there is a known race -// between "stop pod" and "collecting log entries". The alternative -// would be a blocking function with collects logs from all currently -// running pods, but that then would have the disadvantage that -// already deleted pods aren't covered. -// -// Another race occurs is when a pod shuts down. Logging stops, but if -// then the pod is not removed from the apiserver quickly enough, logging -// resumes and dumps the old log again. Previously, this was allowed based -// on the assumption that it is better to log twice than miss log messages -// of pods that started and immediately terminated or when logging temporarily -// stopped. -// -// But it turned out to be rather confusing, so now a heuristic is used: if -// log output of a container was already captured, then capturing does not -// resume if the pod is marked for deletion. -func CopyPodLogs(ctx context.Context, cs clientset.Interface, ns, podName string, to LogOutput) error { - options := meta.ListOptions{} - if podName != "" { - options = meta.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", podName), - } - } - watcher, err := cs.CoreV1().Pods(ns).Watch(context.TODO(), options) - - if err != nil { - return fmt.Errorf("cannot create Pod event watcher: %w", err) - } - - go func() { - var m sync.Mutex - // Key is pod/container name, true if currently logging it. - active := map[string]bool{} - // Key is pod/container/container-id, true if we have ever started to capture its output. - started := map[string]bool{} - - check := func() { - m.Lock() - defer m.Unlock() - - pods, err := cs.CoreV1().Pods(ns).List(context.TODO(), options) - if err != nil { - if to.StatusWriter != nil { - fmt.Fprintf(to.StatusWriter, "ERROR: get pod list in %s: %s\n", ns, err) - } - return - } - - for _, pod := range pods.Items { - for i, c := range pod.Spec.Containers { - // sanity check, array should have entry for each container - if len(pod.Status.ContainerStatuses) <= i { - continue - } - name := pod.ObjectMeta.Name + "/" + c.Name - id := name + "/" + pod.Status.ContainerStatuses[i].ContainerID - if active[name] || - // If we have worked on a container before and it has now terminated, then - // there cannot be any new output and we can ignore it. - (pod.Status.ContainerStatuses[i].State.Terminated != nil && - started[id]) || - // State.Terminated might not have been updated although the container already - // stopped running. Also check whether the pod is deleted. - (pod.DeletionTimestamp != nil && started[id]) || - // Don't attempt to get logs for a container unless it is running or has terminated. - // Trying to get a log would just end up with an error that we would have to suppress. - (pod.Status.ContainerStatuses[i].State.Running == nil && - pod.Status.ContainerStatuses[i].State.Terminated == nil) { - continue - } - readCloser, err := logsForPod(ctx, cs, ns, pod.ObjectMeta.Name, - &v1.PodLogOptions{ - Container: c.Name, - Follow: true, - }) - if err != nil { - // We do get "normal" errors here, like trying to read too early. - // We can ignore those. - if to.StatusWriter != nil && - expectedErrors.FindStringIndex(err.Error()) == nil { - fmt.Fprintf(to.StatusWriter, "WARNING: pod log: %s: %s\n", name, err) - } - continue - } - - // Determine where we write. If this fails, we intentionally return without clearing - // the active[name] flag, which prevents trying over and over again to - // create the output file. - var out io.Writer - var closer io.Closer - var prefix string - if to.LogWriter != nil { - out = to.LogWriter - nodeName := pod.Spec.NodeName - if len(nodeName) > 10 { - nodeName = nodeName[0:4] + ".." + nodeName[len(nodeName)-4:] - } - prefix = name + "@" + nodeName + ": " - } else { - var err error - filename := to.LogPathPrefix + pod.ObjectMeta.Name + "-" + c.Name + ".log" - err = os.MkdirAll(path.Dir(filename), 0755) - if err != nil { - if to.StatusWriter != nil { - fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create directory for %s: %s\n", filename, err) - } - return - } - // The test suite might run the same test multiple times, - // so we have to append here. - file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - if to.StatusWriter != nil { - fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create file %s: %s\n", filename, err) - } - return - } - closer = file - out = file - } - go func() { - if closer != nil { - defer closer.Close() - } - first := true - defer func() { - m.Lock() - // If we never printed anything, then also skip the final message. - if !first { - if prefix != "" { - fmt.Fprintf(out, "%s==== end of pod log ====\n", prefix) - } else { - fmt.Fprintf(out, "==== end of pod log for container %s ====\n", name) - } - } - active[name] = false - m.Unlock() - readCloser.Close() - }() - scanner := bufio.NewScanner(readCloser) - for scanner.Scan() { - line := scanner.Text() - // Filter out the expected "end of stream" error message, - // it would just confuse developers who don't know about it. - // Same for attempts to read logs from a container that - // isn't ready (yet?!). - if !strings.HasPrefix(line, "rpc error: code = Unknown desc = Error: No such container:") && - !strings.HasPrefix(line, "unable to retrieve container logs for ") && - !strings.HasPrefix(line, "Unable to retrieve container logs for ") { - if first { - // Because the same log might be written to multiple times - // in different test instances, log an extra line to separate them. - // Also provides some useful extra information. - if prefix == "" { - fmt.Fprintf(out, "==== start of pod log for container %s ====\n", name) - } else { - fmt.Fprintf(out, "%s==== start of pod log ====\n", prefix) - } - first = false - } - fmt.Fprintf(out, "%s%s\n", prefix, line) - } - } - }() - active[name] = true - started[id] = true - } - } - } - - // Watch events to see whether we can start logging - // and log interesting ones. - check() - for { - select { - case <-watcher.ResultChan(): - check() - case <-ctx.Done(): - return - } - } - }() - - return nil -} - -// logsForPod starts reading the logs for a certain pod. If the pod has more than one -// container, opts.Container must be set. Reading stops when the context is done. -// The stream includes formatted error messages and ends with -// -// rpc error: code = Unknown desc = Error: No such container: 41a... -// -// when the pod gets deleted while streaming. -func logsForPod(ctx context.Context, cs clientset.Interface, ns, pod string, opts *v1.PodLogOptions) (io.ReadCloser, error) { - return cs.CoreV1().Pods(ns).GetLogs(pod, opts).Stream(ctx) -} diff --git a/test/k8s-integration/prepull-image.sh b/test/k8s-integration/prepull-image.sh deleted file mode 100755 index 3abd2cbd..00000000 --- a/test/k8s-integration/prepull-image.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -o nounset -set -o pipefail -set -o xtrace - -# This is taken from prepull.yaml. -readonly prepull_daemonset=prepull-test-containers - -wait_on_prepull() -{ - # Wait up to 30 minutes for the test images to be pulled onto the nodes. - retries=180 - while [[ $retries -ge 0 ]];do - ready=$(kubectl get daemonset "${prepull_daemonset}" -o jsonpath="{.status.numberReady}") - required=$(kubectl get daemonset "${prepull_daemonset}" -o jsonpath="{.status.desiredNumberScheduled}") - if [[ $ready -eq $required ]];then - echo "Daemonset $prepull_daemonset ready" - return 0 - fi - ((retries--)) - sleep 10s - done - echo "Timeout waiting for daemonset $prepull_daemonset" - return -1 - -} - -if [[ -z "${PREPULL_YAML}" ]]; then - # Pre-pull all the test images. The images are currently hard-coded. - # Eventually, we should get the list directly from - # https://github.com/kubernetes-sigs/windows-testing/blob/master/images/PullImages.ps1 - curl https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull-1.18.yaml -o prepull.yaml - PREPULL_YAML=prepull.yaml - echo ${PREPULL_YAML} -fi - -kubectl create -f ${PREPULL_YAML} -wait_on_prepull || exit -1 # Error already printed -# Check the status of the pods. -kubectl get pods -o wide -# Delete the pods anyway since pre-pulling is best-effort -kubectl delete -f ${PREPULL_YAML} --wait=true diff --git a/test/k8s-integration/prepull.yaml b/test/k8s-integration/prepull.yaml deleted file mode 100644 index d3af99be..00000000 --- a/test/k8s-integration/prepull.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: prepull-test-containers -spec: - selector: - matchLabels: - prepull-test-images: e2e - template: - metadata: - labels: - prepull-test-images: e2e - spec: - nodeSelector: - kubernetes.io/os: windows - # To see containers that are known to be used in e2e tests, run this - # command against the build-log.txt output from a test run: - # grep -v 'Node Info: &Node' build-log.txt \ - # | grep -ho -E 'e2eteam/([[:alnum:]_-]+):([[:alnum:]./_-]+)' \ - # | sort | uniq - # NOTE: this command captures only a subset of the test containers, - # unfortunately; not all test containers will have their names printed in - # the test output. Running the command against the output from multiple - # test runs is also recommended. Filtering out the 'Node Info' lines - # avoids capturing containers that are only found in the test log - # *because* we prepulled them. Overall this command may be useful for - # detecting newly-used test containers but there's not a great way to - # prune unused containers from this manifest right now. - # - # Examining initImageConfigs() in https://github.com/kubernetes/kubernetes/blob/master/test/utils/image/manifest.go may also help, but many of the containers listed there are only used for Linux tests. - # - # DaemonSets do not support a RestartPolicy other than 'Always', so we - # run ping in each container to keep it alive so that kubernetes does not - # continually restart the containers while we're prepulling. - containers: - - image: registry.k8s.io/e2e-test-images/agnhost:2.36 - name: agnhost-236 - resources: - requests: - cpu: 1m - command: ['cmd.exe', '/c', 'ping -n 1800 127.0.0.1 >NUL'] diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go deleted file mode 100644 index e9bcf6ea..00000000 --- a/test/k8s-integration/utils.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -import ( - "io/ioutil" - "os" - "os/exec" - - "k8s.io/klog/v2" -) - -func runCommand(action string, cmd *exec.Cmd) error { - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stderr - - klog.Infof("%s", action) - klog.Infof("cmd env=%v", cmd.Env) - klog.Infof("cmd args=%s", cmd.Args) - - err := cmd.Start() - if err != nil { - return err - } - - err = cmd.Wait() - if err != nil { - return err - } - return nil -} - -func generateUniqueTmpDir() string { - dir, err := ioutil.TempDir("", "gcp-pd-driver-tmp") - if err != nil { - klog.Fatalf("Error creating temp dir: %w", err) - } - return dir -} - -func removeDir(dir string) { - err := os.RemoveAll(dir) - if err != nil { - klog.Fatalf("Error removing temp dir: %w", err) - } -} - -func ensureVariable(v *string, set bool, msgOnError string) { - if set && len(*v) == 0 { - klog.Fatal(msgOnError) - } else if !set && len(*v) != 0 { - klog.Fatal(msgOnError) - } -} - -func ensureFlag(v *bool, setTo bool, msgOnError string) { - if *v != setTo { - klog.Fatal(msgOnError) - } -} - -func ensureExactlyOneVariableSet(vars []*string, msgOnError string) { - var count int - for _, v := range vars { - if len(*v) != 0 { - count++ - } - } - - if count != 1 { - klog.Fatal(msgOnError) - } -} - -func shredFile(filePath string) { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - klog.V(4).Infof("File %v was not found, skipping shredding", filePath) - return - } - klog.V(4).Infof("Shredding file %v", filePath) - out, err := exec.Command("shred", "--remove", filePath).CombinedOutput() - if err != nil { - klog.V(4).Infof("Failed to shred file %v: %v\nOutput:%v", filePath, err, out) - } - if _, err := os.Stat(filePath); os.IsNotExist(err) { - klog.V(4).Infof("File %v successfully shredded", filePath) - return - } - - // Shred failed Try to remove the file for good meausure - err = os.Remove(filePath) - if err != nil { - klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err) - } -} - -func ensureVariableVal(v *string, val string, msgOnError string) { - if *v != val { - klog.Fatal(msgOnError) - } -} - -func isVariableSet(v *string) bool { - return len(*v) != 0 -} diff --git a/test/k8s-integration/version.go b/test/k8s-integration/version.go deleted file mode 100644 index a0f4c4ff..00000000 --- a/test/k8s-integration/version.go +++ /dev/null @@ -1,170 +0,0 @@ -// main.version is used to parse and compare GKE based versions. -package main - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - "k8s.io/klog/v2" -) - -var ( - versionNum = `(0|[1-9][0-9]*)` - internalPatchVersion = `(\-gke\.[0-9]+)` - - versionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + internalPatchVersion + "?$") - minorVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `$`) - prereleaseVersionRegex = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + `-(?:alpha|beta).*`) - gkeExtraVersionRegex = regexp.MustCompile(`^(?:gke)\.(0|[1-9][0-9]*)$`) -) - -type version struct { - version [4]int -} - -func (v *version) String() string { - if v.version[3] == -2 { - return fmt.Sprintf("%d.%d.%d-prerelease", v.version[0], v.version[1], v.version[2]) - } else if v.version[3] != -1 { - return fmt.Sprintf("%d.%d.%d-gke.%d", v.version[0], v.version[1], v.version[2], v.version[3]) - } - - return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2]) -} - -func (v *version) major() int { - return v.version[0] -} - -func (v *version) minor() int { - return v.version[1] -} - -func (v *version) patch() int { - return v.version[2] -} - -func (v *version) extra() int { - return v.version[3] -} - -func (v *version) isGKEExtraVersion(extrastr string) bool { - return gkeExtraVersionRegex.MatchString(extrastr) -} - -func (v *version) minorVersion() string { - return fmt.Sprintf("%d.%d", v.major(), v.minor()) -} - -func extractGKEExtraVersion(extra string) (int, error) { - m := gkeExtraVersionRegex.FindStringSubmatch(extra) - if len(m) != 2 { - return -1, fmt.Errorf("Invalid GKE Patch version %q", extra) - } - - v, err := strconv.Atoi(m[1]) - if err != nil { - return -1, fmt.Errorf("GKE extra version atoi failed: %q", extra) - } - - if v < 0 { - return -1, fmt.Errorf("GKE extra version check failed: %q", extra) - } - return v, nil -} - -func parseVersion(vs string) (*version, error) { - // If version has a prefix 'v', remove it before parsing. - if strings.HasPrefix(vs, "v") { - vs = vs[1:] - } - - var submatches []string - var v version - var lastIndex int - - switch { - case versionRegex.MatchString(vs): - submatches = versionRegex.FindStringSubmatch(vs) - lastIndex = 4 - case prereleaseVersionRegex.MatchString(vs): - submatches = prereleaseVersionRegex.FindStringSubmatch(vs) - v.version[3] = -2 - lastIndex = 4 - case minorVersionRegex.MatchString(vs): - submatches = minorVersionRegex.FindStringSubmatch(vs) - v.version[2] = -1 - v.version[3] = -1 - lastIndex = 3 - default: - return nil, fmt.Errorf("version %q is invalid", vs) - } - - // submatches[0] is the whole match, [1]..[3] are the version bits, [4] is the extra - for i, sm := range submatches[1:lastIndex] { - var err error - if v.version[i], err = strconv.Atoi(sm); err != nil { - return nil, fmt.Errorf("submatch %q failed atoi conversion", sm) - } - } - - if minorVersionRegex.MatchString(vs) || prereleaseVersionRegex.MatchString(vs) { - return &v, nil - } - - // Ensure 1.X.Y < 1.X.Y-gke.0 - v.version[3] = -1 - if submatches[4] != "" { - extrastr := submatches[4][1:] - if v.isGKEExtraVersion(extrastr) { - ver, err := extractGKEExtraVersion(extrastr) - if err != nil { - return nil, err - } - v.version[3] = ver - } else { - return nil, fmt.Errorf("GKE extra version check failed: %q", extrastr) - } - } - - return &v, nil -} - -// mustParseVersion parses a GKE cluster version. -func mustParseVersion(version string) *version { - v, err := parseVersion(version) - if err != nil { - klog.Fatalf("Failed to parse GKE version: %q", version) - } - return v -} - -// Helper function to compare versions. -// -// -1 -- if left < right -// 0 -- if left == right -// 1 -- if left > right -func (v *version) compare(right *version) int { - for i, b := range v.version { - if b > right.version[i] { - return 1 - } - if b < right.version[i] { - return -1 - } - } - - return 0 -} - -// Compare versions if left is strictly less than right. -func (v *version) lessThan(right *version) bool { - return v.compare(right) < 0 -} - -// Compare versions if left is greater than or equal to right. -func (v *version) atLeast(right *version) bool { - return v.compare(right) >= 0 -} diff --git a/test/k8s-integration/version_test.go b/test/k8s-integration/version_test.go deleted file mode 100644 index 2f1d3dcd..00000000 --- a/test/k8s-integration/version_test.go +++ /dev/null @@ -1,344 +0,0 @@ -package main - -import ( - "strconv" - "testing" -) - -func TestParseVersion(t *testing.T) { - tests := []struct { - version string - expectErr bool - expectedV version - }{ - // Positive test cases. - { - version: "v1.1.1", - expectedV: version{ - version: [4]int{1, 1, 1, -1}, - }, - }, - { - version: "v1.18.0", - expectedV: version{ - version: [4]int{1, 18, 0, -1}, - }, - }, - { - version: "v1.18.0-gke.0", - expectedV: version{ - version: [4]int{1, 18, 0, 0}, - }, - }, - { - version: "1.18.3-gke.10", - expectedV: version{ - version: [4]int{1, 18, 3, 10}, - }, - }, - { - version: "1.18.9", - expectedV: version{ - version: [4]int{1, 18, 9, -1}, - }, - }, - { - version: "1.18.10-gke.10", - expectedV: version{ - version: [4]int{1, 18, 10, 10}, - }, - }, - { - version: "10.18.10-gke.10", - expectedV: version{ - version: [4]int{10, 18, 10, 10}, - }, - }, - { - version: "v1.19.0", - expectedV: version{ - version: [4]int{1, 19, 0, -1}, - }, - }, - { - version: "100.101.102-gke.103", - expectedV: version{ - version: [4]int{100, 101, 102, 103}, - }, - }, - { - version: "1.20", - expectedV: version{ - version: [4]int{1, 20, -1, -1}, - }, - }, - { - version: "v1.26.0-alpha.0.293+6e3d62ca1c9e11", - expectedV: version{ - version: [4]int{1, 26, 0, -2}, - }, - }, - { - version: "v1.26.0-beta.0.25+4b1b42624e06a9", - expectedV: version{ - version: [4]int{1, 26, 0, -2}, - }, - }, - // Negative test cases - { - version: "1", - expectErr: true, - }, - { - version: "-1.18.9", - expectErr: true, - }, - { - version: "1.-18.9", - expectErr: true, - }, - { - version: "1.18.-9", - expectErr: true, - }, - { - version: "1.18.9-gke.-1", - expectErr: true, - }, - { - version: "1.18.9.1", - expectErr: true, - }, - { - version: "1.18.9-1", - expectErr: true, - }, - { - version: "1.18-gke.0", - expectErr: true, - }, - { - version: "1.18.0-zeta.x", - expectErr: true, - }, - { - version: "1.18.0-zeta.alpha.1", - expectErr: true, - }, - { - version: "alpha.3.673+73326ef01d2d7c", - expectErr: true, - }, - { - version: "1.18-alpha.3.673+73326ef01d2d7c", - expectErr: true, - }, - } - - for i, tc := range tests { - t.Run("TestCase"+strconv.Itoa(i), func(t *testing.T) { - gotV, err := parseVersion(tc.version) - if err != nil { - if !tc.expectErr { - t.Fatalf("Got unexpected err: %v", err) - } - return - } - - if err == nil && tc.expectErr { - t.Fatalf("Got no error but expected one with version %s", tc.version) - return - } - - if gotV.version[0] != tc.expectedV.version[0] || - gotV.version[1] != tc.expectedV.version[1] || - gotV.version[2] != tc.expectedV.version[2] || - gotV.version[3] != tc.expectedV.version[3] { - t.Fatalf("Got version: %s, expected: %s", gotV.String(), tc.expectedV.String()) - } - }) - } -} - -func TestIsVersionLessThan(t *testing.T) { - tests := []struct { - leftVersion string - rightVersion string - expectRes bool - }{ - // Positive cases (left < right). - { - leftVersion: "1.17.5-gke.9", - rightVersion: "1.17.6", - expectRes: true, - }, - { - leftVersion: "1.17.5-gke.9", - rightVersion: "1.18.5", - expectRes: true, - }, - { - leftVersion: "1.17.5-gke.9", - rightVersion: "2.17.5", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "1.18.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "1.18.1-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "1.19.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "2.18.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0-gke.0", - rightVersion: "1.18.0-gke.1", - expectRes: true, - }, - { - leftVersion: "1.17.0-gke.9", - rightVersion: "1.18.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0-gke.9", - rightVersion: "1.18.1-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0-gke.9", - rightVersion: "1.19.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0-gke.9", - rightVersion: "2.18.0-gke.0", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "1.18.1", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "1.19.0", - expectRes: true, - }, - { - leftVersion: "1.18.0", - rightVersion: "2.18.0", - expectRes: true, - }, - // Negative test cases.(left == right) - { - leftVersion: "0.0.0", - rightVersion: "0.0.0", - }, - { - leftVersion: "1.1.1", - rightVersion: "1.1.1", - }, - { - leftVersion: "1.18.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "1.18.0-gke.0", - rightVersion: "1.18.0-gke.0", - }, - // Negative test cases.(left > right) - { - leftVersion: "1.17.6", - rightVersion: "1.17.5-gke.9", - }, - { - leftVersion: "1.18.5", - rightVersion: "1.17.5-gke.9", - }, - { - leftVersion: "2.17.5", - rightVersion: "1.17.5-gke.9", - }, - { - leftVersion: "1.18.0-gke.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "1.18.1-gke.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "1.19.0-gke.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "2.18.0-gke.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "1.18.0-gke.1", - rightVersion: "1.18.0-gke.0", - }, - { - leftVersion: "1.18.0-gke.0", - rightVersion: "1.17.0-gke.9", - }, - { - leftVersion: "1.18.1-gke.0", - rightVersion: "1.18.0-gke.9", - }, - { - leftVersion: "1.19.0-gke.0", - rightVersion: "1.18.0-gke.9", - }, - { - leftVersion: "2.18.0-gke.0", - rightVersion: "1.18.0-gke.9", - }, - { - leftVersion: "1.18.1", - rightVersion: "1.18.0", - }, - { - leftVersion: "1.19.0", - rightVersion: "1.18.0", - }, - { - leftVersion: "2.18.0", - rightVersion: "1.18.0", - }, - } - - for i, tc := range tests { - t.Run("TestCase"+strconv.Itoa(i), func(t *testing.T) { - left, err := parseVersion(tc.leftVersion) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - return - } - right, err := parseVersion(tc.rightVersion) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - return - } - - got := left.lessThan(right) - if got != tc.expectRes { - t.Fatalf("Unpexpected compare value: %v, expected %v, left: %q, right: %q", got, tc.expectRes, left.String(), right.String()) - } - }) - } -} diff --git a/test/remote/archiver.go b/test/remote/archiver.go deleted file mode 100644 index aacd0036..00000000 --- a/test/remote/archiver.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - - "k8s.io/klog/v2" -) - -func CreateDriverArchive(archiveName, architecture, pkgPath, binPath string) (string, error) { - klog.V(2).Infof("Building archive...") - tarDir, err := ioutil.TempDir("", "driver-temp-archive") - if err != nil { - return "", fmt.Errorf("failed to create temporary directory %v", err.Error()) - } - defer os.RemoveAll(tarDir) - - // Call the suite function to setup the test package. - err = setupBinaries(architecture, tarDir, pkgPath, binPath) - if err != nil { - return "", fmt.Errorf("failed to setup test package %q: %v", tarDir, err.Error()) - } - - // Build the tar - out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tarDir, ".").CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err.Error(), out) - } - - dir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("failed to get working directory %v", err.Error()) - } - return filepath.Join(dir, archiveName), nil -} - -func setupBinaries(architecture, tarDir, pkgPath, binPath string) error { - klog.V(4).Infof("Making binaries and copying to temp dir...") - out, err := exec.Command("make", "-C", pkgPath, "GOARCH="+architecture).CombinedOutput() - if err != nil { - return fmt.Errorf("Failed to make at %s: %v: %v", pkgPath, string(out), err.Error()) - } - - // Copy binaries - if _, err := os.Stat(binPath); err != nil { - return fmt.Errorf("failed to locate test binary %s: %v", binPath, err.Error()) - } - out, err = exec.Command("cp", binPath, tarDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to copy %q: %v Output: %q", binPath, err.Error(), out) - } - - return nil -} diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go deleted file mode 100644 index 7c86e657..00000000 --- a/test/remote/client-wrappers.go +++ /dev/null @@ -1,311 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "context" - "fmt" - "time" - - csipb "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc" - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - - "k8s.io/apimachinery/pkg/util/wait" -) - -var ( - stdVolCap = &csipb.VolumeCapability{ - AccessType: &csipb.VolumeCapability_Mount{ - Mount: &csipb.VolumeCapability_MountVolume{}, - }, - AccessMode: &csipb.VolumeCapability_AccessMode{ - Mode: csipb.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - blockVolCap = &csipb.VolumeCapability{ - AccessType: &csipb.VolumeCapability_Block{ - Block: &csipb.VolumeCapability_BlockVolume{}, - }, - AccessMode: &csipb.VolumeCapability_AccessMode{ - Mode: csipb.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - stdVolCaps = []*csipb.VolumeCapability{ - stdVolCap, - } - stdCapRange = &csipb.CapacityRange{ - RequiredBytes: common.GbToBytes(20), - } -) - -type CsiClient struct { - conn *grpc.ClientConn - idClient csipb.IdentityClient - nodeClient csipb.NodeClient - ctrlClient csipb.ControllerClient - - endpoint string -} - -func CreateCSIClient(endpoint string) *CsiClient { - return &CsiClient{endpoint: endpoint} -} - -func (c *CsiClient) AssertCSIConnection() error { - var err error - - if err != nil { - return err - } - if c.conn == nil { - var conn *grpc.ClientConn - err = wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) { - conn, err = grpc.Dial( - c.endpoint, - grpc.WithInsecure(), - ) - if err != nil { - klog.Warningf("Client failed to dail endpoint %v", c.endpoint) - return false, nil - } - return true, nil - }) - if err != nil || conn == nil { - return fmt.Errorf("Failed to get client connection: %v", err.Error()) - } - c.conn = conn - c.idClient = csipb.NewIdentityClient(conn) - c.nodeClient = csipb.NewNodeClient(conn) - c.ctrlClient = csipb.NewControllerClient(conn) - } - return nil -} - -func (c *CsiClient) CloseConn() error { - return c.conn.Close() -} - -func (c *CsiClient) CreateVolumeWithCaps(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, caps []*csipb.VolumeCapability, volContentSrc *csipb.VolumeContentSource) (*csipb.Volume, error) { - capRange := &csipb.CapacityRange{ - RequiredBytes: common.GbToBytes(sizeInGb), - } - cvr := &csipb.CreateVolumeRequest{ - Name: volName, - VolumeCapabilities: caps, - Parameters: params, - CapacityRange: capRange, - } - if topReq != nil { - cvr.AccessibilityRequirements = topReq - } - if volContentSrc != nil { - cvr.VolumeContentSource = volContentSrc - } - cresp, err := c.ctrlClient.CreateVolume(context.Background(), cvr) - if err != nil { - return nil, err - } - return cresp.Volume, nil -} - -func (c *CsiClient) CreateVolume(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, volContentSrc *csipb.VolumeContentSource) (*csipb.Volume, error) { - return c.CreateVolumeWithCaps(volName, params, sizeInGb, topReq, stdVolCaps, volContentSrc) -} - -func (c *CsiClient) DeleteVolume(volId string) error { - dvr := &csipb.DeleteVolumeRequest{ - VolumeId: volId, - } - _, err := c.ctrlClient.DeleteVolume(context.Background(), dvr) - return err -} - -func (c *CsiClient) ControllerPublishVolume(volId, nodeId string, forceAttach bool) error { - cpreq := &csipb.ControllerPublishVolumeRequest{ - VolumeId: volId, - NodeId: nodeId, - VolumeCapability: stdVolCap, - Readonly: false, - } - if forceAttach { - cpreq.VolumeContext = map[string]string{ - "force-attach": "true", - } - } - _, err := c.ctrlClient.ControllerPublishVolume(context.Background(), cpreq) - return err -} - -func (c *CsiClient) ListVolumes() (map[string]([]string), error) { - resp, err := c.ctrlClient.ListVolumes(context.Background(), &csipb.ListVolumesRequest{}) - if err != nil { - return nil, err - } - vols := map[string]([]string){} - for _, e := range resp.Entries { - vols[e.Volume.VolumeId] = e.Status.PublishedNodeIds - } - return vols, nil -} - -func (c *CsiClient) ControllerUnpublishVolume(volId, nodeId string) error { - cupreq := &csipb.ControllerUnpublishVolumeRequest{ - VolumeId: volId, - NodeId: nodeId, - } - _, err := c.ctrlClient.ControllerUnpublishVolume(context.Background(), cupreq) - return err -} - -func (c *CsiClient) NodeStageExt4Volume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, stdVolCap) -} - -func (c *CsiClient) NodeStageBlockVolume(volId, stageDir string) error { - return c.NodeStageVolume(volId, stageDir, blockVolCap) -} - -func (c *CsiClient) NodeStageVolume(volId, stageDir string, volumeCap *csipb.VolumeCapability) error { - nodeStageReq := &csipb.NodeStageVolumeRequest{ - VolumeId: volId, - StagingTargetPath: stageDir, - VolumeCapability: volumeCap, - } - _, err := c.nodeClient.NodeStageVolume(context.Background(), nodeStageReq) - return err -} - -func (c *CsiClient) NodeUnstageVolume(volId, stageDir string) error { - nodeUnstageReq := &csipb.NodeUnstageVolumeRequest{ - VolumeId: volId, - StagingTargetPath: stageDir, - } - _, err := c.nodeClient.NodeUnstageVolume(context.Background(), nodeUnstageReq) - return err -} - -func (c *CsiClient) NodeUnpublishVolume(volumeID, publishDir string) error { - nodeUnpublishReq := &csipb.NodeUnpublishVolumeRequest{ - VolumeId: volumeID, - TargetPath: publishDir, - } - _, err := c.nodeClient.NodeUnpublishVolume(context.Background(), nodeUnpublishReq) - return err -} - -func (c *CsiClient) NodePublishVolume(volumeID, stageDir, publishDir string) error { - nodePublishReq := &csipb.NodePublishVolumeRequest{ - VolumeId: volumeID, - StagingTargetPath: stageDir, - TargetPath: publishDir, - VolumeCapability: stdVolCap, - Readonly: false, - } - _, err := c.nodeClient.NodePublishVolume(context.Background(), nodePublishReq) - return err -} - -func (c *CsiClient) NodePublishBlockVolume(volumeID, stageDir, publishDir string) error { - nodePublishReq := &csipb.NodePublishVolumeRequest{ - VolumeId: volumeID, - StagingTargetPath: stageDir, - TargetPath: publishDir, - VolumeCapability: blockVolCap, - Readonly: false, - } - _, err := c.nodeClient.NodePublishVolume(context.Background(), nodePublishReq) - return err -} - -func (c *CsiClient) ControllerExpandVolume(volumeID string, sizeGb int64) error { - controllerExpandReq := &csipb.ControllerExpandVolumeRequest{ - VolumeId: volumeID, - CapacityRange: &csipb.CapacityRange{ - RequiredBytes: common.GbToBytes(sizeGb), - }, - } - _, err := c.ctrlClient.ControllerExpandVolume(context.Background(), controllerExpandReq) - return err -} - -func (c *CsiClient) NodeExpandVolume(volumeID, volumePath string, sizeGb int64) (*csipb.NodeExpandVolumeResponse, error) { - nodeExpandReq := &csipb.NodeExpandVolumeRequest{ - VolumeId: volumeID, - VolumePath: volumePath, - CapacityRange: &csipb.CapacityRange{ - RequiredBytes: common.GbToBytes(sizeGb), - }, - } - return c.nodeClient.NodeExpandVolume(context.Background(), nodeExpandReq) -} - -func (c *CsiClient) NodeGetInfo() (*csipb.NodeGetInfoResponse, error) { - resp, err := c.nodeClient.NodeGetInfo(context.Background(), &csipb.NodeGetInfoRequest{}) - return resp, err -} - -func (c *CsiClient) NodeGetVolumeStats(volumeID, volumePath string) (available, capacity, used, inodesFree, inodes, inodesUsed int64, err error) { - resp, err := c.nodeClient.NodeGetVolumeStats(context.Background(), &csipb.NodeGetVolumeStatsRequest{ - VolumeId: volumeID, - VolumePath: volumePath, - }) - if err != nil { - return - } - for _, usage := range resp.Usage { - if usage == nil { - continue - } - unit := usage.GetUnit() - switch unit { - case csipb.VolumeUsage_BYTES: - available = usage.GetAvailable() - capacity = usage.GetTotal() - used = usage.GetUsed() - case csipb.VolumeUsage_INODES: - inodesFree = usage.GetAvailable() - inodes = usage.GetTotal() - inodesUsed = usage.GetUsed() - default: - err = fmt.Errorf("unknown key %s in usage", unit.String()) - return - } - } - return -} - -func (c *CsiClient) CreateSnapshot(snapshotName, sourceVolumeId string, params map[string]string) (string, error) { - - csr := &csipb.CreateSnapshotRequest{ - Name: snapshotName, - SourceVolumeId: sourceVolumeId, - Parameters: params, - } - cresp, err := c.ctrlClient.CreateSnapshot(context.Background(), csr) - if err != nil { - return "", err - } - return cresp.GetSnapshot().GetSnapshotId(), nil -} - -func (c *CsiClient) DeleteSnapshot(snapshotID string) error { - dsr := &csipb.DeleteSnapshotRequest{ - SnapshotId: snapshotID, - } - _, err := c.ctrlClient.DeleteSnapshot(context.Background(), dsr) - return err -} diff --git a/test/remote/instance.go b/test/remote/instance.go deleted file mode 100644 index 8aa5b2fd..00000000 --- a/test/remote/instance.go +++ /dev/null @@ -1,429 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "context" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - "time" - - "golang.org/x/oauth2/google" - computealpha "google.golang.org/api/compute/v0.alpha" - computebeta "google.golang.org/api/compute/v0.beta" - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/googleapi" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" - "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" - gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" -) - -const ( - defaultFirewallRule = "default-allow-ssh" - - // timestampFormat is the timestamp format used in the e2e directory name. - timestampFormat = "20060102T150405" -) - -type InstanceInfo struct { - project string - architecture string - zone string - name string - machineType string - - // External IP is filled in after instance creation - externalIP string - - computeService *compute.Service -} - -func (i *InstanceInfo) GetIdentity() (string, string, string) { - return i.project, i.zone, i.name -} - -func (i *InstanceInfo) GetName() string { - return i.name -} - -func (i *InstanceInfo) GetNodeID() string { - return common.CreateNodeID(i.project, i.zone, i.name) -} - -func CreateInstanceInfo(project, instanceArchitecture, instanceZone, name, machineType string, cs *compute.Service) (*InstanceInfo, error) { - return &InstanceInfo{ - project: project, - architecture: instanceArchitecture, - zone: instanceZone, - name: name, - machineType: machineType, - computeService: cs, - }, nil -} - -// Provision a gce instance using image -func (i *InstanceInfo) CreateOrGetInstance(imageURL, serviceAccount string) error { - var err error - var instance *compute.Instance - klog.V(4).Infof("Creating instance: %v", i.name) - - myuuid := string(uuid.NewUUID()) - - err = i.createDefaultFirewallRule() - if err != nil { - return fmt.Errorf("Failed to create firewall rule: %v", err.Error()) - } - - newInst := &compute.Instance{ - Name: i.name, - MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", i.zone, i.machineType), - NetworkInterfaces: []*compute.NetworkInterface{ - { - AccessConfigs: []*compute.AccessConfig{ - { - Type: "ONE_TO_ONE_NAT", - Name: "External NAT", - }, - }}, - }, - Disks: []*compute.AttachedDisk{ - { - AutoDelete: true, - Boot: true, - Type: "PERSISTENT", - InitializeParams: &compute.AttachedDiskInitializeParams{ - DiskName: "my-root-pd-" + myuuid, - SourceImage: imageURL, - }, - }, - }, - } - - saObj := &compute.ServiceAccount{ - Email: serviceAccount, - Scopes: []string{"https://www.googleapis.com/auth/cloud-platform"}, - } - newInst.ServiceAccounts = []*compute.ServiceAccount{saObj} - - if pubkey, ok := os.LookupEnv("JENKINS_GCE_SSH_PUBLIC_KEY_FILE"); ok { - klog.V(4).Infof("JENKINS_GCE_SSH_PUBLIC_KEY_FILE set to %v, adding public key to Instance", pubkey) - meta, err := generateMetadataWithPublicKey(pubkey) - if err != nil { - return err - } - newInst.Metadata = meta - } - - // If instance exists but machine-type doesn't match, delete instance - curInst, _ := i.computeService.Instances.Get(i.project, i.zone, newInst.Name).Do() - if curInst != nil { - if !strings.Contains(curInst.MachineType, newInst.MachineType) { - klog.V(4).Infof("Instance machine type doesn't match the required one. Delete instance.") - if _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do(); err != nil { - return err - } - - start := time.Now() - err := wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance to be deleted. %v elapsed", time.Since(start)) - if curInst, _ = i.computeService.Instances.Get(i.project, i.zone, i.name).Do(); curInst != nil { - return false, nil - } - return true, nil - }) - if err != nil { - return err - } - } - } - - if curInst == nil { - op, err := i.computeService.Instances.Insert(i.project, i.zone, newInst).Do() - klog.V(4).Infof("Inserted instance %v in project: %v, zone: %v", newInst.Name, i.project, i.zone) - if err != nil { - ret := fmt.Sprintf("could not create instance %s: API error: %v", i.name, err.Error()) - if op != nil { - ret = fmt.Sprintf("%s. op error: %v", ret, op.Error) - } - return errors.New(ret) - } else if op.Error != nil { - return fmt.Errorf("could not create instance %s: %+v", i.name, op.Error) - } - } else { - klog.V(4).Infof("Compute service GOT instance %v, skipping instance creation", newInst.Name) - } - - start := time.Now() - err = wait.Poll(15*time.Second, 5*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", i.name, time.Since(start)) - - instance, err = i.computeService.Instances.Get(i.project, i.zone, i.name).Do() - if err != nil { - klog.Errorf("Failed to get instance %v: %w", i.name, err) - return false, nil - } - - if strings.ToUpper(instance.Status) != "RUNNING" { - klog.Warningf("instance %s not in state RUNNING, was %s", i.name, instance.Status) - return false, nil - } - - externalIP := getexternalIP(instance) - if len(externalIP) > 0 { - i.externalIP = externalIP - } - - if sshOut, err := i.SSHCheckAlive(); err != nil { - err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", i.name, err.Error()) - klog.Warningf("SSH encountered an error: %v, output: %v", err, sshOut) - return false, nil - } - klog.V(4).Infof("Instance %v in state RUNNING and available by SSH", i.name) - return true, nil - }) - - // If instance didn't reach running state in time, return with error now. - if err != nil { - return err - } - - // Instance reached running state in time, make sure that cloud-init is complete - klog.V(2).Infof("Instance %v has been created successfully", i.name) - return nil -} - -func (i *InstanceInfo) DeleteInstance() { - klog.V(4).Infof("Deleting instance %q", i.name) - _, err := i.computeService.Instances.Delete(i.project, i.zone, i.name).Do() - if err != nil { - if isGCEError(err, "notFound") { - return - } - klog.Errorf("Error deleting instance %q: %w", i.name, err) - } -} - -func (i *InstanceInfo) DetachDisk(diskName string) error { - klog.V(4).Infof("Detaching disk %q", diskName) - op, err := i.computeService.Instances.DetachDisk(i.project, i.zone, i.name, diskName).Do() - if err != nil { - if isGCEError(err, "notFound") { - return nil - } - klog.Errorf("Error deleting disk %q: %w", diskName, err) - } - - start := time.Now() - if err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) { - klog.V(2).Infof("Waiting for disk %q to be detached from instance %q. %v elapsed", diskName, i.name, time.Since(start)) - - op, err = i.computeService.ZoneOperations.Get(i.project, i.zone, op.Name).Do() - if err != nil { - return true, fmt.Errorf("Failed to get operation %q, err: %v", op.Name, err) - } - return op.Status == "DONE", nil - }); err != nil { - return err - } - - klog.V(4).Infof("Disk %q has been successfully detached from instance %q\n%v", diskName, i.name, op.Error) - return nil -} - -func getexternalIP(instance *compute.Instance) string { - for i := range instance.NetworkInterfaces { - ni := instance.NetworkInterfaces[i] - for j := range ni.AccessConfigs { - ac := ni.AccessConfigs[j] - if len(ac.NatIP) > 0 { - return ac.NatIP - } - } - } - return "" -} - -func getTimestamp() string { - return fmt.Sprintf(time.Now().Format(timestampFormat)) -} - -// Create default SSH filewall rule if it does not exist -func (i *InstanceInfo) createDefaultFirewallRule() error { - var err error - klog.V(4).Infof("Creating default firewall rule %s...", defaultFirewallRule) - - if _, err = i.computeService.Firewalls.Get(i.project, defaultFirewallRule).Do(); err != nil { - klog.V(4).Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule) - f := &compute.Firewall{ - Name: defaultFirewallRule, - Allowed: []*compute.FirewallAllowed{ - { - IPProtocol: "tcp", - Ports: []string{"22"}, - }, - }, - } - _, err = i.computeService.Firewalls.Insert(i.project, f).Do() - if err != nil { - if gce.IsGCEError(err, "alreadyExists") { - klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) - return nil - } - return fmt.Errorf("Failed to insert required default SSH firewall Rule %v: %v", defaultFirewallRule, err.Error()) - } - } else { - klog.V(4).Infof("Default firewall rule %v already exists, skipping creation", defaultFirewallRule) - } - return nil -} - -func GetComputeClient() (*compute.Service, error) { - const retries = 10 - const backoff = time.Second * 6 - - klog.V(4).Infof("Getting compute client...") - - // Setup the gce client for provisioning instances - // Getting credentials on gce jenkins is flaky, so try a couple times - var err error - var cs *compute.Service - for i := 0; i < retries; i++ { - if i > 0 { - time.Sleep(backoff) - } - - var client *http.Client - client, err = google.DefaultClient(context.Background(), compute.ComputeScope) - if err != nil { - continue - } - - cs, err = compute.New(client) - if err != nil { - continue - } - return cs, nil - } - return nil, err -} - -func GetComputeAlphaClient() (*computealpha.Service, error) { - const retries = 10 - const backoff = time.Second * 6 - - klog.V(4).Infof("Getting compute client...") - - // Setup the gce client for provisioning instances - // Getting credentials on gce jenkins is flaky, so try a couple times - var err error - var cs *computealpha.Service - for i := 0; i < retries; i++ { - if i > 0 { - time.Sleep(backoff) - } - - var client *http.Client - client, err = google.DefaultClient(context.Background(), computealpha.ComputeScope) - if err != nil { - continue - } - - cs, err = computealpha.New(client) - if err != nil { - continue - } - return cs, nil - } - return nil, err -} - -func GetComputeBetaClient() (*computebeta.Service, error) { - const retries = 10 - const backoff = time.Second * 6 - - klog.V(4).Infof("Getting compute client...") - - // Setup the gce client for provisioning instances - // Getting credentials on gce jenkins is flaky, so try a couple times - var err error - var cs *computebeta.Service - for i := 0; i < retries; i++ { - if i > 0 { - time.Sleep(backoff) - } - - var client *http.Client - client, err = google.DefaultClient(context.Background(), computebeta.ComputeScope) - if err != nil { - continue - } - - cs, err = computebeta.New(client) - if err != nil { - continue - } - return cs, nil - } - return nil, err -} - -func generateMetadataWithPublicKey(pubKeyFile string) (*compute.Metadata, error) { - publicKeyByte, err := ioutil.ReadFile(pubKeyFile) - if err != nil { - return nil, err - } - - publicKey := string(publicKeyByte) - - // Take username and prepend it to the public key - tokens := strings.Split(publicKey, " ") - if len(tokens) != 3 { - return nil, fmt.Errorf("Public key not comprised of 3 parts, instead was: %v", publicKey) - } - publicKey = strings.TrimSpace(tokens[2]) + ":" + publicKey - newMeta := &compute.Metadata{ - Items: []*compute.MetadataItems{ - { - Key: "ssh-keys", - Value: &publicKey, - }, - }, - } - return newMeta, nil -} - -// isGCEError returns true if given error is a googleapi.Error with given -// reason (e.g. "resourceInUseByAnotherResource") -func isGCEError(err error, reason string) bool { - var apiErr *googleapi.Error - if !errors.As(err, &apiErr) { - return false - } - - for _, e := range apiErr.Errors { - if e.Reason == reason { - return true - } - } - return false -} diff --git a/test/remote/runner.go b/test/remote/runner.go deleted file mode 100644 index 2fb8d8c9..00000000 --- a/test/remote/runner.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "path" - "path/filepath" - "strconv" - "strings" - - "k8s.io/klog/v2" -) - -func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) { - - // Create the temp staging directory - klog.V(4).Infof("Staging test binaries on %q", i.name) - - // Do not sudo here, so that we can use scp to copy test archive to the directdory. - if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil { - // Exit failure with the error - return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err.Error(), output) - } - - // Copy the archive to the staging directory - if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil { - // Exit failure with the error - return -1, fmt.Errorf("failed to copy test archive: %v, output: %q", err.Error(), output) - } - - // Extract the archive - archiveName := path.Base(archivePath) - cmd := getSSHCommand(" && ", - fmt.Sprintf("cd %s", remoteWorkspace), - fmt.Sprintf("tar -xzvf ./%s", archiveName), - ) - klog.V(4).Infof("Extracting tar on %q", i.name) - // Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but - // we want the extracted files to be owned by the current user. - if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil { - // Exit failure with the error - return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err.Error(), output) - } - - klog.V(4).Infof("Starting driver on %q", i.name) - // When the process is killed the driver should close the TCP endpoint, then we want to download the logs - output, err := i.SSH(driverRunCmd) - if err != nil { - // Exit failure with the error - return -1, fmt.Errorf("failed start driver, got output: %v, error: %v", output, err.Error()) - } - - // Get the driver PID - // ps -aux | grep /tmp/gce-pd-e2e-0180801T114407/gce-pd-csi-driver | awk '{print $2}' - driverPIDCmd := getSSHCommand(" | ", - "ps -aux", - fmt.Sprintf("grep %s", remoteWorkspace), - "grep -v grep", - // All ye who try to deal with escaped/non-escaped quotes with exec beware. - //`awk "{print \$2}"`, - ) - driverPIDString, err := i.SSHNoSudo("sh", "-c", driverPIDCmd) - if err != nil { - // Exit failure with the error - return -1, fmt.Errorf("failed to get PID of driver, got output: %v, error: %v", output, err.Error()) - } - - driverPID, err := strconv.Atoi(strings.Fields(driverPIDString)[1]) - if err != nil { - return -1, fmt.Errorf("failed to convert driver PID from string %s to int: %v", driverPIDString, err.Error()) - } - - return driverPID, nil -} - -func NewWorkspaceDir(workspaceDirPrefix string) string { - return filepath.Join("/tmp", workspaceDirPrefix+getTimestamp()) -} diff --git a/test/remote/setup-teardown.go b/test/remote/setup-teardown.go deleted file mode 100644 index b20893c0..00000000 --- a/test/remote/setup-teardown.go +++ /dev/null @@ -1,140 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "os" - - compute "google.golang.org/api/compute/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/klog/v2" -) - -// TestContext holds the CSI Client handle to a remotely connected Driver -// as well as a handle to the Instance that the driver is running on -type TestContext struct { - Instance *InstanceInfo - Client *CsiClient - proc *processes -} - -// ClientConfig contains all the parameters required to package a new -// driver and run it remotely on a GCE Instance -type ClientConfig struct { - // Absolute path of the package - PkgPath string - // Absolute path of the driver binary to copy remotely - BinPath string - // Path on remote instance workspace - WorkspaceDir string - // Command to run on remote instance to start the driver - RunDriverCmd string - // Port to use as SSH tunnel on both remote and local side. - Port string -} - -type processes struct { - sshTunnel int - remoteDriver int -} - -// SetupInstance sets up the specified GCE Instance for E2E testing and returns a handle to the instance object for future use. -func SetupInstance(instanceProject, instanceArchitecture, instanceZone, instanceName, instanceMachineType, instanceServiceAccount, instanceImageURL string, cs *compute.Service) (*InstanceInfo, error) { - // Create the instance in the requisite zone - instance, err := CreateInstanceInfo(instanceProject, instanceArchitecture, instanceZone, instanceName, instanceMachineType, cs) - if err != nil { - return nil, err - } - - err = instance.CreateOrGetInstance(instanceImageURL, instanceServiceAccount) - if err != nil { - return nil, err - } - return instance, nil -} - -// SetupNewDriverAndClient gets the driver binary, runs it on the provided instance and connects -// a CSI client to it through SHH tunnelling. It returns a TestContext with both a handle to the instance -// that the driver is on and the CSI Client object to make CSI calls to the remote driver. -func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) { - archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID()) - archivePath, err := CreateDriverArchive(archiveName, instance.architecture, config.PkgPath, config.BinPath) - if err != nil { - return nil, err - } - defer func() { - err = os.Remove(archivePath) - if err != nil { - klog.Warningf("Failed to remove archive file %s: %v", archivePath, err) - } - }() - - // Upload archive to instance and run binaries - driverPID, err := instance.UploadAndRun(archivePath, config.WorkspaceDir, config.RunDriverCmd) - if err != nil { - return nil, err - } - - // Create an SSH tunnel from port to port - sshPID, err := instance.CreateSSHTunnel(config.Port, config.Port) - if err != nil { - return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", sshPID, err.Error()) - } - - client := CreateCSIClient(fmt.Sprintf("localhost:%s", config.Port)) - err = client.AssertCSIConnection() - if err != nil { - return nil, fmt.Errorf("asserting csi connection failed with: %v", err.Error()) - } - - return &TestContext{ - Instance: instance, - Client: client, - proc: &processes{ - sshTunnel: sshPID, - remoteDriver: driverPID, - }, - }, nil -} - -// TeardownDriverAndClient closes the CSI Client connection, closes the SSH tunnel -// Kills the driver process on the GCE instance, and cleans up the remote driver workspace -func TeardownDriverAndClient(context *TestContext) error { - // Close the client connection - err := context.Client.CloseConn() - if err != nil { - return fmt.Errorf("failed to close CSI Client connection: %v", err.Error()) - } - // Close the SSH tunnel - proc, err := os.FindProcess(context.proc.sshTunnel) - if err != nil { - return fmt.Errorf("unable to efind process for ssh tunnel %v: %v", context.proc.sshTunnel, err.Error()) - } - if err = proc.Kill(); err != nil { - return fmt.Errorf("failed to kill ssh tunnel process %v: %v", context.proc.sshTunnel, err.Error()) - } - - // Kill the driver process on remote - cmd := fmt.Sprintf("kill %v", context.proc.remoteDriver) - output, err := context.Instance.SSH(cmd) - if err != nil { - return fmt.Errorf("failed to kill driver on remote instance, got output %s: %v", output, err.Error()) - } - - return nil -} diff --git a/test/remote/ssh.go b/test/remote/ssh.go deleted file mode 100644 index 521b2519..00000000 --- a/test/remote/ssh.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package remote - -import ( - "fmt" - "os" - "os/exec" - "os/user" - "strings" - - "k8s.io/klog/v2" -) - -var ( - sshOption = "-o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o LogLevel=ERROR" - sshDefaultKey string -) - -func init() { - usr, err := user.Current() - if err != nil { - klog.Fatal(err.Error()) - } - sshDefaultKey = fmt.Sprintf("%s/.ssh/google_compute_engine", usr.HomeDir) - -} - -// GetHostnameOrIP converts hostname into ip and apply user if necessary. -func (i *InstanceInfo) GetSSHTarget() string { - var target string - - if _, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { - target = fmt.Sprintf("prow@%s", i.externalIP) - } else { - target = fmt.Sprintf("%s", i.externalIP) - } - return target -} - -// getSSHCommand handles proper quoting so that multiple commands are executed in the same shell over ssh -func getSSHCommand(sep string, args ...string) string { - return fmt.Sprintf("'%s'", strings.Join(args, sep)) -} - -// SSH executes ssh command with runSSHCommand as root. The `sudo` makes sure that all commands -// are executed by root, so that there won't be permission mismatch between different commands. -func (i *InstanceInfo) SSH(cmd ...string) (string, error) { - return runSSHCommand("ssh", append([]string{i.GetSSHTarget(), "--", "sudo"}, cmd...)...) -} - -func (i *InstanceInfo) CreateSSHTunnel(localPort, serverPort string) (int, error) { - args := []string{"-nNT", "-L", fmt.Sprintf("%s:localhost:%s", localPort, serverPort), i.GetSSHTarget()} - if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { - klog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk) - args = append([]string{"-i", pk}, args...) - } else { - args = append([]string{"-i", sshDefaultKey}, args...) - } - args = append(strings.Split(sshOption, " "), args...) - cmd := exec.Command("ssh", args...) - err := cmd.Start() - if err != nil { - return 0, err - } - - return cmd.Process.Pid, nil -} - -// SSHNoSudo executes ssh command with runSSHCommand as normal user. Sometimes we need this, -// for example creating a directory that we'll copy files there with scp. -func (i *InstanceInfo) SSHNoSudo(cmd ...string) (string, error) { - return runSSHCommand("ssh", append([]string{i.GetSSHTarget(), "--"}, cmd...)...) -} - -// SSHCheckAlive just pings the server quickly to check whether it is reachable by SSH -func (i *InstanceInfo) SSHCheckAlive() (string, error) { - return runSSHCommand("ssh", []string{i.GetSSHTarget(), "-o", "ConnectTimeout=10", "--", "echo"}...) -} - -// runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options -func runSSHCommand(cmd string, args ...string) (string, error) { - if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok { - klog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk) - args = append([]string{"-i", pk}, args...) - } else { - args = append([]string{"-i", sshDefaultKey}, args...) - } - args = append(strings.Split(sshOption, " "), args...) - - klog.V(4).Infof("Executing SSH command: %v %v", cmd, args) - - output, err := exec.Command(cmd, args...).CombinedOutput() - if err != nil { - return string(output), fmt.Errorf("command [%s %s] failed with error: %v", cmd, strings.Join(args, " "), err.Error()) - } - return string(output), nil -} diff --git a/test/run-e2e-local.sh b/test/run-e2e-local.sh index 6e60ccc8..330ca37f 100755 --- a/test/run-e2e-local.sh +++ b/test/run-e2e-local.sh @@ -11,4 +11,9 @@ readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver # This requires application default credentials to be set up, eg by # `gcloud auth application-default login` -ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" --v=6 --logtostderr +CLOUDTOP_HOST= +if hostname | grep -q c.googlers.com ; then + CLOUDTOP_HOST=--cloudtop-host +fi + +ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" "${CLOUDTOP_HOST}" --v=6 --logtostderr diff --git a/test/run-k8s-integration-ci.sh b/test/run-k8s-integration-ci.sh index 5f199615..90a1051d 100755 --- a/test/run-k8s-integration-ci.sh +++ b/test/run-k8s-integration-ci.sh @@ -33,6 +33,15 @@ readonly test_disk_image_snapshot=${TEST_DISK_IMAGE_SNAPSHOT:-true} readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault" +# Install golang. +version=1.22.3 +wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q +# Remove the existing GoLang installation directory +rm -rf /usr/local/go && tar -xzf go_tar.tar.gz -C /usr/local +# Add the GoLang binary directory to systems PATH env, allowing prow tests +# to run go commands with this go version. +export PATH=$PATH:/usr/local/go/bin && go version && rm go_tar.tar.gz + storage_classes=sc-balanced.yaml,sc-ssd.yaml,sc-xfs.yaml if [[ $test_pd_labels = true ]] ; then diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index a72db9eb..03813ffb 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -17,8 +17,8 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" readonly do_k8s_build="${GCE_PD_DO_K8S_BUILD:-false}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest} -readonly kube_version=${GCE_PD_KUBE_VERSION:-stable} -readonly test_version=${TEST_VERSION:-stable} +readonly kube_version=${GCE_PD_KUBE_VERSION:-latest} +readonly test_version=${TEST_VERSION:-latest} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly gce_region=${GCE_CLUSTER_REGION:-} readonly image_type=${IMAGE_TYPE:-cos_containerd} diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go index 5e8d5344..398c457b 100644 --- a/test/sanity/sanity_test.go +++ b/test/sanity/sanity_test.go @@ -80,14 +80,20 @@ func TestSanity(t *testing.T) { t.Fatalf("Failed to get cloud provider: %v", err.Error()) } + fallbackRequisiteZones := []string{} + enableStoragePools := false + multiZoneVolumeHandleConfig := driver.MultiZoneVolumeHandleConfig{} + listVolumesConfig := driver.ListVolumesConfig{} + mounter := mountmanager.NewFakeSafeMounter() deviceUtils := deviceutils.NewFakeDeviceUtils(true) // Initialize GCE Driver identityServer := driver.NewIdentityServer(gceDriver) - controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute) - nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(mounter), &fakeCryptMapper{}) - err = gceDriver.SetupGCEDriver(driverName, vendorVersion, extraLabels, identityServer, controllerServer, nodeServer) + controllerServer := driver.NewControllerServer(gceDriver, cloudProvider, 0, 5*time.Minute, fallbackRequisiteZones, enableStoragePools, multiZoneVolumeHandleConfig, listVolumesConfig) + fakeStatter := mountmanager.NewFakeStatterWithOptions(mounter, mountmanager.FakeStatterOptions{IsBlock: false}) + nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), fakeStatter, &fakeCryptMapper{}) + err = gceDriver.SetupGCEDriver(driverName, vendorVersion, extraLabels, nil, identityServer, controllerServer, nodeServer) if err != nil { t.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error()) } @@ -111,7 +117,7 @@ func TestSanity(t *testing.T) { }() go func() { - gceDriver.Run(endpoint, 10000) + gceDriver.Run(endpoint, 10000, false) }() // TODO(#818): Fix failing tests and remove test skip flag.