diff --git a/.github/workflows/benchmark-action.yaml b/.github/workflows/benchmark-action.yaml index 8071c8d5c..5a1d05857 100644 --- a/.github/workflows/benchmark-action.yaml +++ b/.github/workflows/benchmark-action.yaml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v3.1.0 - uses: actions/setup-go@v3 # default version of go is 1.10 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Install Carvel Tools run: ./hack/install-deps.sh # Run benchmark with `go test -bench` and stores the output to a file diff --git a/.github/workflows/dependency-updater.yml b/.github/workflows/dependency-updater.yml index c44fb8bc5..06c867e23 100644 --- a/.github/workflows/dependency-updater.yml +++ b/.github/workflows/dependency-updater.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Update Dependencies File run: go run ./hack/dependencies.go update - name: Create Pull Request diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index b1aa6c02e..217590738 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - uses: actions/checkout@v3.5.2 with: fetch-depth: '0' diff --git a/.github/workflows/release-process.yml b/.github/workflows/release-process.yml index 1e854db21..564de478c 100644 --- a/.github/workflows/release-process.yml +++ b/.github/workflows/release-process.yml @@ -35,7 +35,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Run release script run: | diff --git a/.github/workflows/test-gh.yml b/.github/workflows/test-gh.yml index 737090a43..a56c1b05a 100644 --- a/.github/workflows/test-gh.yml +++ b/.github/workflows/test-gh.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Check out code uses: actions/checkout@v3.1.0 with: diff --git a/.github/workflows/test-kctrl-gh.yml b/.github/workflows/test-kctrl-gh.yml index 795a5fece..ea6cda32a 100644 --- a/.github/workflows/test-kctrl-gh.yml +++ b/.github/workflows/test-kctrl-gh.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Check out code uses: actions/checkout@v3.1.0 with: diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml index 3d7f89038..602cb062f 100644 --- a/.github/workflows/trivy-scan.yml +++ b/.github/workflows/trivy-scan.yml @@ -69,7 +69,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Build the kapp-controller artifacts run: | ./hack/install-deps.sh diff --git a/.github/workflows/upgrade-testing.yml b/.github/workflows/upgrade-testing.yml index ccf07286b..b35383a40 100644 --- a/.github/workflows/upgrade-testing.yml +++ b/.github/workflows/upgrade-testing.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.10 + go-version: 1.20.11 - name: Check out code uses: actions/checkout@v3.1.0 - name: Install Carvel Tools diff --git a/Dockerfile b/Dockerfile index a25c3cc05..ed796128b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.20.10 AS deps +FROM --platform=$BUILDPLATFORM golang:1.20.11 AS deps ARG TARGETOS TARGETARCH KCTRL_VER=development WORKDIR /workspace diff --git a/cli/go.mod b/cli/go.mod index d2fbd8042..24c920060 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -13,15 +13,15 @@ require ( github.com/otiai10/copy v1.0.2 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.5 + github.com/stretchr/testify v1.8.0 github.com/vmware-tanzu/carvel-kapp-controller v0.39.0 github.com/vmware-tanzu/carvel-vendir v0.24.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.25.0 - k8s.io/apiextensions-apiserver v0.25.0 - k8s.io/apimachinery v0.25.0 - k8s.io/client-go v0.25.0 + k8s.io/api v0.25.13 + k8s.io/apiextensions-apiserver v0.25.13 + k8s.io/apimachinery v0.25.13 + k8s.io/client-go v0.25.13 sigs.k8s.io/controller-runtime v0.13.0 sigs.k8s.io/yaml v1.3.0 ) @@ -57,7 +57,7 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/go-version v1.3.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -92,7 +92,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/component-base v0.25.0 // indirect + k8s.io/component-base v0.25.13 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect diff --git a/cli/go.sum b/cli/go.sum index 8d7b88263..075ced17a 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -461,8 +461,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -631,14 +632,14 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -772,8 +773,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -832,7 +833,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1356,28 +1357,28 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= -k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= -k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= -k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= +k8s.io/api v0.25.13 h1:nOQWK5/ngLIG2CqmVV7uTFDsPCGkDk4kIGJ26t2AwIo= +k8s.io/api v0.25.13/go.mod h1:yGpHyrivZ0enqWqT5s1pN98a4Q834rZkIUEABpleEtw= +k8s.io/apiextensions-apiserver v0.25.13 h1:tdB/ULSvlbZIecU2bDOoCb46Ozk5VyLL6IPysC8gUpE= +k8s.io/apiextensions-apiserver v0.25.13/go.mod h1:uqYfnY2GOYXlfpnT6No9dxrxDbMjaYvh/Bpm0iNMYnA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= -k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= +k8s.io/apimachinery v0.25.13 h1:byRHkSinOOVdo0pvjdblauFYfwAnx+JB8Pqi9w9weik= +k8s.io/apimachinery v0.25.13/go.mod h1:IFwbcNi3gKkfDhuy0VYu3+BwbxbiIov3p6FR8ge1Epc= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= -k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= +k8s.io/client-go v0.25.13 h1:Wan/8RXVNxSgFI/wMfWwJjmLglRYuLItytMWNiGo9LY= +k8s.io/client-go v0.25.13/go.mod h1:b2on3RSCwHdmvnUQx4/bkgMAs19M7BlUDze3WJuK0TE= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.25.0 h1:haVKlLkPCFZhkcqB6WCvpVxftrg6+FK5x1ZuaIDaQ5Y= -k8s.io/component-base v0.25.0/go.mod h1:F2Sumv9CnbBlqrpdf7rKZTmmd2meJq0HizeyY/yAFxk= +k8s.io/component-base v0.25.13 h1:Lxpq2nu7Q4SueubBlU8kEcTgdaNXsktYg9bLsqvAV+4= +k8s.io/component-base v0.25.13/go.mod h1:Fh9SqG+Uyumtc+7HNT2FBZCxksIIFaukoL2LjqbBtgY= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= diff --git a/cli/vendor/github.com/google/uuid/null.go b/cli/vendor/github.com/google/uuid/null.go new file mode 100644 index 000000000..d7fcbf286 --- /dev/null +++ b/cli/vendor/github.com/google/uuid/null.go @@ -0,0 +1,118 @@ +// Copyright 2021 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "database/sql/driver" + "encoding/json" + "fmt" +) + +var jsonNull = []byte("null") + +// NullUUID represents a UUID that may be null. +// NullUUID implements the SQL driver.Scanner interface so +// it can be used as a scan destination: +// +// var u uuid.NullUUID +// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&u) +// ... +// if u.Valid { +// // use u.UUID +// } else { +// // NULL value +// } +// +type NullUUID struct { + UUID UUID + Valid bool // Valid is true if UUID is not NULL +} + +// Scan implements the SQL driver.Scanner interface. +func (nu *NullUUID) Scan(value interface{}) error { + if value == nil { + nu.UUID, nu.Valid = Nil, false + return nil + } + + err := nu.UUID.Scan(value) + if err != nil { + nu.Valid = false + return err + } + + nu.Valid = true + return nil +} + +// Value implements the driver Valuer interface. +func (nu NullUUID) Value() (driver.Value, error) { + if !nu.Valid { + return nil, nil + } + // Delegate to UUID Value function + return nu.UUID.Value() +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (nu NullUUID) MarshalBinary() ([]byte, error) { + if nu.Valid { + return nu.UUID[:], nil + } + + return []byte(nil), nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (nu *NullUUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(nu.UUID[:], data) + nu.Valid = true + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (nu NullUUID) MarshalText() ([]byte, error) { + if nu.Valid { + return nu.UUID.MarshalText() + } + + return jsonNull, nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (nu *NullUUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err != nil { + nu.Valid = false + return err + } + nu.UUID = id + nu.Valid = true + return nil +} + +// MarshalJSON implements json.Marshaler. +func (nu NullUUID) MarshalJSON() ([]byte, error) { + if nu.Valid { + return json.Marshal(nu.UUID) + } + + return jsonNull, nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (nu *NullUUID) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, jsonNull) { + *nu = NullUUID{} + return nil // valid null UUID + } + err := json.Unmarshal(data, &nu.UUID) + nu.Valid = err == nil + return err +} diff --git a/cli/vendor/github.com/google/uuid/uuid.go b/cli/vendor/github.com/google/uuid/uuid.go index 60d26bb50..a57207aeb 100644 --- a/cli/vendor/github.com/google/uuid/uuid.go +++ b/cli/vendor/github.com/google/uuid/uuid.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "strings" + "sync" ) // A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC @@ -33,7 +34,15 @@ const ( Future // Reserved for future definition. ) -var rander = rand.Reader // random function +const randPoolSize = 16 * 16 + +var ( + rander = rand.Reader // random function + poolEnabled = false + poolMu sync.Mutex + poolPos = randPoolSize // protected with poolMu + pool [randPoolSize]byte // protected with poolMu +) type invalidLengthError struct{ len int } @@ -41,6 +50,12 @@ func (err invalidLengthError) Error() string { return fmt.Sprintf("invalid UUID length: %d", err.len) } +// IsInvalidLengthError is matcher function for custom error invalidLengthError +func IsInvalidLengthError(err error) bool { + _, ok := err.(invalidLengthError) + return ok +} + // Parse decodes s into a UUID or returns an error. Both the standard UUID // forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the @@ -249,3 +264,31 @@ func SetRand(r io.Reader) { } rander = r } + +// EnableRandPool enables internal randomness pool used for Random +// (Version 4) UUID generation. The pool contains random bytes read from +// the random number generator on demand in batches. Enabling the pool +// may improve the UUID generation throughput significantly. +// +// Since the pool is stored on the Go heap, this feature may be a bad fit +// for security sensitive applications. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func EnableRandPool() { + poolEnabled = true +} + +// DisableRandPool disables the randomness pool if it was previously +// enabled with EnableRandPool. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func DisableRandPool() { + poolEnabled = false + defer poolMu.Unlock() + poolMu.Lock() + poolPos = randPoolSize +} diff --git a/cli/vendor/github.com/google/uuid/version4.go b/cli/vendor/github.com/google/uuid/version4.go index 86160fbd0..7697802e4 100644 --- a/cli/vendor/github.com/google/uuid/version4.go +++ b/cli/vendor/github.com/google/uuid/version4.go @@ -27,6 +27,8 @@ func NewString() string { // The strength of the UUIDs is based on the strength of the crypto/rand // package. // +// Uses the randomness pool if it was enabled with EnableRandPool. +// // A note about uniqueness derived from the UUID Wikipedia entry: // // Randomly generated UUIDs have 122 random bits. One's annual risk of being @@ -35,7 +37,10 @@ func NewString() string { // equivalent to the odds of creating a few tens of trillions of UUIDs in a // year and having one duplicate. func NewRandom() (UUID, error) { - return NewRandomFromReader(rander) + if !poolEnabled { + return NewRandomFromReader(rander) + } + return newRandomFromPool() } // NewRandomFromReader returns a UUID based on bytes read from a given io.Reader. @@ -49,3 +54,23 @@ func NewRandomFromReader(r io.Reader) (UUID, error) { uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 return uuid, nil } + +func newRandomFromPool() (UUID, error) { + var uuid UUID + poolMu.Lock() + if poolPos == randPoolSize { + _, err := io.ReadFull(rander, pool[:]) + if err != nil { + poolMu.Unlock() + return Nil, err + } + poolPos = 0 + } + copy(uuid[:], pool[poolPos:(poolPos+16)]) + poolPos += 16 + poolMu.Unlock() + + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/cli/vendor/github.com/stretchr/testify/assert/assertion_format.go b/cli/vendor/github.com/stretchr/testify/assert/assertion_format.go index 27e2420ed..7880b8f94 100644 --- a/cli/vendor/github.com/stretchr/testify/assert/assertion_format.go +++ b/cli/vendor/github.com/stretchr/testify/assert/assertion_format.go @@ -736,6 +736,16 @@ func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta tim return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...) } +// WithinRangef asserts that a time is within a time range (inclusive). +// +// assert.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return WithinRange(t, actual, start, end, append([]interface{}{msg}, args...)...) +} + // YAMLEqf asserts that two YAML strings are equivalent. func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { diff --git a/cli/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/cli/vendor/github.com/stretchr/testify/assert/assertion_forward.go index d9ea368d0..339515b8b 100644 --- a/cli/vendor/github.com/stretchr/testify/assert/assertion_forward.go +++ b/cli/vendor/github.com/stretchr/testify/assert/assertion_forward.go @@ -1461,6 +1461,26 @@ func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta return WithinDurationf(a.t, expected, actual, delta, msg, args...) } +// WithinRange asserts that a time is within a time range (inclusive). +// +// a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinRange(a.t, actual, start, end, msgAndArgs...) +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return WithinRangef(a.t, actual, start, end, msg, args...) +} + // YAMLEq asserts that two YAML strings are equivalent. func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { diff --git a/cli/vendor/github.com/stretchr/testify/assert/assertions.go b/cli/vendor/github.com/stretchr/testify/assert/assertions.go index 580fdea4c..fa1245b18 100644 --- a/cli/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/cli/vendor/github.com/stretchr/testify/assert/assertions.go @@ -8,6 +8,7 @@ import ( "fmt" "math" "os" + "path/filepath" "reflect" "regexp" "runtime" @@ -144,7 +145,8 @@ func CallerInfo() []string { if len(parts) > 1 { dir := parts[len(parts)-2] if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" { - callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + path, _ := filepath.Abs(file) + callers = append(callers, fmt.Sprintf("%s:%d", path, line)) } } @@ -816,7 +818,6 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok return true // we consider nil to be equal to the nil set } - subsetValue := reflect.ValueOf(subset) defer func() { if e := recover(); e != nil { ok = false @@ -826,14 +827,32 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok listKind := reflect.TypeOf(list).Kind() subsetKind := reflect.TypeOf(subset).Kind() - if listKind != reflect.Array && listKind != reflect.Slice { + if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) } - if subsetKind != reflect.Array && subsetKind != reflect.Slice { + if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) } + subsetValue := reflect.ValueOf(subset) + if subsetKind == reflect.Map && listKind == reflect.Map { + listValue := reflect.ValueOf(list) + subsetKeys := subsetValue.MapKeys() + + for i := 0; i < len(subsetKeys); i++ { + subsetKey := subsetKeys[i] + subsetElement := subsetValue.MapIndex(subsetKey).Interface() + listElement := listValue.MapIndex(subsetKey).Interface() + + if !ObjectsAreEqual(subsetElement, listElement) { + return Fail(t, fmt.Sprintf("\"%s\" does not contain \"%s\"", list, subsetElement), msgAndArgs...) + } + } + + return true + } + for i := 0; i < subsetValue.Len(); i++ { element := subsetValue.Index(i).Interface() ok, found := containsElement(list, element) @@ -860,7 +879,6 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) return Fail(t, "nil is the empty set which is a subset of every set", msgAndArgs...) } - subsetValue := reflect.ValueOf(subset) defer func() { if e := recover(); e != nil { ok = false @@ -870,14 +888,32 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) listKind := reflect.TypeOf(list).Kind() subsetKind := reflect.TypeOf(subset).Kind() - if listKind != reflect.Array && listKind != reflect.Slice { + if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...) } - if subsetKind != reflect.Array && subsetKind != reflect.Slice { + if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) } + subsetValue := reflect.ValueOf(subset) + if subsetKind == reflect.Map && listKind == reflect.Map { + listValue := reflect.ValueOf(list) + subsetKeys := subsetValue.MapKeys() + + for i := 0; i < len(subsetKeys); i++ { + subsetKey := subsetKeys[i] + subsetElement := subsetValue.MapIndex(subsetKey).Interface() + listElement := listValue.MapIndex(subsetKey).Interface() + + if !ObjectsAreEqual(subsetElement, listElement) { + return true + } + } + + return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) + } + for i := 0; i < subsetValue.Len(); i++ { element := subsetValue.Index(i).Interface() ok, found := containsElement(list, element) @@ -1110,6 +1146,27 @@ func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, return true } +// WithinRange asserts that a time is within a time range (inclusive). +// +// assert.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func WithinRange(t TestingT, actual, start, end time.Time, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if end.Before(start) { + return Fail(t, "Start should be before end", msgAndArgs...) + } + + if actual.Before(start) { + return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is before the range", actual, start, end), msgAndArgs...) + } else if actual.After(end) { + return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is after the range", actual, start, end), msgAndArgs...) + } + + return true +} + func toFloat(x interface{}) (float64, bool) { var xf float64 xok := true diff --git a/cli/vendor/github.com/stretchr/testify/require/require.go b/cli/vendor/github.com/stretchr/testify/require/require.go index 59c48277a..880853f5a 100644 --- a/cli/vendor/github.com/stretchr/testify/require/require.go +++ b/cli/vendor/github.com/stretchr/testify/require/require.go @@ -1864,6 +1864,32 @@ func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta tim t.FailNow() } +// WithinRange asserts that a time is within a time range (inclusive). +// +// assert.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func WithinRange(t TestingT, actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinRange(t, actual, start, end, msgAndArgs...) { + return + } + t.FailNow() +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// assert.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.WithinRangef(t, actual, start, end, msg, args...) { + return + } + t.FailNow() +} + // YAMLEq asserts that two YAML strings are equivalent. func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { diff --git a/cli/vendor/github.com/stretchr/testify/require/require_forward.go b/cli/vendor/github.com/stretchr/testify/require/require_forward.go index 5bb07c89c..960bf6f2c 100644 --- a/cli/vendor/github.com/stretchr/testify/require/require_forward.go +++ b/cli/vendor/github.com/stretchr/testify/require/require_forward.go @@ -1462,6 +1462,26 @@ func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta WithinDurationf(a.t, expected, actual, delta, msg, args...) } +// WithinRange asserts that a time is within a time range (inclusive). +// +// a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second)) +func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinRange(a.t, actual, start, end, msgAndArgs...) +} + +// WithinRangef asserts that a time is within a time range (inclusive). +// +// a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted") +func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + WithinRangef(a.t, actual, start, end, msg, args...) +} + // YAMLEq asserts that two YAML strings are equivalent. func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { diff --git a/cli/vendor/k8s.io/api/core/v1/generated.proto b/cli/vendor/k8s.io/api/core/v1/generated.proto index 256f5da32..e4e562e32 100644 --- a/cli/vendor/k8s.io/api/core/v1/generated.proto +++ b/cli/vendor/k8s.io/api/core/v1/generated.proto @@ -1761,7 +1761,8 @@ message HTTPGetAction { // HTTPHeader describes a custom header to be used in HTTP probes message HTTPHeader { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. optional string name = 1; // The header field value diff --git a/cli/vendor/k8s.io/api/core/v1/types.go b/cli/vendor/k8s.io/api/core/v1/types.go index 754a23613..e60e180c2 100644 --- a/cli/vendor/k8s.io/api/core/v1/types.go +++ b/cli/vendor/k8s.io/api/core/v1/types.go @@ -2112,7 +2112,8 @@ type SecretEnvSource struct { // HTTPHeader describes a custom header to be used in HTTP probes type HTTPHeader struct { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. Name string `json:"name" protobuf:"bytes,1,opt,name=name"` // The header field value Value string `json:"value" protobuf:"bytes,2,opt,name=value"` @@ -4266,6 +4267,9 @@ const ( // LoadBalancerPortsError represents the condition of the requested ports // on the cloud load balancer instance. LoadBalancerPortsError = "LoadBalancerPortsError" + // LoadBalancerPortsErrorReason reason in ServiceStatus condition LoadBalancerPortsError + // means the LoadBalancer was not able to be configured correctly. + LoadBalancerPortsErrorReason = "LoadBalancerMixedProtocolNotSupported" ) // ServiceStatus represents the current status of a service. @@ -6624,6 +6628,13 @@ const ( PortForwardRequestIDHeader = "requestID" ) +const ( + // MixedProtocolNotSupported error in PortStatus means that the cloud provider + // can't publish the port on the load balancer because mixed values of protocols + // on the same LoadBalancer type of Service are not supported by the cloud provider. + MixedProtocolNotSupported = "MixedProtocolNotSupported" +) + // PortStatus represents the error condition of a service port type PortStatus struct { diff --git a/cli/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go b/cli/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go index 6bae4bb76..77707706a 100644 --- a/cli/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/cli/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -808,7 +808,7 @@ func (HTTPGetAction) SwaggerDoc() map[string]string { var map_HTTPHeader = map[string]string{ "": "HTTPHeader describes a custom header to be used in HTTP probes", - "name": "The header field name", + "name": "The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.", "value": "The header field value", } diff --git a/cli/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_proto.go b/cli/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_proto.go index 6dd6d8999..ab68181e9 100644 --- a/cli/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_proto.go +++ b/cli/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/micro_time_proto.go @@ -27,9 +27,12 @@ func (m *MicroTime) ProtoMicroTime() *Timestamp { if m == nil { return &Timestamp{} } + + // truncate precision to microseconds to match JSON marshaling/unmarshaling + truncatedNanoseconds := time.Duration(m.Time.Nanosecond()).Truncate(time.Microsecond) return &Timestamp{ Seconds: m.Time.Unix(), - Nanos: int32(m.Time.Nanosecond()), + Nanos: int32(truncatedNanoseconds), } } @@ -51,7 +54,10 @@ func (m *MicroTime) Unmarshal(data []byte) error { if err := p.Unmarshal(data); err != nil { return err } - m.Time = time.Unix(p.Seconds, int64(p.Nanos)).Local() + + // truncate precision to microseconds to match JSON marshaling/unmarshaling + truncatedNanoseconds := time.Duration(p.Nanos).Truncate(time.Microsecond) + m.Time = time.Unix(p.Seconds, int64(truncatedNanoseconds)).Local() return nil } diff --git a/cli/vendor/k8s.io/apimachinery/pkg/runtime/converter.go b/cli/vendor/k8s.io/apimachinery/pkg/runtime/converter.go index 90bf487e3..62eb27afc 100644 --- a/cli/vendor/k8s.io/apimachinery/pkg/runtime/converter.go +++ b/cli/vendor/k8s.io/apimachinery/pkg/runtime/converter.go @@ -231,7 +231,7 @@ func (c *fromUnstructuredContext) pushKey(key string) { } -// FromUnstructuredWIthValidation converts an object from map[string]interface{} representation into a concrete type. +// FromUnstructuredWithValidation converts an object from map[string]interface{} representation into a concrete type. // It uses encoding/json/Unmarshaler if object implements it or reflection if not. // It takes a validationDirective that indicates how to behave when it encounters unknown fields. func (c *unstructuredConverter) FromUnstructuredWithValidation(u map[string]interface{}, obj interface{}, returnUnknownFields bool) error { @@ -465,7 +465,7 @@ func sliceFromUnstructured(sv, dv reflect.Value, ctx *fromUnstructuredContext) e } dv.SetBytes(data) } else { - dv.Set(reflect.Zero(dt)) + dv.Set(reflect.MakeSlice(dt, 0, 0)) } return nil } diff --git a/cli/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go b/cli/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go index d37dfbf73..73876f688 100644 --- a/cli/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go +++ b/cli/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go @@ -199,14 +199,18 @@ func newAuthenticator(c *cache, isTerminalFunc func(int) bool, config *api.ExecC now: time.Now, environ: os.Environ, - defaultDialer: defaultDialer, - connTracker: connTracker, + connTracker: connTracker, } for _, env := range config.Env { a.env = append(a.env, env.Name+"="+env.Value) } + // these functions are made comparable and stored in the cache so that repeated clientset + // construction with the same rest.Config results in a single TLS cache and Authenticator + a.getCert = &transport.GetCertHolder{GetCert: a.cert} + a.dial = &transport.DialHolder{Dial: defaultDialer.DialContext} + return c.put(key, a), nil } @@ -261,8 +265,6 @@ type Authenticator struct { now func() time.Time environ func() []string - // defaultDialer is used for clients which don't specify a custom dialer - defaultDialer *connrotation.Dialer // connTracker tracks all connections opened that we need to close when rotating a client certificate connTracker *connrotation.ConnectionTracker @@ -273,6 +275,12 @@ type Authenticator struct { mu sync.Mutex cachedCreds *credentials exp time.Time + + // getCert makes Authenticator.cert comparable to support TLS config caching + getCert *transport.GetCertHolder + // dial is used for clients which do not specify a custom dialer + // it is comparable to support TLS config caching + dial *transport.DialHolder } type credentials struct { @@ -300,18 +308,20 @@ func (a *Authenticator) UpdateTransportConfig(c *transport.Config) error { if c.HasCertCallback() { return errors.New("can't add TLS certificate callback: transport.Config.TLS.GetCert already set") } - c.TLS.GetCert = a.cert + c.TLS.GetCert = a.getCert.GetCert + c.TLS.GetCertHolder = a.getCert // comparable for TLS config caching - var d *connrotation.Dialer if c.Dial != nil { // if c has a custom dialer, we have to wrap it - d = connrotation.NewDialerWithTracker(c.Dial, a.connTracker) + // TLS config caching is not supported for this config + d := connrotation.NewDialerWithTracker(c.Dial, a.connTracker) + c.Dial = d.DialContext + c.DialHolder = nil } else { - d = a.defaultDialer + c.Dial = a.dial.Dial + c.DialHolder = a.dial // comparable for TLS config caching } - c.Dial = d.DialContext - return nil } diff --git a/cli/vendor/k8s.io/client-go/rest/request.go b/cli/vendor/k8s.io/client-go/rest/request.go index dba933f7d..a1f0a591d 100644 --- a/cli/vendor/k8s.io/client-go/rest/request.go +++ b/cli/vendor/k8s.io/client-go/rest/request.go @@ -34,6 +34,7 @@ import ( "time" "golang.org/x/net/http2" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -116,8 +117,11 @@ type Request struct { subresource string // output - err error - body io.Reader + err error + + // only one of body / bodyBytes may be set. requests using body are not retriable. + body io.Reader + bodyBytes []byte retryFn requestRetryFunc } @@ -443,12 +447,15 @@ func (r *Request) Body(obj interface{}) *Request { return r } glogBody("Request Body", data) - r.body = bytes.NewReader(data) + r.body = nil + r.bodyBytes = data case []byte: glogBody("Request Body", t) - r.body = bytes.NewReader(t) + r.body = nil + r.bodyBytes = t case io.Reader: r.body = t + r.bodyBytes = nil case runtime.Object: // callers may pass typed interface pointers, therefore we must check nil with reflection if reflect.ValueOf(t).IsNil() { @@ -465,7 +472,8 @@ func (r *Request) Body(obj interface{}) *Request { return r } glogBody("Request Body", data) - r.body = bytes.NewReader(data) + r.body = nil + r.bodyBytes = data r.SetHeader("Content-Type", r.c.content.ContentType) default: r.err = fmt.Errorf("unknown type used for body: %+v", obj) @@ -508,6 +516,87 @@ func (r *Request) URL() *url.URL { return finalURL } +// finalURLTemplate is similar to URL(), but will make all specific parameter values equal +// - instead of name or namespace, "{name}" and "{namespace}" will be used, and all query +// parameters will be reset. This creates a copy of the url so as not to change the +// underlying object. +func (r Request) finalURLTemplate() url.URL { + newParams := url.Values{} + v := []string{"{value}"} + for k := range r.params { + newParams[k] = v + } + r.params = newParams + u := r.URL() + if u == nil { + return url.URL{} + } + + segments := strings.Split(u.Path, "/") + groupIndex := 0 + index := 0 + trimmedBasePath := "" + if r.c.base != nil && strings.Contains(u.Path, r.c.base.Path) { + p := strings.TrimPrefix(u.Path, r.c.base.Path) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + // store the base path that we have trimmed so we can append it + // before returning the URL + trimmedBasePath = r.c.base.Path + segments = strings.Split(p, "/") + groupIndex = 1 + } + if len(segments) <= 2 { + return *u + } + + const CoreGroupPrefix = "api" + const NamedGroupPrefix = "apis" + isCoreGroup := segments[groupIndex] == CoreGroupPrefix + isNamedGroup := segments[groupIndex] == NamedGroupPrefix + if isCoreGroup { + // checking the case of core group with /api/v1/... format + index = groupIndex + 2 + } else if isNamedGroup { + // checking the case of named group with /apis/apps/v1/... format + index = groupIndex + 3 + } else { + // this should not happen that the only two possibilities are /api... and /apis..., just want to put an + // outlet here in case more API groups are added in future if ever possible: + // https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-groups + // if a wrong API groups name is encountered, return the {prefix} for url.Path + u.Path = "/{prefix}" + u.RawQuery = "" + return *u + } + // switch segLength := len(segments) - index; segLength { + switch { + // case len(segments) - index == 1: + // resource (with no name) do nothing + case len(segments)-index == 2: + // /$RESOURCE/$NAME: replace $NAME with {name} + segments[index+1] = "{name}" + case len(segments)-index == 3: + if segments[index+2] == "finalize" || segments[index+2] == "status" { + // /$RESOURCE/$NAME/$SUBRESOURCE: replace $NAME with {name} + segments[index+1] = "{name}" + } else { + // /namespace/$NAMESPACE/$RESOURCE: replace $NAMESPACE with {namespace} + segments[index+1] = "{namespace}" + } + case len(segments)-index >= 4: + segments[index+1] = "{namespace}" + // /namespace/$NAMESPACE/$RESOURCE/$NAME: replace $NAMESPACE with {namespace}, $NAME with {name} + if segments[index+3] != "finalize" && segments[index+3] != "status" { + // /$RESOURCE/$NAME/$SUBRESOURCE: replace $NAME with {name} + segments[index+3] = "{name}" + } + } + u.Path = path.Join(trimmedBasePath, path.Join(segments...)) + return *u +} + func (r *Request) tryThrottleWithInfo(ctx context.Context, retryInfo string) error { if r.rateLimiter == nil { return nil @@ -537,7 +626,7 @@ func (r *Request) tryThrottleWithInfo(ctx context.Context, retryInfo string) err // but we use a throttled logger to prevent spamming. globalThrottledLogger.Infof("%s", message) } - metrics.RateLimiterLatency.Observe(ctx, r.verb, *r.URL(), latency) + metrics.RateLimiterLatency.Observe(ctx, r.verb, r.finalURLTemplate(), latency) return err } @@ -744,9 +833,7 @@ func (r *Request) Stream(ctx context.Context) (io.ReadCloser, error) { if err != nil { return nil, err } - if r.body != nil { - req.Body = ioutil.NopCloser(r.body) - } + resp, err := client.Do(req) updateURLMetrics(ctx, r, resp, err) retry.After(ctx, r, resp, err) @@ -808,8 +895,20 @@ func (r *Request) requestPreflightCheck() error { } func (r *Request) newHTTPRequest(ctx context.Context) (*http.Request, error) { + var body io.Reader + switch { + case r.body != nil && r.bodyBytes != nil: + return nil, fmt.Errorf("cannot set both body and bodyBytes") + case r.body != nil: + body = r.body + case r.bodyBytes != nil: + // Create a new reader specifically for this request. + // Giving each request a dedicated reader allows retries to avoid races resetting the request body. + body = bytes.NewReader(r.bodyBytes) + } + url := r.URL().String() - req, err := http.NewRequest(r.verb, url, r.body) + req, err := http.NewRequest(r.verb, url, body) if err != nil { return nil, err } @@ -826,7 +925,7 @@ func (r *Request) request(ctx context.Context, fn func(*http.Request, *http.Resp // Metrics for total request latency start := time.Now() defer func() { - metrics.RequestLatency.Observe(ctx, r.verb, *r.URL(), time.Since(start)) + metrics.RequestLatency.Observe(ctx, r.verb, r.finalURLTemplate(), time.Since(start)) }() if r.err != nil { diff --git a/cli/vendor/k8s.io/client-go/rest/with_retry.go b/cli/vendor/k8s.io/client-go/rest/with_retry.go index bdcc6f3a9..ab2694754 100644 --- a/cli/vendor/k8s.io/client-go/rest/with_retry.go +++ b/cli/vendor/k8s.io/client-go/rest/with_retry.go @@ -154,6 +154,11 @@ func (r *withRetry) IsNextRetry(ctx context.Context, restReq *Request, httpReq * return false } + if restReq.body != nil { + // we have an opaque reader, we can't safely reset it + return false + } + r.attempts++ r.retryAfter = &RetryAfter{Attempt: r.attempts} if r.attempts > r.maxRetries { @@ -210,18 +215,6 @@ func (r *withRetry) Before(ctx context.Context, request *Request) error { return nil } - // At this point we've made atleast one attempt, post which the response - // body should have been fully read and closed in order for it to be safe - // to reset the request body before we reconnect, in order for us to reuse - // the same TCP connection. - if seeker, ok := request.body.(io.Seeker); ok && request.body != nil { - if _, err := seeker.Seek(0, io.SeekStart); err != nil { - err = fmt.Errorf("failed to reset the request body while retrying a request: %v", err) - r.trackPreviousError(err) - return err - } - } - // if we are here, we have made attempt(s) at least once before. if request.backoff != nil { delay := request.backoff.CalculateBackoff(url) diff --git a/cli/vendor/k8s.io/client-go/tools/cache/controller.go b/cli/vendor/k8s.io/client-go/tools/cache/controller.go index 0762da3be..96005ff58 100644 --- a/cli/vendor/k8s.io/client-go/tools/cache/controller.go +++ b/cli/vendor/k8s.io/client-go/tools/cache/controller.go @@ -353,17 +353,6 @@ func NewIndexerInformer( return clientState, newInformer(lw, objType, resyncPeriod, h, clientState, nil) } -// TransformFunc allows for transforming an object before it will be processed -// and put into the controller cache and before the corresponding handlers will -// be called on it. -// TransformFunc (similarly to ResourceEventHandler functions) should be able -// to correctly handle the tombstone of type cache.DeletedFinalStateUnknown -// -// The most common usage pattern is to clean-up some parts of the object to -// reduce component memory usage if a given component doesn't care about them. -// given controller doesn't care for them -type TransformFunc func(interface{}) (interface{}, error) - // NewTransformingInformer returns a Store and a controller for populating // the store while also providing event notifications. You should only used // the returned Store for Get/List operations; Add/Modify/Deletes will cause @@ -411,19 +400,11 @@ func processDeltas( // Object which receives event notifications from the given deltas handler ResourceEventHandler, clientState Store, - transformer TransformFunc, deltas Deltas, ) error { // from oldest to newest for _, d := range deltas { obj := d.Object - if transformer != nil { - var err error - obj, err = transformer(obj) - if err != nil { - return err - } - } switch d.Type { case Sync, Replaced, Added, Updated: @@ -475,6 +456,7 @@ func newInformer( fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{ KnownObjects: clientState, EmitDeltaTypeReplaced: true, + Transformer: transformer, }) cfg := &Config{ @@ -486,7 +468,7 @@ func newInformer( Process: func(obj interface{}) error { if deltas, ok := obj.(Deltas); ok { - return processDeltas(h, clientState, transformer, deltas) + return processDeltas(h, clientState, deltas) } return errors.New("object given as Process argument is not Deltas") }, diff --git a/cli/vendor/k8s.io/client-go/tools/cache/delta_fifo.go b/cli/vendor/k8s.io/client-go/tools/cache/delta_fifo.go index 0c13a41f0..84f3ab9ca 100644 --- a/cli/vendor/k8s.io/client-go/tools/cache/delta_fifo.go +++ b/cli/vendor/k8s.io/client-go/tools/cache/delta_fifo.go @@ -51,6 +51,10 @@ type DeltaFIFOOptions struct { // When true, `Replaced` events will be sent for items passed to a Replace() call. // When false, `Sync` events will be sent instead. EmitDeltaTypeReplaced bool + + // If set, will be called for objects before enqueueing them. Please + // see the comment on TransformFunc for details. + Transformer TransformFunc } // DeltaFIFO is like FIFO, but differs in two ways. One is that the @@ -129,8 +133,32 @@ type DeltaFIFO struct { // emitDeltaTypeReplaced is whether to emit the Replaced or Sync // DeltaType when Replace() is called (to preserve backwards compat). emitDeltaTypeReplaced bool + + // Called with every object if non-nil. + transformer TransformFunc } +// TransformFunc allows for transforming an object before it will be processed. +// TransformFunc (similarly to ResourceEventHandler functions) should be able +// to correctly handle the tombstone of type cache.DeletedFinalStateUnknown. +// +// New in v1.27: In such cases, the contained object will already have gone +// through the transform object separately (when it was added / updated prior +// to the delete), so the TransformFunc can likely safely ignore such objects +// (i.e., just return the input object). +// +// The most common usage pattern is to clean-up some parts of the object to +// reduce component memory usage if a given component doesn't care about them. +// +// New in v1.27: unless the object is a DeletedFinalStateUnknown, TransformFunc +// sees the object before any other actor, and it is now safe to mutate the +// object in place instead of making a copy. +// +// Note that TransformFunc is called while inserting objects into the +// notification queue and is therefore extremely performance sensitive; please +// do not do anything that will take a long time. +type TransformFunc func(interface{}) (interface{}, error) + // DeltaType is the type of a change (addition, deletion, etc) type DeltaType string @@ -227,6 +255,7 @@ func NewDeltaFIFOWithOptions(opts DeltaFIFOOptions) *DeltaFIFO { knownObjects: opts.KnownObjects, emitDeltaTypeReplaced: opts.EmitDeltaTypeReplaced, + transformer: opts.Transformer, } f.cond.L = &f.lock return f @@ -411,6 +440,21 @@ func (f *DeltaFIFO) queueActionLocked(actionType DeltaType, obj interface{}) err if err != nil { return KeyError{obj, err} } + + // Every object comes through this code path once, so this is a good + // place to call the transform func. If obj is a + // DeletedFinalStateUnknown tombstone, then the containted inner object + // will already have gone through the transformer, but we document that + // this can happen. In cases involving Replace(), such an object can + // come through multiple times. + if f.transformer != nil { + var err error + obj, err = f.transformer(obj) + if err != nil { + return err + } + } + oldDeltas := f.items[id] newDeltas := append(oldDeltas, Delta{actionType, obj}) newDeltas = dedupDeltas(newDeltas) @@ -566,12 +610,11 @@ func (f *DeltaFIFO) Pop(process PopProcessFunc) (interface{}, error) { // using the Sync or Replace DeltaType and then (2) it does some deletions. // In particular: for every pre-existing key K that is not the key of // an object in `list` there is the effect of -// `Delete(DeletedFinalStateUnknown{K, O})` where O is current object -// of K. If `f.knownObjects == nil` then the pre-existing keys are -// those in `f.items` and the current object of K is the `.Newest()` -// of the Deltas associated with K. Otherwise the pre-existing keys -// are those listed by `f.knownObjects` and the current object of K is -// what `f.knownObjects.GetByKey(K)` returns. +// `Delete(DeletedFinalStateUnknown{K, O})` where O is the latest known +// object of K. The pre-existing keys are those in the union set of the keys in +// `f.items` and `f.knownObjects` (if not nil). The last known object for key K is +// the one present in the last delta in `f.items`. If there is no delta for K +// in `f.items`, it is the object in `f.knownObjects` func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { f.lock.Lock() defer f.lock.Unlock() @@ -595,51 +638,23 @@ func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { } } - if f.knownObjects == nil { - // Do deletion detection against our own list. - queuedDeletions := 0 - for k, oldItem := range f.items { - if keys.Has(k) { - continue - } - // Delete pre-existing items not in the new list. - // This could happen if watch deletion event was missed while - // disconnected from apiserver. - var deletedObj interface{} - if n := oldItem.Newest(); n != nil { - deletedObj = n.Object - } - queuedDeletions++ - if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { - return err - } - } - - if !f.populated { - f.populated = true - // While there shouldn't be any queued deletions in the initial - // population of the queue, it's better to be on the safe side. - f.initialPopulationCount = keys.Len() + queuedDeletions - } - - return nil - } - - // Detect deletions not already in the queue. - knownKeys := f.knownObjects.ListKeys() + // Do deletion detection against objects in the queue queuedDeletions := 0 - for _, k := range knownKeys { + for k, oldItem := range f.items { if keys.Has(k) { continue } - - deletedObj, exists, err := f.knownObjects.GetByKey(k) - if err != nil { - deletedObj = nil - klog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) - } else if !exists { - deletedObj = nil - klog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) + // Delete pre-existing items not in the new list. + // This could happen if watch deletion event was missed while + // disconnected from apiserver. + var deletedObj interface{} + if n := oldItem.Newest(); n != nil { + deletedObj = n.Object + + // if the previous object is a DeletedFinalStateUnknown, we have to extract the actual Object + if d, ok := deletedObj.(DeletedFinalStateUnknown); ok { + deletedObj = d.Obj + } } queuedDeletions++ if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { @@ -647,6 +662,32 @@ func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { } } + if f.knownObjects != nil { + // Detect deletions for objects not present in the queue, but present in KnownObjects + knownKeys := f.knownObjects.ListKeys() + for _, k := range knownKeys { + if keys.Has(k) { + continue + } + if len(f.items[k]) > 0 { + continue + } + + deletedObj, exists, err := f.knownObjects.GetByKey(k) + if err != nil { + deletedObj = nil + klog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) + } else if !exists { + deletedObj = nil + klog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) + } + queuedDeletions++ + if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { + return err + } + } + } + if !f.populated { f.populated = true f.initialPopulationCount = keys.Len() + queuedDeletions diff --git a/cli/vendor/k8s.io/client-go/tools/cache/shared_informer.go b/cli/vendor/k8s.io/client-go/tools/cache/shared_informer.go index 9f42782d1..35ebd396c 100644 --- a/cli/vendor/k8s.io/client-go/tools/cache/shared_informer.go +++ b/cli/vendor/k8s.io/client-go/tools/cache/shared_informer.go @@ -190,10 +190,7 @@ type SharedInformer interface { // // Must be set before starting the informer. // - // Note: Since the object given to the handler may be already shared with - // other goroutines, it is advisable to copy the object being - // transform before mutating it at all and returning the copy to prevent - // data races. + // Please see the comment on TransformFunc for more details. SetTransform(handler TransformFunc) error } @@ -404,6 +401,7 @@ func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) { fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{ KnownObjects: s.indexer, EmitDeltaTypeReplaced: true, + Transformer: s.transform, }) cfg := &Config{ @@ -568,7 +566,7 @@ func (s *sharedIndexInformer) HandleDeltas(obj interface{}) error { defer s.blockDeltas.Unlock() if deltas, ok := obj.(Deltas); ok { - return processDeltas(s, s.indexer, s.transform, deltas) + return processDeltas(s, s.indexer, deltas) } return errors.New("object given as Process argument is not Deltas") } diff --git a/cli/vendor/k8s.io/client-go/transport/cache.go b/cli/vendor/k8s.io/client-go/transport/cache.go index 214f0a79c..477c22cde 100644 --- a/cli/vendor/k8s.io/client-go/transport/cache.go +++ b/cli/vendor/k8s.io/client-go/transport/cache.go @@ -17,6 +17,7 @@ limitations under the License. package transport import ( + "context" "fmt" "net" "net/http" @@ -55,6 +56,9 @@ type tlsCacheKey struct { serverName string nextProtos string disableCompression bool + // these functions are wrapped to allow them to be used as map keys + getCert *GetCertHolder + dial *DialHolder } func (t tlsCacheKey) String() string { @@ -62,7 +66,8 @@ func (t tlsCacheKey) String() string { if len(t.keyData) > 0 { keyText = "" } - return fmt.Sprintf("insecure:%v, caData:%#v, certData:%#v, keyData:%s, serverName:%s, disableCompression:%t", t.insecure, t.caData, t.certData, keyText, t.serverName, t.disableCompression) + return fmt.Sprintf("insecure:%v, caData:%#v, certData:%#v, keyData:%s, serverName:%s, disableCompression:%t, getCert:%p, dial:%p", + t.insecure, t.caData, t.certData, keyText, t.serverName, t.disableCompression, t.getCert, t.dial) } func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { @@ -92,8 +97,10 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { return http.DefaultTransport, nil } - dial := config.Dial - if dial == nil { + var dial func(ctx context.Context, network, address string) (net.Conn, error) + if config.Dial != nil { + dial = config.Dial + } else { dial = (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -102,7 +109,7 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { // If we use are reloading files, we need to handle certificate rotation properly // TODO(jackkleeman): We can also add rotation here when config.HasCertCallback() is true - if config.TLS.ReloadTLSFiles { + if config.TLS.ReloadTLSFiles && tlsConfig != nil && tlsConfig.GetClientCertificate != nil { dynamicCertDialer := certRotatingDialer(tlsConfig.GetClientCertificate, dial) tlsConfig.GetClientCertificate = dynamicCertDialer.GetClientCertificate dial = dynamicCertDialer.connDialer.DialContext @@ -138,10 +145,18 @@ func tlsConfigKey(c *Config) (tlsCacheKey, bool, error) { return tlsCacheKey{}, false, err } - if c.TLS.GetCert != nil || c.Dial != nil || c.Proxy != nil { + if c.Proxy != nil { // cannot determine equality for functions return tlsCacheKey{}, false, nil } + if c.Dial != nil && c.DialHolder == nil { + // cannot determine equality for dial function that doesn't have non-nil DialHolder set as well + return tlsCacheKey{}, false, nil + } + if c.TLS.GetCert != nil && c.TLS.GetCertHolder == nil { + // cannot determine equality for getCert function that doesn't have non-nil GetCertHolder set as well + return tlsCacheKey{}, false, nil + } k := tlsCacheKey{ insecure: c.TLS.Insecure, @@ -149,6 +164,8 @@ func tlsConfigKey(c *Config) (tlsCacheKey, bool, error) { serverName: c.TLS.ServerName, nextProtos: strings.Join(c.TLS.NextProtos, ","), disableCompression: c.DisableCompression, + getCert: c.TLS.GetCertHolder, + dial: c.DialHolder, } if c.TLS.ReloadTLSFiles { diff --git a/cli/vendor/k8s.io/client-go/transport/config.go b/cli/vendor/k8s.io/client-go/transport/config.go index 89de798f6..fd853c0b3 100644 --- a/cli/vendor/k8s.io/client-go/transport/config.go +++ b/cli/vendor/k8s.io/client-go/transport/config.go @@ -68,7 +68,11 @@ type Config struct { WrapTransport WrapperFunc // Dial specifies the dial function for creating unencrypted TCP connections. + // If specified, this transport will be non-cacheable unless DialHolder is also set. Dial func(ctx context.Context, network, address string) (net.Conn, error) + // DialHolder can be populated to make transport configs cacheable. + // If specified, DialHolder.Dial must be equal to Dial. + DialHolder *DialHolder // Proxy is the proxy func to be used for all requests made by this // transport. If Proxy is nil, http.ProxyFromEnvironment is used. If Proxy @@ -78,6 +82,11 @@ type Config struct { Proxy func(*http.Request) (*url.URL, error) } +// DialHolder is used to make the wrapped function comparable so that it can be used as a map key. +type DialHolder struct { + Dial func(ctx context.Context, network, address string) (net.Conn, error) +} + // ImpersonationConfig has all the available impersonation options type ImpersonationConfig struct { // UserName matches user.Info.GetName() @@ -143,5 +152,15 @@ type TLSConfig struct { // To use only http/1.1, set to ["http/1.1"]. NextProtos []string - GetCert func() (*tls.Certificate, error) // Callback that returns a TLS client certificate. CertData, CertFile, KeyData and KeyFile supercede this field. + // Callback that returns a TLS client certificate. CertData, CertFile, KeyData and KeyFile supercede this field. + // If specified, this transport is non-cacheable unless CertHolder is populated. + GetCert func() (*tls.Certificate, error) + // CertHolder can be populated to make transport configs that set GetCert cacheable. + // If set, CertHolder.GetCert must be equal to GetCert. + GetCertHolder *GetCertHolder +} + +// GetCertHolder is used to make the wrapped function comparable so that it can be used as a map key. +type GetCertHolder struct { + GetCert func() (*tls.Certificate, error) } diff --git a/cli/vendor/k8s.io/client-go/transport/transport.go b/cli/vendor/k8s.io/client-go/transport/transport.go index b4a7bfa67..eabfce72d 100644 --- a/cli/vendor/k8s.io/client-go/transport/transport.go +++ b/cli/vendor/k8s.io/client-go/transport/transport.go @@ -24,6 +24,7 @@ import ( "fmt" "io/ioutil" "net/http" + "reflect" "sync" "time" @@ -39,6 +40,10 @@ func New(config *Config) (http.RoundTripper, error) { return nil, fmt.Errorf("using a custom transport with TLS certificate options or the insecure flag is not allowed") } + if !isValidHolders(config) { + return nil, fmt.Errorf("misconfigured holder for dialer or cert callback") + } + var ( rt http.RoundTripper err error @@ -56,6 +61,26 @@ func New(config *Config) (http.RoundTripper, error) { return HTTPWrappersForConfig(config, rt) } +func isValidHolders(config *Config) bool { + if config.TLS.GetCertHolder != nil { + if config.TLS.GetCertHolder.GetCert == nil || + config.TLS.GetCert == nil || + reflect.ValueOf(config.TLS.GetCertHolder.GetCert).Pointer() != reflect.ValueOf(config.TLS.GetCert).Pointer() { + return false + } + } + + if config.DialHolder != nil { + if config.DialHolder.Dial == nil || + config.Dial == nil || + reflect.ValueOf(config.DialHolder.Dial).Pointer() != reflect.ValueOf(config.Dial).Pointer() { + return false + } + } + + return true +} + // TLSConfigFor returns a tls.Config that will provide the transport level security defined // by the provided Config. Will return nil if no transport level security is requested. func TLSConfigFor(c *Config) (*tls.Config, error) { diff --git a/cli/vendor/k8s.io/client-go/util/cert/cert.go b/cli/vendor/k8s.io/client-go/util/cert/cert.go index 75143ec07..447c3013c 100644 --- a/cli/vendor/k8s.io/client-go/util/cert/cert.go +++ b/cli/vendor/k8s.io/client-go/util/cert/cert.go @@ -26,6 +26,7 @@ import ( "encoding/pem" "fmt" "io/ioutil" + "math" "math/big" "net" "path/filepath" @@ -44,6 +45,7 @@ type Config struct { Organization []string AltNames AltNames Usages []x509.ExtKeyUsage + NotBefore time.Time } // AltNames contains the domain names and IP addresses that will be added @@ -57,14 +59,24 @@ type AltNames struct { // NewSelfSignedCACert creates a CA certificate func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) { now := time.Now() + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) + notBefore := now.UTC() + if !cfg.NotBefore.IsZero() { + notBefore = cfg.NotBefore.UTC() + } tmpl := x509.Certificate{ - SerialNumber: new(big.Int).SetInt64(0), + SerialNumber: serial, Subject: pkix.Name{ CommonName: cfg.CommonName, Organization: cfg.Organization, }, DNSNames: []string{cfg.CommonName}, - NotBefore: now.UTC(), + NotBefore: notBefore, NotAfter: now.Add(duration365d * 10).UTC(), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, @@ -116,9 +128,14 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a if err != nil { return nil, nil, err } - + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) caTemplate := x509.Certificate{ - SerialNumber: big.NewInt(1), + SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()), }, @@ -144,9 +161,14 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a if err != nil { return nil, nil, err } - + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err = cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) template := x509.Certificate{ - SerialNumber: big.NewInt(2), + SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()), }, diff --git a/cli/vendor/modules.txt b/cli/vendor/modules.txt index caeae14ae..9aef3aaaa 100644 --- a/cli/vendor/modules.txt +++ b/cli/vendor/modules.txt @@ -122,7 +122,7 @@ github.com/google/go-containerregistry/pkg/name ## explicit; go 1.12 github.com/google/gofuzz github.com/google/gofuzz/bytesource -# github.com/google/uuid v1.2.0 +# github.com/google/uuid v1.3.0 ## explicit github.com/google/uuid # github.com/hashicorp/go-version v1.3.0 @@ -206,7 +206,7 @@ github.com/spf13/cobra # github.com/spf13/pflag v1.0.5 ## explicit; go 1.12 github.com/spf13/pflag -# github.com/stretchr/testify v1.7.5 +# github.com/stretchr/testify v1.8.0 ## explicit; go 1.13 github.com/stretchr/testify/assert github.com/stretchr/testify/require @@ -383,7 +383,7 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# k8s.io/api v0.25.0 +# k8s.io/api v0.25.13 ## explicit; go 1.19 k8s.io/api/admission/v1 k8s.io/api/admission/v1beta1 @@ -433,12 +433,12 @@ k8s.io/api/scheduling/v1beta1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 -# k8s.io/apiextensions-apiserver v0.25.0 +# k8s.io/apiextensions-apiserver v0.25.13 ## explicit; go 1.19 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 k8s.io/apiextensions-apiserver/pkg/apiserver/schema -# k8s.io/apimachinery v0.25.0 +# k8s.io/apimachinery v0.25.13 ## explicit; go 1.19 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -486,7 +486,7 @@ k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/client-go v0.25.0 +# k8s.io/client-go v0.25.13 ## explicit; go 1.19 k8s.io/client-go/applyconfigurations/admissionregistration/v1 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 @@ -622,7 +622,7 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/jsonpath k8s.io/client-go/util/keyutil k8s.io/client-go/util/workqueue -# k8s.io/component-base v0.25.0 +# k8s.io/component-base v0.25.13 ## explicit; go 1.19 k8s.io/component-base/config k8s.io/component-base/config/v1alpha1 diff --git a/go.mod b/go.mod index 690cba4e6..baeae8aef 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,11 @@ require ( golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.6.0 gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.25.5 - k8s.io/apimachinery v0.25.5 - k8s.io/apiserver v0.25.0 - k8s.io/client-go v0.25.5 - k8s.io/code-generator v0.25.0 + k8s.io/api v0.25.13 + k8s.io/apimachinery v0.25.13 + k8s.io/apiserver v0.25.13 + k8s.io/client-go v0.25.13 + k8s.io/code-generator v0.25.13 k8s.io/kube-aggregator v0.22.17 k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 sigs.k8s.io/controller-runtime v0.13.1 @@ -34,7 +34,7 @@ require ( github.com/spf13/cobra v1.4.0 golang.org/x/sync v0.1.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/component-base v0.25.5 + k8s.io/component-base v0.25.13 k8s.io/klog/v2 v2.70.1 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed ) @@ -65,7 +65,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/hashicorp/go-version v1.2.1 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -117,9 +117,9 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - k8s.io/apiextensions-apiserver v0.25.0 // indirect + k8s.io/apiextensions-apiserver v0.25.13 // indirect k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index fc78098ef..3033c59b6 100644 --- a/go.sum +++ b/go.sum @@ -227,8 +227,8 @@ github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -898,31 +898,31 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/api v0.22.17/go.mod h1:6qVojJ3y+qIq7JSMwTH0BcPHl3dch4HefIC+4nguZhs= -k8s.io/api v0.25.5 h1:mqyHf7aoaYMpdvO87mqpol+Qnsmo+y09S0PMIXwiZKo= -k8s.io/api v0.25.5/go.mod h1:RzplZX0Z8rV/WhSTfEvnyd91bBhBQTRWo85qBQwRmb8= +k8s.io/api v0.25.13 h1:nOQWK5/ngLIG2CqmVV7uTFDsPCGkDk4kIGJ26t2AwIo= +k8s.io/api v0.25.13/go.mod h1:yGpHyrivZ0enqWqT5s1pN98a4Q834rZkIUEABpleEtw= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= -k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= -k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= +k8s.io/apiextensions-apiserver v0.25.13 h1:tdB/ULSvlbZIecU2bDOoCb46Ozk5VyLL6IPysC8gUpE= +k8s.io/apiextensions-apiserver v0.25.13/go.mod h1:uqYfnY2GOYXlfpnT6No9dxrxDbMjaYvh/Bpm0iNMYnA= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.17/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU= -k8s.io/apimachinery v0.25.5 h1:SQomYHvv+aO43qdu3QKRf9YuI0oI8w3RrOQ1qPbAUGY= -k8s.io/apimachinery v0.25.5/go.mod h1:1S2i1QHkmxc8+EZCIxe/fX5hpldVXk4gvnJInMEb8D4= +k8s.io/apimachinery v0.25.13 h1:byRHkSinOOVdo0pvjdblauFYfwAnx+JB8Pqi9w9weik= +k8s.io/apimachinery v0.25.13/go.mod h1:IFwbcNi3gKkfDhuy0VYu3+BwbxbiIov3p6FR8ge1Epc= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/apiserver v0.22.17/go.mod h1:zNXYCtXZ91AkmIUZgQ8lT9vdlDqgSkokJpds/F6DdGU= -k8s.io/apiserver v0.25.0 h1:8kl2ifbNffD440MyvHtPaIz1mw4mGKVgWqM0nL+oyu4= -k8s.io/apiserver v0.25.0/go.mod h1:BKwsE+PTC+aZK+6OJQDPr0v6uS91/HWxX7evElAH6xo= +k8s.io/apiserver v0.25.13 h1:vROP8uv4tqcASYUm6gsBbM04p5umfZI0vHd5WmNbxj4= +k8s.io/apiserver v0.25.13/go.mod h1:EdnRllTlbHg4sjuzFUxoJU8h0OWtJyJbB45dK4QCnrs= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/client-go v0.22.17/go.mod h1:SQPVpN+E/5Q/aSV7fYDT8VKVdaljhxI/t/84ADVJoC4= -k8s.io/client-go v0.25.5 h1:7QWVK0Ph4bLn0UwotPTc2FTgm8shreQXyvXnnHDd8rE= -k8s.io/client-go v0.25.5/go.mod h1:bOeoaUUdpyz3WDFGo+Xm3nOQFh2KuYXRDwrvbAPtFQA= +k8s.io/client-go v0.25.13 h1:Wan/8RXVNxSgFI/wMfWwJjmLglRYuLItytMWNiGo9LY= +k8s.io/client-go v0.25.13/go.mod h1:b2on3RSCwHdmvnUQx4/bkgMAs19M7BlUDze3WJuK0TE= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/code-generator v0.22.17/go.mod h1:iOZwYADSgFPNGWfqHFfg1V0TNJnl1t0WyZluQp4baqU= -k8s.io/code-generator v0.25.0 h1:QP8fJuXu882ztf6dsqJsso/Btm94pMd68TAZC1rE6KI= -k8s.io/code-generator v0.25.0/go.mod h1:B6jZgI3DvDFAualltPitbYMQ74NjaCFxum3YeKZZ+3w= +k8s.io/code-generator v0.25.13 h1:DCjLGhhBNjGWt54fOyMw4mKqgQJT4dCzcihKsLkVXNw= +k8s.io/code-generator v0.25.13/go.mod h1:FA5a4rk4tMTCgmiDeNdRjml+AGvm72SwZYwD5lBrezY= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= k8s.io/component-base v0.22.17/go.mod h1:Mrcvmxs+Ctx/xCYGWoFAvfZO9DC4gDgLtUbPJ4PjjUE= -k8s.io/component-base v0.25.5 h1:tVni0kgpceq71MDMBSixp8Y621YGvTS/1zq3RABgX9A= -k8s.io/component-base v0.25.5/go.mod h1:9J+e9uIUwUOG2x5q5+aaOR0b8QI5OIqwqPAbeODkYpc= +k8s.io/component-base v0.25.13 h1:Lxpq2nu7Q4SueubBlU8kEcTgdaNXsktYg9bLsqvAV+4= +k8s.io/component-base v0.25.13/go.mod h1:Fh9SqG+Uyumtc+7HNT2FBZCxksIIFaukoL2LjqbBtgY= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= @@ -946,8 +946,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 h1:2WjukG7txtEsbXsSKWtTibCdsyYAhcu6KFnttyDdZOQ= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 h1:fAPTNEpzQMOLMGwOHNbUkR2xXTQwMJOZYNx+/mLlOh0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37/go.mod h1:vfnxT4FXNT8eGvO+xi/DsyC/qHmdujqwrUa1WSspCsk= sigs.k8s.io/controller-runtime v0.13.1 h1:tUsRCSJVM1QQOOeViGeX3GMT3dQF1eePPw6sEE3xSlg= sigs.k8s.io/controller-runtime v0.13.1/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= sigs.k8s.io/controller-tools v0.7.0 h1:iZIz1vEcavyEfxjcTLs1WH/MPf4vhPCtTKhoHqV8/G0= diff --git a/hack/dependencies.yml b/hack/dependencies.yml index d364f3ac9..c28a5003e 100644 --- a/hack/dependencies.yml +++ b/hack/dependencies.yml @@ -1,51 +1,51 @@ - checksums: darwin: - amd64: 11445c51206dd2eddb21b33bcbe11e936c5a8ba1db48f2b9b6b0cc60e51956b7 - arm64: d79657cbfd5df6abc22fbaf24a18fb850dfcc3e2068f4655262be33654a2359a + amd64: da83487a7a1d9678d456e9725ecaaebbc02213f8a12190abd88939089c9bc855 + arm64: 2c3d42db34cce6d449ffbd66be8a720d4a0fd39d5754ba05a7842c56304ee716 linux: - amd64: 59520c3c7981581f121f69f5a1394eabacc712fe691a8bd670993ab70e162280 - arm64: 7c49db15ebfa2d53ddeebb3ab895da23b7bd07d39695d16bbea1174308892314 + amd64: abf46655ffcf4b3f0821a96b94eb30c840765c02c50cfe73c895de99abed8186 + arm64: 2fe100931575627060b49b605bce372d1851153bbc9d15c7956616bfa060e58b dev: true name: ytt repo: carvel-dev/ytt urlTemplate: https://github.com/carvel-dev/{{.Name}}/releases/download/{{.Version}}/{{.Name}}-{{.OS}}-{{.Arch}} - version: v0.44.10 + version: v0.44.11 - checksums: darwin: - amd64: c7a0217773d976e84c732a8407180483b63173cfb275ef537529d4ed29c53410 - arm64: 5e7fd1132c8c5e5804c78b047b2e37b7d8be4e673139a6b53db083680fd3e0b5 + amd64: 7666468c9de7cfda987f12d935fee391e88468a0c65bc0c976ce543759c9ad98 + arm64: b3c4885798d33fa5fbd56d695723605ce80f6468bd7ca9aebbf9805ccc4849fb linux: - amd64: 5be70121240cb3a58d1e859fea89f07073ae00662454562d4761d5f47c111103 - arm64: 982eaac7f35bf4d06fabf69aa6b6f1af83357b0514c53b47c40a87d2a8fba2bf + amd64: 1d619a0046dd6d8c588f1bb9855c183de3a2314698e5ac68242286e4bf781a5e + arm64: 681cb522dcf14c22c3ea32e9bfe5d999bff59abaa7eb4795c5031f11560cfd4e dev: true name: kbld repo: carvel-dev/kbld urlTemplate: https://github.com/carvel-dev/{{.Name}}/releases/download/{{.Version}}/{{.Name}}-{{.OS}}-{{.Arch}} - version: v0.36.10 + version: v0.36.11 - checksums: darwin: - amd64: e14a238d272e764104c2cfe46fbf974d7769b522f3ab52161ac6866a94449db0 - arm64: 21fc51b51c1717ab4a77b3aa0af8edf472edf54c08c33a33d98e1a85788ded26 + amd64: d208b8a3dc15ffe733b702493a13dfb30711d45ca283193c656afd995539bee0 + arm64: 45e5af2b983a55baa6f8c8d10bad0a44f7f67417abd4e8eb0b2bce743b39fa9f linux: - amd64: 6d5a5ae0b9871b229f01a812c218f3cc17323575d0ca74931c84c916ce6a1d58 - arm64: 53ef17c48ce5fb3dbd2cd8f1b3de2b8ae2b5d29cf2c8f50f61628ed677743df3 + amd64: 62e9afb3a77625a3b824175739d90e1953394aba51e1dd957fdbc1124b8e8a26 + arm64: fa4dfa7a53dc060b7292723c1cc8aef5a745e0f7539c6f3261e61aaa5e59631f dev: true name: kapp repo: carvel-dev/kapp urlTemplate: https://github.com/carvel-dev/{{.Name}}/releases/download/{{.Version}}/{{.Name}}-{{.OS}}-{{.Arch}} - version: v0.54.8 + version: v0.54.9 - checksums: darwin: - amd64: 5bab34a576817c150707d4a80810cd32a291327e28873674bacf774c8bad19e5 - arm64: 6ec22c1f2fbad1b4d3646cb0d2236e2da9f56f5a74296a0620ff1522e4821f2e + amd64: 79a9a133ce6c829be0421ae55b383de2bdaecd25b05dacf9eb9e67d8209cab46 + arm64: d5bdc9b8511479a7a616c7ed6ce85e0b1ec3cc0d12d63f877530730b61010e35 linux: - amd64: 90837271fbc03da6e72f2cc94b7daaccaf2541813c9946aa7ada8df0abfb3798 - arm64: a7ff71ec3d2beab46ef00f70a046a067719b923a35cd835883677d5532a22e51 + amd64: 00522aca902d31876f9b4948006fb65da0f019e1a5ea2e315db9b41d689c8a0c + arm64: 45403a0417f63c235ccb3f7f0d33e3ff9bba6b904bdfd0abd667e0a455e46280 dev: true name: vendir repo: carvel-dev/vendir urlTemplate: https://github.com/carvel-dev/{{.Name}}/releases/download/{{.Version}}/{{.Name}}-{{.OS}}-{{.Arch}} - version: v0.32.10 + version: v0.32.11 - checksums: linux: amd64: 1b2313cd198d45eab00cc37c38f6b1ca0a948ba279c29e322bdf426d406129b5 diff --git a/pkg/apiserver/openapi/zz_generated.openapi.go b/pkg/apiserver/openapi/zz_generated.openapi.go index c524abf65..d2727730a 100644 --- a/pkg/apiserver/openapi/zz_generated.openapi.go +++ b/pkg/apiserver/openapi/zz_generated.openapi.go @@ -5729,7 +5729,7 @@ func schema_k8sio_api_core_v1_HTTPHeader(ref common.ReferenceCallback) common.Op Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "The header field name", + Description: "The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.", Default: "", Type: []string{"string"}, Format: "", diff --git a/vendor/github.com/google/uuid/null.go b/vendor/github.com/google/uuid/null.go new file mode 100644 index 000000000..d7fcbf286 --- /dev/null +++ b/vendor/github.com/google/uuid/null.go @@ -0,0 +1,118 @@ +// Copyright 2021 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "database/sql/driver" + "encoding/json" + "fmt" +) + +var jsonNull = []byte("null") + +// NullUUID represents a UUID that may be null. +// NullUUID implements the SQL driver.Scanner interface so +// it can be used as a scan destination: +// +// var u uuid.NullUUID +// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&u) +// ... +// if u.Valid { +// // use u.UUID +// } else { +// // NULL value +// } +// +type NullUUID struct { + UUID UUID + Valid bool // Valid is true if UUID is not NULL +} + +// Scan implements the SQL driver.Scanner interface. +func (nu *NullUUID) Scan(value interface{}) error { + if value == nil { + nu.UUID, nu.Valid = Nil, false + return nil + } + + err := nu.UUID.Scan(value) + if err != nil { + nu.Valid = false + return err + } + + nu.Valid = true + return nil +} + +// Value implements the driver Valuer interface. +func (nu NullUUID) Value() (driver.Value, error) { + if !nu.Valid { + return nil, nil + } + // Delegate to UUID Value function + return nu.UUID.Value() +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (nu NullUUID) MarshalBinary() ([]byte, error) { + if nu.Valid { + return nu.UUID[:], nil + } + + return []byte(nil), nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (nu *NullUUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(nu.UUID[:], data) + nu.Valid = true + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (nu NullUUID) MarshalText() ([]byte, error) { + if nu.Valid { + return nu.UUID.MarshalText() + } + + return jsonNull, nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (nu *NullUUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err != nil { + nu.Valid = false + return err + } + nu.UUID = id + nu.Valid = true + return nil +} + +// MarshalJSON implements json.Marshaler. +func (nu NullUUID) MarshalJSON() ([]byte, error) { + if nu.Valid { + return json.Marshal(nu.UUID) + } + + return jsonNull, nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (nu *NullUUID) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, jsonNull) { + *nu = NullUUID{} + return nil // valid null UUID + } + err := json.Unmarshal(data, &nu.UUID) + nu.Valid = err == nil + return err +} diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go index 60d26bb50..a57207aeb 100644 --- a/vendor/github.com/google/uuid/uuid.go +++ b/vendor/github.com/google/uuid/uuid.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "strings" + "sync" ) // A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC @@ -33,7 +34,15 @@ const ( Future // Reserved for future definition. ) -var rander = rand.Reader // random function +const randPoolSize = 16 * 16 + +var ( + rander = rand.Reader // random function + poolEnabled = false + poolMu sync.Mutex + poolPos = randPoolSize // protected with poolMu + pool [randPoolSize]byte // protected with poolMu +) type invalidLengthError struct{ len int } @@ -41,6 +50,12 @@ func (err invalidLengthError) Error() string { return fmt.Sprintf("invalid UUID length: %d", err.len) } +// IsInvalidLengthError is matcher function for custom error invalidLengthError +func IsInvalidLengthError(err error) bool { + _, ok := err.(invalidLengthError) + return ok +} + // Parse decodes s into a UUID or returns an error. Both the standard UUID // forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the @@ -249,3 +264,31 @@ func SetRand(r io.Reader) { } rander = r } + +// EnableRandPool enables internal randomness pool used for Random +// (Version 4) UUID generation. The pool contains random bytes read from +// the random number generator on demand in batches. Enabling the pool +// may improve the UUID generation throughput significantly. +// +// Since the pool is stored on the Go heap, this feature may be a bad fit +// for security sensitive applications. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func EnableRandPool() { + poolEnabled = true +} + +// DisableRandPool disables the randomness pool if it was previously +// enabled with EnableRandPool. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func DisableRandPool() { + poolEnabled = false + defer poolMu.Unlock() + poolMu.Lock() + poolPos = randPoolSize +} diff --git a/vendor/github.com/google/uuid/version4.go b/vendor/github.com/google/uuid/version4.go index 86160fbd0..7697802e4 100644 --- a/vendor/github.com/google/uuid/version4.go +++ b/vendor/github.com/google/uuid/version4.go @@ -27,6 +27,8 @@ func NewString() string { // The strength of the UUIDs is based on the strength of the crypto/rand // package. // +// Uses the randomness pool if it was enabled with EnableRandPool. +// // A note about uniqueness derived from the UUID Wikipedia entry: // // Randomly generated UUIDs have 122 random bits. One's annual risk of being @@ -35,7 +37,10 @@ func NewString() string { // equivalent to the odds of creating a few tens of trillions of UUIDs in a // year and having one duplicate. func NewRandom() (UUID, error) { - return NewRandomFromReader(rander) + if !poolEnabled { + return NewRandomFromReader(rander) + } + return newRandomFromPool() } // NewRandomFromReader returns a UUID based on bytes read from a given io.Reader. @@ -49,3 +54,23 @@ func NewRandomFromReader(r io.Reader) (UUID, error) { uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 return uuid, nil } + +func newRandomFromPool() (UUID, error) { + var uuid UUID + poolMu.Lock() + if poolPos == randPoolSize { + _, err := io.ReadFull(rander, pool[:]) + if err != nil { + poolMu.Unlock() + return Nil, err + } + poolPos = 0 + } + copy(uuid[:], pool[poolPos:(poolPos+16)]) + poolPos += 16 + poolMu.Unlock() + + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/vendor/k8s.io/api/core/v1/generated.proto b/vendor/k8s.io/api/core/v1/generated.proto index 256f5da32..e4e562e32 100644 --- a/vendor/k8s.io/api/core/v1/generated.proto +++ b/vendor/k8s.io/api/core/v1/generated.proto @@ -1761,7 +1761,8 @@ message HTTPGetAction { // HTTPHeader describes a custom header to be used in HTTP probes message HTTPHeader { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. optional string name = 1; // The header field value diff --git a/vendor/k8s.io/api/core/v1/types.go b/vendor/k8s.io/api/core/v1/types.go index 754a23613..e60e180c2 100644 --- a/vendor/k8s.io/api/core/v1/types.go +++ b/vendor/k8s.io/api/core/v1/types.go @@ -2112,7 +2112,8 @@ type SecretEnvSource struct { // HTTPHeader describes a custom header to be used in HTTP probes type HTTPHeader struct { - // The header field name + // The header field name. + // This will be canonicalized upon output, so case-variant names will be understood as the same header. Name string `json:"name" protobuf:"bytes,1,opt,name=name"` // The header field value Value string `json:"value" protobuf:"bytes,2,opt,name=value"` @@ -4266,6 +4267,9 @@ const ( // LoadBalancerPortsError represents the condition of the requested ports // on the cloud load balancer instance. LoadBalancerPortsError = "LoadBalancerPortsError" + // LoadBalancerPortsErrorReason reason in ServiceStatus condition LoadBalancerPortsError + // means the LoadBalancer was not able to be configured correctly. + LoadBalancerPortsErrorReason = "LoadBalancerMixedProtocolNotSupported" ) // ServiceStatus represents the current status of a service. @@ -6624,6 +6628,13 @@ const ( PortForwardRequestIDHeader = "requestID" ) +const ( + // MixedProtocolNotSupported error in PortStatus means that the cloud provider + // can't publish the port on the load balancer because mixed values of protocols + // on the same LoadBalancer type of Service are not supported by the cloud provider. + MixedProtocolNotSupported = "MixedProtocolNotSupported" +) + // PortStatus represents the error condition of a service port type PortStatus struct { diff --git a/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go b/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go index 6bae4bb76..77707706a 100644 --- a/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/vendor/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -808,7 +808,7 @@ func (HTTPGetAction) SwaggerDoc() map[string]string { var map_HTTPHeader = map[string]string{ "": "HTTPHeader describes a custom header to be used in HTTP probes", - "name": "The header field name", + "name": "The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.", "value": "The header field value", } diff --git a/vendor/k8s.io/apimachinery/pkg/runtime/converter.go b/vendor/k8s.io/apimachinery/pkg/runtime/converter.go index 90bf487e3..62eb27afc 100644 --- a/vendor/k8s.io/apimachinery/pkg/runtime/converter.go +++ b/vendor/k8s.io/apimachinery/pkg/runtime/converter.go @@ -231,7 +231,7 @@ func (c *fromUnstructuredContext) pushKey(key string) { } -// FromUnstructuredWIthValidation converts an object from map[string]interface{} representation into a concrete type. +// FromUnstructuredWithValidation converts an object from map[string]interface{} representation into a concrete type. // It uses encoding/json/Unmarshaler if object implements it or reflection if not. // It takes a validationDirective that indicates how to behave when it encounters unknown fields. func (c *unstructuredConverter) FromUnstructuredWithValidation(u map[string]interface{}, obj interface{}, returnUnknownFields bool) error { @@ -465,7 +465,7 @@ func sliceFromUnstructured(sv, dv reflect.Value, ctx *fromUnstructuredContext) e } dv.SetBytes(data) } else { - dv.Set(reflect.Zero(dt)) + dv.Set(reflect.MakeSlice(dt, 0, 0)) } return nil } diff --git a/vendor/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader.go b/vendor/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader.go index abf509a97..d02732739 100644 --- a/vendor/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader.go +++ b/vendor/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader.go @@ -163,17 +163,7 @@ func (a *requestHeaderAuthRequestHandler) AuthenticateRequest(req *http.Request) extra := newExtra(req.Header, a.extraHeaderPrefixes.Value()) // clear headers used for authentication - for _, headerName := range a.nameHeaders.Value() { - req.Header.Del(headerName) - } - for _, headerName := range a.groupHeaders.Value() { - req.Header.Del(headerName) - } - for k := range extra { - for _, prefix := range a.extraHeaderPrefixes.Value() { - req.Header.Del(prefix + k) - } - } + ClearAuthenticationHeaders(req.Header, a.nameHeaders, a.groupHeaders, a.extraHeaderPrefixes) return &authenticator.Response{ User: &user.DefaultInfo{ @@ -184,6 +174,26 @@ func (a *requestHeaderAuthRequestHandler) AuthenticateRequest(req *http.Request) }, true, nil } +func ClearAuthenticationHeaders(h http.Header, nameHeaders, groupHeaders, extraHeaderPrefixes StringSliceProvider) { + for _, headerName := range nameHeaders.Value() { + h.Del(headerName) + } + for _, headerName := range groupHeaders.Value() { + h.Del(headerName) + } + for _, prefix := range extraHeaderPrefixes.Value() { + for k := range h { + if hasPrefixIgnoreCase(k, prefix) { + delete(h, k) // we have the raw key so avoid relying on canonicalization + } + } + } +} + +func hasPrefixIgnoreCase(s, prefix string) bool { + return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) +} + func headerValue(h http.Header, headerNames []string) string { for _, headerName := range headerNames { headerValue := h.Get(headerName) @@ -226,7 +236,7 @@ func newExtra(h http.Header, headerPrefixes []string) map[string][]string { // we have to iterate over prefixes first in order to have proper ordering inside the value slices for _, prefix := range headerPrefixes { for headerName, vv := range h { - if !strings.HasPrefix(strings.ToLower(headerName), strings.ToLower(prefix)) { + if !hasPrefixIgnoreCase(headerName, prefix) { continue } diff --git a/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go b/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go index d69cfef32..d6741bf3a 100644 --- a/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go +++ b/vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go @@ -27,6 +27,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authentication/request/headerrequest" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" @@ -38,15 +40,20 @@ type recordMetrics func(context.Context, *authenticator.Response, bool, error, a // stores any such user found onto the provided context for the request. If authentication fails or returns an error // the failed handler is used. On success, "Authorization" header is removed from the request and handler // is invoked to serve the request. -func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences) http.Handler { - return withAuthentication(handler, auth, failed, apiAuds, recordAuthMetrics) +func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, requestHeaderConfig *authenticatorfactory.RequestHeaderConfig) http.Handler { + return withAuthentication(handler, auth, failed, apiAuds, requestHeaderConfig, recordAuthMetrics) } -func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, metrics recordMetrics) http.Handler { +func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, requestHeaderConfig *authenticatorfactory.RequestHeaderConfig, metrics recordMetrics) http.Handler { if auth == nil { klog.Warning("Authentication is disabled") return handler } + standardRequestHeaderConfig := &authenticatorfactory.RequestHeaderConfig{ + UsernameHeaders: headerrequest.StaticStringSlice{"X-Remote-User"}, + GroupHeaders: headerrequest.StaticStringSlice{"X-Remote-Group"}, + ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"X-Remote-Extra-"}, + } return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { authenticationStart := time.Now() @@ -76,6 +83,24 @@ func withAuthentication(handler http.Handler, auth authenticator.Request, failed // authorization header is not required anymore in case of a successful authentication. req.Header.Del("Authorization") + // delete standard front proxy headers + headerrequest.ClearAuthenticationHeaders( + req.Header, + standardRequestHeaderConfig.UsernameHeaders, + standardRequestHeaderConfig.GroupHeaders, + standardRequestHeaderConfig.ExtraHeaderPrefixes, + ) + + // also delete any custom front proxy headers + if requestHeaderConfig != nil { + headerrequest.ClearAuthenticationHeaders( + req.Header, + requestHeaderConfig.UsernameHeaders, + requestHeaderConfig.GroupHeaders, + requestHeaderConfig.ExtraHeaderPrefixes, + ) + } + req = req.WithContext(genericapirequest.WithUser(req.Context(), resp.User)) handler.ServeHTTP(w, req) }) diff --git a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go index 1f3ce8094..0e88cdbb7 100644 --- a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go +++ b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go @@ -164,8 +164,13 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int userInfo, _ := request.UserFrom(ctx) if objectMeta, err := meta.Accessor(obj); err == nil { - // Wipe fields which cannot take user-provided values - rest.WipeObjectMetaSystemFields(objectMeta) + preserveObjectMetaSystemFields := false + if c, ok := r.(rest.SubresourceObjectMetaPreserver); ok && len(scope.Subresource) > 0 { + preserveObjectMetaSystemFields = c.PreserveRequestObjectMetaSystemFieldsOnSubresourceCreate() + } + if !preserveObjectMetaSystemFields { + rest.WipeObjectMetaSystemFields(objectMeta) + } // ensure namespace on the object is correct, or error if a conflicting namespace was set in the object if err := rest.EnsureObjectNamespaceMatchesRequestNamespace(rest.ExpectedNamespaceForResource(namespace, scope.Resource), objectMeta); err != nil { diff --git a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go index a7712f115..2a7a5da4a 100644 --- a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go +++ b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go @@ -183,12 +183,10 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc return } - // enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided - // timeout inside the parent context is lower than requestTimeoutUpperBound. - ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound) - defer cancel() - - ctx = request.WithNamespace(ctx, namespace) + // DELETECOLLECTION can be a lengthy operation, + // we should not impose any 34s timeout here. + // NOTE: This is similar to LIST which does not enforce a 34s timeout. + ctx := request.WithNamespace(req.Context(), namespace) outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope) if err != nil { @@ -231,7 +229,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc return } if len(body) > 0 { - s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer) + s, err := negotiation.NegotiateInputSerializer(req, false, metainternalversionscheme.Codecs) if err != nil { scope.err(err, w, req) return diff --git a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go index 366d88fcc..a1f27f1d1 100644 --- a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go +++ b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/equality.go @@ -22,6 +22,7 @@ import ( "os" "reflect" "strconv" + "sync" "time" "k8s.io/apimachinery/pkg/api/equality" @@ -33,42 +34,41 @@ import ( "k8s.io/klog/v2" ) -func determineAvoidNoopTimestampUpdatesEnabled() bool { - if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists { - if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil { - return ret - } else { - klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err) - } - } - - // enabled by default - return true -} - var ( - avoidNoopTimestampUpdatesEnabled = determineAvoidNoopTimestampUpdatesEnabled() + avoidTimestampEqualities conversion.Equalities + initAvoidTimestampEqualities sync.Once ) -var avoidTimestampEqualities = func() conversion.Equalities { - var eqs = equality.Semantic.Copy() +func getAvoidTimestampEqualities() conversion.Equalities { + initAvoidTimestampEqualities.Do(func() { + if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists { + if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil && !ret { + // leave avoidTimestampEqualities empty. + return + } else { + klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err) + } + } - err := eqs.AddFunc( - func(a, b metav1.ManagedFieldsEntry) bool { - // Two objects' managed fields are equivalent if, ignoring timestamp, - // the objects are deeply equal. - a.Time = nil - b.Time = nil - return reflect.DeepEqual(a, b) - }, - ) + var eqs = equality.Semantic.Copy() + err := eqs.AddFunc( + func(a, b metav1.ManagedFieldsEntry) bool { + // Two objects' managed fields are equivalent if, ignoring timestamp, + // the objects are deeply equal. + a.Time = nil + b.Time = nil + return reflect.DeepEqual(a, b) + }, + ) - if err != nil { - panic(err) - } + if err != nil { + panic(fmt.Errorf("failed to instantiate semantic equalities: %w", err)) + } - return eqs -}() + avoidTimestampEqualities = eqs + }) + return avoidTimestampEqualities +} // IgnoreManagedFieldsTimestampsTransformer reverts timestamp updates // if the non-managed parts of the object are equivalent @@ -77,7 +77,8 @@ func IgnoreManagedFieldsTimestampsTransformer( newObj runtime.Object, oldObj runtime.Object, ) (res runtime.Object, err error) { - if !avoidNoopTimestampUpdatesEnabled { + equalities := getAvoidTimestampEqualities() + if len(equalities.Equalities) == 0 { return newObj, nil } @@ -154,11 +155,11 @@ func IgnoreManagedFieldsTimestampsTransformer( // This condition ensures the managed fields are always compared first. If // this check fails, the if statement will short circuit. If the check // succeeds the slow path is taken which compares entire objects. - if !avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) { + if !equalities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) { return newObj, nil } - if avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) { + if equalities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) { // Remove any changed timestamps, so that timestamp is not the only // change seen by etcd. // diff --git a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go index acb08800e..4824235d1 100644 --- a/vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go +++ b/vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go @@ -144,8 +144,10 @@ var gzipPool = &sync.Pool{ } const ( - // defaultGzipContentEncodingLevel is set to 4 which uses less CPU than the default level - defaultGzipContentEncodingLevel = 4 + // defaultGzipContentEncodingLevel is set to 1 which uses least CPU compared to higher levels, yet offers + // similar compression ratios (off by at most 1.5x, but typically within 1.1x-1.3x). For further details see - + // https://github.com/kubernetes/kubernetes/issues/112296 + defaultGzipContentEncodingLevel = 1 // defaultGzipThresholdBytes is compared to the size of the first write from the stream // (usually the entire object), and if the size is smaller no gzipping will be performed // if the client requests it. diff --git a/vendor/k8s.io/apiserver/pkg/registry/rest/rest.go b/vendor/k8s.io/apiserver/pkg/registry/rest/rest.go index 6330ea8f5..7b8d90e60 100644 --- a/vendor/k8s.io/apiserver/pkg/registry/rest/rest.go +++ b/vendor/k8s.io/apiserver/pkg/registry/rest/rest.go @@ -203,6 +203,13 @@ type NamedCreater interface { Create(ctx context.Context, name string, obj runtime.Object, createValidation ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) } +// SubresourceObjectMetaPreserver adds configuration options to a Creater for subresources. +type SubresourceObjectMetaPreserver interface { + // PreserveRequestObjectMetaSystemFieldsOnSubresourceCreate indicates that a + // handler should preserve fields of ObjectMeta that are managed by the system. + PreserveRequestObjectMetaSystemFieldsOnSubresourceCreate() bool +} + // UpdatedObjectInfo provides information about an updated object to an Updater. // It requires access to the old object in order to return the newly updated object. type UpdatedObjectInfo interface { diff --git a/vendor/k8s.io/apiserver/pkg/server/config.go b/vendor/k8s.io/apiserver/pkg/server/config.go index d21ea2ef0..fd650f6d3 100644 --- a/vendor/k8s.io/apiserver/pkg/server/config.go +++ b/vendor/k8s.io/apiserver/pkg/server/config.go @@ -309,6 +309,8 @@ type AuthenticationInfo struct { APIAudiences authenticator.Audiences // Authenticator determines which subject is making the request Authenticator authenticator.Request + + RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig } type AuthorizationInfo struct { @@ -813,7 +815,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { if c.FlowControl != nil { workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig() requestWorkEstimator := flowcontrolrequest.NewWorkEstimator( - c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg) + c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats) handler = filterlatency.TrackCompleted(handler) handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator) handler = filterlatency.TrackStarted(handler, "priorityandfairness") @@ -834,7 +836,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler { failedHandler = filterlatency.TrackCompleted(failedHandler) handler = filterlatency.TrackCompleted(handler) - handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences) + handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig) handler = filterlatency.TrackStarted(handler, "authentication") handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true") diff --git a/vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go b/vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go index 3f7bd4ad5..c2511e792 100644 --- a/vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go +++ b/vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go @@ -216,6 +216,9 @@ func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) { // See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357. tunnelCtx := context.TODO() tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption, + grpc.WithBlock(), + grpc.WithReturnConnectionError(), + grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, err diff --git a/vendor/k8s.io/apiserver/pkg/server/options/authentication.go b/vendor/k8s.io/apiserver/pkg/server/options/authentication.go index 8ff771af0..283ecc450 100644 --- a/vendor/k8s.io/apiserver/pkg/server/options/authentication.go +++ b/vendor/k8s.io/apiserver/pkg/server/options/authentication.go @@ -76,6 +76,16 @@ func (s *RequestHeaderAuthenticationOptions) Validate() []error { allErrors = append(allErrors, err) } + if len(s.UsernameHeaders) > 0 && !caseInsensitiveHas(s.UsernameHeaders, "X-Remote-User") { + klog.Warningf("--requestheader-username-headers is set without specifying the standard X-Remote-User header - API aggregation will not work") + } + if len(s.GroupHeaders) > 0 && !caseInsensitiveHas(s.GroupHeaders, "X-Remote-Group") { + klog.Warningf("--requestheader-group-headers is set without specifying the standard X-Remote-Group header - API aggregation will not work") + } + if len(s.ExtraHeaderPrefixes) > 0 && !caseInsensitiveHas(s.ExtraHeaderPrefixes, "X-Remote-Extra-") { + klog.Warningf("--requestheader-extra-headers-prefix is set without specifying the standard X-Remote-Extra- header prefix - API aggregation will not work") + } + return allErrors } @@ -89,6 +99,15 @@ func checkForWhiteSpaceOnly(flag string, headerNames ...string) error { return nil } +func caseInsensitiveHas(headers []string, header string) bool { + for _, h := range headers { + if strings.EqualFold(h, header) { + return true + } + } + return false +} + func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) { if s == nil { return @@ -354,6 +373,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut } if requestHeaderConfig != nil { cfg.RequestHeaderConfig = requestHeaderConfig + authenticationInfo.RequestHeaderConfig = requestHeaderConfig if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil { return fmt.Errorf("unable to load request-header-client-ca-file: %v", err) } diff --git a/vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go index d8de4cd84..3aebab7de 100644 --- a/vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -22,8 +22,6 @@ import ( "io/ioutil" "strings" - "k8s.io/klog/v2" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -31,6 +29,7 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/value" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/klog/v2" ) // Backend describes the storage servers, the information here should be enough @@ -53,8 +52,12 @@ type StorageFactory interface { // centralized control over the shape of etcd directories ResourcePrefix(groupResource schema.GroupResource) string + // Configs gets configurations for all of registered storage destinations. + Configs() []storagebackend.Config + // Backends gets all backends for all registered storage destinations. // Used for getting all instances for health validations. + // Deprecated: Use Configs instead Backends() []Backend } @@ -288,28 +291,76 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* return storageConfig.ForResource(groupResource), nil } +// Configs implements StorageFactory. +func (s *DefaultStorageFactory) Configs() []storagebackend.Config { + return configs(s.StorageConfig, s.Overrides) +} + +// Configs gets configurations for all of registered storage destinations. +func Configs(storageConfig storagebackend.Config) []storagebackend.Config { + return configs(storageConfig, nil) +} + +// Returns all storage configurations including those for group resource overrides +func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config { + locations := sets.NewString() + configs := []storagebackend.Config{} + for _, loc := range storageConfig.Transport.ServerList { + // copy + newConfig := storageConfig + newConfig.Transport.ServerList = []string{loc} + configs = append(configs, newConfig) + locations.Insert(loc) + } + + for _, override := range grOverrides { + for _, loc := range override.etcdLocation { + if locations.Has(loc) { + continue + } + // copy + newConfig := storageConfig + override.Apply(&newConfig, &StorageCodecConfig{}) + newConfig.Transport.ServerList = []string{loc} + configs = append(configs, newConfig) + locations.Insert(loc) + } + } + return configs +} + +// Backends implements StorageFactory. +func (s *DefaultStorageFactory) Backends() []Backend { + return backends(s.StorageConfig, s.Overrides) +} + // Backends returns all backends for all registered storage destinations. // Used for getting all instances for health validations. -func (s *DefaultStorageFactory) Backends() []Backend { - servers := sets.NewString(s.StorageConfig.Transport.ServerList...) +// Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber. +func Backends(storageConfig storagebackend.Config) []Backend { + return backends(storageConfig, nil) +} + +func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend { + servers := sets.NewString(storageConfig.Transport.ServerList...) - for _, overrides := range s.Overrides { + for _, overrides := range grOverrides { servers.Insert(overrides.etcdLocation...) } tlsConfig := &tls.Config{ InsecureSkipVerify: true, } - if len(s.StorageConfig.Transport.CertFile) > 0 && len(s.StorageConfig.Transport.KeyFile) > 0 { - cert, err := tls.LoadX509KeyPair(s.StorageConfig.Transport.CertFile, s.StorageConfig.Transport.KeyFile) + if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 { + cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile) if err != nil { klog.Errorf("failed to load key pair while getting backends: %s", err) } else { tlsConfig.Certificates = []tls.Certificate{cert} } } - if len(s.StorageConfig.Transport.TrustedCAFile) > 0 { - if caCert, err := ioutil.ReadFile(s.StorageConfig.Transport.TrustedCAFile); err != nil { + if len(storageConfig.Transport.TrustedCAFile) > 0 { + if caCert, err := ioutil.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil { klog.Errorf("failed to read ca file while getting backends: %s", err) } else { caPool := x509.NewCertPool() diff --git a/vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go b/vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go index 24dd64d9e..0d70f2d65 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go +++ b/vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go @@ -477,7 +477,7 @@ func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions return nil, err } - if err := c.ready.wait(); err != nil { + if err := c.ready.wait(ctx); err != nil { return nil, errors.NewServiceUnavailable(err.Error()) } @@ -567,7 +567,7 @@ func (c *Cacher) Get(ctx context.Context, key string, opts storage.GetOptions, o // Do not create a trace - it's not for free and there are tons // of Get requests. We can add it if it will be really needed. - if err := c.ready.wait(); err != nil { + if err := c.ready.wait(ctx); err != nil { return errors.NewServiceUnavailable(err.Error()) } @@ -657,7 +657,7 @@ func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptio utiltrace.Field{Key: "type", Value: c.objectType.String()}) defer trace.LogIfLong(500 * time.Millisecond) - if err := c.ready.wait(); err != nil { + if err := c.ready.wait(ctx); err != nil { return errors.NewServiceUnavailable(err.Error()) } trace.Step("Ready") @@ -1066,7 +1066,7 @@ func filterWithAttrsFunction(key string, p storage.SelectionPredicate) filterWit // LastSyncResourceVersion returns resource version to which the underlying cache is synced. func (c *Cacher) LastSyncResourceVersion() (uint64, error) { - if err := c.ready.wait(); err != nil { + if err := c.ready.wait(context.Background()); err != nil { return 0, errors.NewServiceUnavailable(err.Error()) } diff --git a/vendor/k8s.io/apiserver/pkg/storage/cacher/ready.go b/vendor/k8s.io/apiserver/pkg/storage/cacher/ready.go index 8278dd2b2..47e03fe9e 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/cacher/ready.go +++ b/vendor/k8s.io/apiserver/pkg/storage/cacher/ready.go @@ -17,6 +17,7 @@ limitations under the License. package cacher import ( + "context" "fmt" "sync" ) @@ -30,67 +31,111 @@ const ( ) // ready is a three state condition variable that blocks until is Ready if is not Stopped. -// Its initial state is Pending. +// Its initial state is Pending and its state machine diagram is as follow. +// +// Pending <------> Ready -----> Stopped +// +// | ^ +// └---------------------------┘ type ready struct { - state status - c *sync.Cond + state status // represent the state of the variable + lock sync.RWMutex // protect the state variable + restartLock sync.Mutex // protect the transition from ready to pending where the channel is recreated + waitCh chan struct{} // blocks until is ready or stopped } func newReady() *ready { return &ready{ - c: sync.NewCond(&sync.RWMutex{}), - state: Pending, + waitCh: make(chan struct{}), + state: Pending, } } +// done close the channel once the state is Ready or Stopped +func (r *ready) done() chan struct{} { + r.restartLock.Lock() + defer r.restartLock.Unlock() + return r.waitCh +} + // wait blocks until it is Ready or Stopped, it returns an error if is Stopped. -func (r *ready) wait() error { - r.c.L.Lock() - defer r.c.L.Unlock() - for r.state == Pending { - r.c.Wait() - } - switch r.state { - case Ready: - return nil - case Stopped: - return fmt.Errorf("apiserver cacher is stopped") - default: - return fmt.Errorf("unexpected apiserver cache state: %v", r.state) +func (r *ready) wait(ctx context.Context) error { + for { + // r.done() only blocks if state is Pending + select { + case <-ctx.Done(): + return ctx.Err() + case <-r.done(): + } + + r.lock.RLock() + switch r.state { + case Pending: + // since we allow to switch between the states Pending and Ready + // if there is a quick transition from Pending -> Ready -> Pending + // a process that was waiting can get unblocked and see a Pending + // state again. If the state is Pending we have to wait again to + // avoid an inconsistent state on the system, with some processes not + // waiting despite the state moved back to Pending. + r.lock.RUnlock() + case Ready: + r.lock.RUnlock() + return nil + case Stopped: + r.lock.RUnlock() + return fmt.Errorf("apiserver cacher is stopped") + default: + r.lock.RUnlock() + return fmt.Errorf("unexpected apiserver cache state: %v", r.state) + } } } // check returns true only if it is Ready. func (r *ready) check() bool { - // TODO: Make check() function more sophisticated, in particular - // allow it to behave as "waitWithTimeout". - rwMutex := r.c.L.(*sync.RWMutex) - rwMutex.RLock() - defer rwMutex.RUnlock() + r.lock.RLock() + defer r.lock.RUnlock() return r.state == Ready } // set the state to Pending (false) or Ready (true), it does not have effect if the state is Stopped. func (r *ready) set(ok bool) { - r.c.L.Lock() - defer r.c.L.Unlock() + r.lock.Lock() + defer r.lock.Unlock() if r.state == Stopped { return } - if ok { + if ok && r.state == Pending { r.state = Ready - } else { + select { + case <-r.waitCh: + default: + close(r.waitCh) + } + } else if !ok && r.state == Ready { + // creating the waitCh can be racy if + // something enter the wait() method + select { + case <-r.waitCh: + r.restartLock.Lock() + r.waitCh = make(chan struct{}) + r.restartLock.Unlock() + default: + } r.state = Pending } - r.c.Broadcast() } // stop the condition variable and set it as Stopped. This state is irreversible. func (r *ready) stop() { - r.c.L.Lock() - defer r.c.L.Unlock() + r.lock.Lock() + defer r.lock.Unlock() if r.state != Stopped { r.state = Stopped - r.c.Broadcast() + } + select { + case <-r.waitCh: + default: + close(r.waitCh) } } diff --git a/vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go b/vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go index ad051d2d6..3d4898103 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go +++ b/vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go @@ -28,6 +28,7 @@ type etcdHealth struct { } // EtcdHealthCheck decodes data returned from etcd /healthz handler. +// Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber. func EtcdHealthCheck(data []byte) error { obj := etcdHealth{} if err := json.Unmarshal(data, &obj); err != nil { diff --git a/vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go b/vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go index 163eb111b..b2e4c6746 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go +++ b/vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go @@ -98,16 +98,21 @@ func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, func newStore(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, groupResource schema.GroupResource, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) *store { versioner := storage.APIObjectVersioner{} + // for compatibility with etcd2 impl. + // no-op for default prefix of '/registry'. + // keeps compatibility with etcd2 impl for custom prefixes that don't start with '/' + pathPrefix := path.Join("/", prefix) + if !strings.HasSuffix(pathPrefix, "/") { + // Ensure the pathPrefix ends in "/" here to simplify key concatenation later. + pathPrefix += "/" + } result := &store{ - client: c, - codec: codec, - versioner: versioner, - transformer: transformer, - pagingEnabled: pagingEnabled, - // for compatibility with etcd2 impl. - // no-op for default prefix of '/registry'. - // keeps compatibility with etcd2 impl for custom prefixes that don't start with '/' - pathPrefix: path.Join("/", prefix), + client: c, + codec: codec, + versioner: versioner, + transformer: transformer, + pagingEnabled: pagingEnabled, + pathPrefix: pathPrefix, groupResource: groupResource, groupResourceString: groupResource.String(), watcher: newWatcher(c, codec, newFunc, versioner, transformer), @@ -123,9 +128,12 @@ func (s *store) Versioner() storage.Versioner { // Get implements storage.Interface.Get. func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error { - key = path.Join(s.pathPrefix, key) + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } startTime := time.Now() - getResp, err := s.client.KV.Get(ctx, key) + getResp, err := s.client.KV.Get(ctx, preparedKey) metrics.RecordEtcdRequestLatency("get", getTypeName(out), startTime) if err != nil { return err @@ -138,11 +146,11 @@ func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, ou if opts.IgnoreNotFound { return runtime.SetZeroValue(out) } - return storage.NewKeyNotFoundError(key, 0) + return storage.NewKeyNotFoundError(preparedKey, 0) } kv := getResp.Kvs[0] - data, _, err := s.transformer.TransformFromStorage(ctx, kv.Value, authenticatedDataString(key)) + data, _, err := s.transformer.TransformFromStorage(ctx, kv.Value, authenticatedDataString(preparedKey)) if err != nil { return storage.NewInternalError(err.Error()) } @@ -152,6 +160,10 @@ func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, ou // Create implements storage.Interface.Create. func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } trace := utiltrace.New("Create etcd3", utiltrace.Field{"audit-id", endpointsrequest.GetAuditIDTruncated(ctx)}, utiltrace.Field{"key", key}, @@ -170,14 +182,13 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, if err != nil { return err } - key = path.Join(s.pathPrefix, key) opts, err := s.ttlOpts(ctx, int64(ttl)) if err != nil { return err } - newData, err := s.transformer.TransformToStorage(ctx, data, authenticatedDataString(key)) + newData, err := s.transformer.TransformToStorage(ctx, data, authenticatedDataString(preparedKey)) trace.Step("TransformToStorage finished", utiltrace.Field{"err", err}) if err != nil { return storage.NewInternalError(err.Error()) @@ -185,9 +196,9 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, startTime := time.Now() txnResp, err := s.client.KV.Txn(ctx).If( - notFound(key), + notFound(preparedKey), ).Then( - clientv3.OpPut(key, string(newData), opts...), + clientv3.OpPut(preparedKey, string(newData), opts...), ).Commit() metrics.RecordEtcdRequestLatency("create", getTypeName(obj), startTime) trace.Step("Txn call finished", utiltrace.Field{"err", err}) @@ -196,7 +207,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, } if !txnResp.Succeeded { - return storage.NewKeyExistsError(key, 0) + return storage.NewKeyExistsError(preparedKey, 0) } if out != nil { @@ -212,12 +223,15 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, func (s *store) Delete( ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } v, err := conversion.EnforcePtr(out) if err != nil { return fmt.Errorf("unable to convert output object to pointer: %v", err) } - key = path.Join(s.pathPrefix, key) - return s.conditionalDelete(ctx, key, out, v, preconditions, validateDeletion, cachedExistingObject) + return s.conditionalDelete(ctx, preparedKey, out, v, preconditions, validateDeletion, cachedExistingObject) } func (s *store) conditionalDelete( @@ -330,6 +344,10 @@ func (s *store) conditionalDelete( func (s *store) GuaranteedUpdate( ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool, preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } trace := utiltrace.New("GuaranteedUpdate etcd3", utiltrace.Field{"audit-id", endpointsrequest.GetAuditIDTruncated(ctx)}, utiltrace.Field{"key", key}, @@ -340,16 +358,15 @@ func (s *store) GuaranteedUpdate( if err != nil { return fmt.Errorf("unable to convert output object to pointer: %v", err) } - key = path.Join(s.pathPrefix, key) getCurrentState := func() (*objState, error) { startTime := time.Now() - getResp, err := s.client.KV.Get(ctx, key) + getResp, err := s.client.KV.Get(ctx, preparedKey) metrics.RecordEtcdRequestLatency("get", getTypeName(destination), startTime) if err != nil { return nil, err } - return s.getState(ctx, getResp, key, v, ignoreNotFound) + return s.getState(ctx, getResp, preparedKey, v, ignoreNotFound) } var origState *objState @@ -365,9 +382,9 @@ func (s *store) GuaranteedUpdate( } trace.Step("initial value restored") - transformContext := authenticatedDataString(key) + transformContext := authenticatedDataString(preparedKey) for { - if err := preconditions.Check(key, origState.obj); err != nil { + if err := preconditions.Check(preparedKey, origState.obj); err != nil { // If our data is already up to date, return the error if origStateIsCurrent { return err @@ -453,11 +470,11 @@ func (s *store) GuaranteedUpdate( startTime := time.Now() txnResp, err := s.client.KV.Txn(ctx).If( - clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev), + clientv3.Compare(clientv3.ModRevision(preparedKey), "=", origState.rev), ).Then( - clientv3.OpPut(key, string(newData), opts...), + clientv3.OpPut(preparedKey, string(newData), opts...), ).Else( - clientv3.OpGet(key), + clientv3.OpGet(preparedKey), ).Commit() metrics.RecordEtcdRequestLatency("update", getTypeName(destination), startTime) trace.Step("Txn call finished", utiltrace.Field{"err", err}) @@ -467,8 +484,8 @@ func (s *store) GuaranteedUpdate( trace.Step("Transaction committed") if !txnResp.Succeeded { getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange()) - klog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key) - origState, err = s.getState(ctx, getResp, key, v, ignoreNotFound) + klog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", preparedKey) + origState, err = s.getState(ctx, getResp, preparedKey, v, ignoreNotFound) if err != nil { return err } @@ -502,18 +519,21 @@ func getNewItemFunc(listObj runtime.Object, v reflect.Value) func() runtime.Obje } func (s *store) Count(key string) (int64, error) { - key = path.Join(s.pathPrefix, key) + preparedKey, err := s.prepareKey(key) + if err != nil { + return 0, err + } // We need to make sure the key ended with "/" so that we only get children "directories". // e.g. if we have key "/a", "/a/b", "/ab", getting keys with prefix "/a" will return all three, // while with prefix "/a/" will return only "/a/b" which is the correct answer. - if !strings.HasSuffix(key, "/") { - key += "/" + if !strings.HasSuffix(preparedKey, "/") { + preparedKey += "/" } startTime := time.Now() - getResp, err := s.client.KV.Get(context.Background(), key, clientv3.WithRange(clientv3.GetPrefixRangeEnd(key)), clientv3.WithCountOnly()) - metrics.RecordEtcdRequestLatency("listWithCount", key, startTime) + getResp, err := s.client.KV.Get(context.Background(), preparedKey, clientv3.WithRange(clientv3.GetPrefixRangeEnd(preparedKey)), clientv3.WithCountOnly()) + metrics.RecordEtcdRequestLatency("listWithCount", preparedKey, startTime) if err != nil { return 0, err } @@ -522,6 +542,10 @@ func (s *store) Count(key string) (int64, error) { // GetList implements storage.Interface. func (s *store) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { + preparedKey, err := s.prepareKey(key) + if err != nil { + return err + } recursive := opts.Recursive resourceVersion := opts.ResourceVersion match := opts.ResourceVersionMatch @@ -542,16 +566,15 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption if err != nil || v.Kind() != reflect.Slice { return fmt.Errorf("need ptr to slice: %v", err) } - key = path.Join(s.pathPrefix, key) // For recursive lists, we need to make sure the key ended with "/" so that we only // get children "directories". e.g. if we have key "/a", "/a/b", "/ab", getting keys // with prefix "/a" will return all three, while with prefix "/a/" will return only // "/a/b" which is the correct answer. - if recursive && !strings.HasSuffix(key, "/") { - key += "/" + if recursive && !strings.HasSuffix(preparedKey, "/") { + preparedKey += "/" } - keyPrefix := key + keyPrefix := preparedKey // set the appropriate clientv3 options to filter the returned data set var limitOption *clientv3.OpOption @@ -590,7 +613,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption rangeEnd := clientv3.GetPrefixRangeEnd(keyPrefix) options = append(options, clientv3.WithRange(rangeEnd)) - key = continueKey + preparedKey = continueKey // If continueRV > 0, the LIST request needs a specific resource version. // continueRV==0 is invalid. @@ -657,7 +680,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption }() for { startTime := time.Now() - getResp, err = s.client.KV.Get(ctx, key, options...) + getResp, err = s.client.KV.Get(ctx, preparedKey, options...) if recursive { metrics.RecordEtcdRequestLatency("list", getTypeName(listPtr), startTime) } else { @@ -729,7 +752,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption } *limitOption = clientv3.WithLimit(limit) } - key = string(lastKey) + "\x00" + preparedKey = string(lastKey) + "\x00" if withRev == 0 { withRev = returnedRV options = append(options, clientv3.WithRev(withRev)) @@ -794,12 +817,15 @@ func growSlice(v reflect.Value, maxCapacity int, sizes ...int) { // Watch implements storage.Interface.Watch. func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { + preparedKey, err := s.prepareKey(key) + if err != nil { + return nil, err + } rev, err := s.versioner.ParseResourceVersion(opts.ResourceVersion) if err != nil { return nil, err } - key = path.Join(s.pathPrefix, key) - return s.watcher.Watch(ctx, key, int64(rev), opts.Recursive, opts.ProgressNotify, opts.Predicate) + return s.watcher.Watch(ctx, preparedKey, int64(rev), opts.Recursive, opts.ProgressNotify, opts.Predicate) } func (s *store) getState(ctx context.Context, getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) { @@ -911,6 +937,30 @@ func (s *store) validateMinimumResourceVersion(minimumResourceVersion string, ac return nil } +func (s *store) prepareKey(key string) (string, error) { + if key == ".." || + strings.HasPrefix(key, "../") || + strings.HasSuffix(key, "/..") || + strings.Contains(key, "/../") { + return "", fmt.Errorf("invalid key: %q", key) + } + if key == "." || + strings.HasPrefix(key, "./") || + strings.HasSuffix(key, "/.") || + strings.Contains(key, "/./") { + return "", fmt.Errorf("invalid key: %q", key) + } + if key == "" || key == "/" { + return "", fmt.Errorf("empty key: %q", key) + } + // We ensured that pathPrefix ends in '/' in construction, so skip any leading '/' in the key now. + startIndex := 0 + if key[0] == '/' { + startIndex = 1 + } + return s.pathPrefix + key[startIndex:], nil +} + // decode decodes value of bytes into object. It will also set the object resource version to rev. // On success, objPtr would be set to the object. func decode(codec runtime.Codec, versioner storage.Versioner, value []byte, objPtr runtime.Object, rev int64) error { diff --git a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 55e2cc0b4..2f9908bd3 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -127,13 +127,12 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan // constructing the etcd v3 client blocks and times out if etcd is not available. // retry in a loop in the background until we successfully create the client, storing the client or error encountered - lock := sync.Mutex{} - var client *clientv3.Client + lock := sync.RWMutex{} + var prober *etcd3Prober clientErr := fmt.Errorf("etcd client connection not yet established") go wait.PollUntil(time.Second, func() (bool, error) { - newClient, err := newETCD3Client(c.Transport) - + newProber, err := newETCD3Prober(c) lock.Lock() defer lock.Unlock() @@ -141,7 +140,7 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan select { case <-stopCh: if err == nil { - newClient.Close() + newProber.Close() } return true, nil default: @@ -151,7 +150,7 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan clientErr = err return false, nil } - client = newClient + prober = newProber clientErr = nil return true, nil }, stopCh) @@ -163,8 +162,8 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan lock.Lock() defer lock.Unlock() - if client != nil { - client.Close() + if prober != nil { + prober.Close() clientErr = fmt.Errorf("server is shutting down") } }() @@ -183,7 +182,7 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118 - _, err := client.Get(ctx, path.Join("/", c.Prefix, "health")) + err := prober.Probe(ctx) if err == nil { return nil } @@ -191,6 +190,49 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan }, nil } +func newETCD3Prober(c storagebackend.Config) (*etcd3Prober, error) { + client, err := newETCD3Client(c.Transport) + if err != nil { + return nil, err + } + return &etcd3Prober{ + client: client, + prefix: c.Prefix, + }, nil +} + +type etcd3Prober struct { + prefix string + + mux sync.RWMutex + client *clientv3.Client + closed bool +} + +func (p *etcd3Prober) Close() error { + p.mux.Lock() + defer p.mux.Unlock() + if !p.closed { + p.closed = true + return p.client.Close() + } + return fmt.Errorf("prober was closed") +} + +func (p *etcd3Prober) Probe(ctx context.Context) error { + p.mux.RLock() + defer p.mux.RUnlock() + if p.closed { + return fmt.Errorf("prober was closed") + } + // See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118 + _, err := p.client.Get(ctx, path.Join("/", p.prefix, "health")) + if err != nil { + return fmt.Errorf("error getting data from etcd: %w", err) + } + return nil +} + var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, error) { tlsInfo := transport.TLSInfo{ CertFile: c.CertFile, diff --git a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go index 4c8a409d6..c8cdd19b9 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go +++ b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go @@ -17,6 +17,7 @@ limitations under the License. package factory import ( + "context" "fmt" "k8s.io/apimachinery/pkg/runtime" @@ -61,3 +62,20 @@ func CreateReadyCheck(c storagebackend.Config, stopCh <-chan struct{}) (func() e return nil, fmt.Errorf("unknown storage type: %s", c.Type) } } + +func CreateProber(c storagebackend.Config) (Prober, error) { + switch c.Type { + case storagebackend.StorageTypeETCD2: + return nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type) + case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3: + return newETCD3Prober(c) + default: + return nil, fmt.Errorf("unknown storage type: %s", c.Type) + } +} + +// Prober is an interface that defines the Probe function for doing etcd readiness/liveness checks. +type Prober interface { + Probe(ctx context.Context) error + Close() error +} diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go index cd4eff7be..9468a18cb 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go @@ -59,6 +59,11 @@ import ( const timeFmt = "2006-01-02T15:04:05.999" +const ( + // priorityLevelMaxSeatsPercent is the percentage of the nominalCL used as max seats allocatable from work estimator + priorityLevelMaxSeatsPercent = float64(0.15) +) + // This file contains a simple local (to the apiserver) controller // that digests API Priority and Fairness config objects (FlowSchema // and PriorityLevelConfiguration) into the data structure that the @@ -136,6 +141,12 @@ type configController struct { // watchTracker implements the necessary WatchTracker interface. WatchTracker + // MaxSeatsTracker tracks the maximum seats that should be allocatable from the + // work estimator for a given priority level. This controller does not enforce + // any limits on max seats stored in this tracker, it is up to the work estimator + // to set lower/upper limits on max seats (currently min=1, max=10). + MaxSeatsTracker + // the most recent update attempts, ordered by increasing age. // Consumer trims to keep only the last minute's worth of entries. // The controller uses this to limit itself to at most six updates @@ -214,6 +225,7 @@ func newTestableController(config TestableConfig) *configController { flowcontrolClient: config.FlowcontrolClient, priorityLevelStates: make(map[string]*priorityLevelState), WatchTracker: NewWatchTracker(), + MaxSeatsTracker: NewMaxSeatsTracker(), } klog.V(2).Infof("NewTestableController %q with serverConcurrencyLimit=%d, requestWaitLimit=%s, name=%s, asFieldManager=%q", cfgCtlr.name, cfgCtlr.serverConcurrencyLimit, cfgCtlr.requestWaitLimit, cfgCtlr.name, cfgCtlr.asFieldManager) // Start with longish delay because conflicts will be between @@ -628,6 +640,7 @@ func (meal *cfgMeal) processOldPLsLocked() { // draining and no use is coming from another // goroutine klog.V(3).Infof("Removing undesired priority level %q (nilQueues=%v), Type=%v", plName, plState.queues == nil, plState.pl.Spec.Type) + meal.cfgCtlr.MaxSeatsTracker.ForgetPriorityLevel(plName) continue } if !plState.quiescing { @@ -675,6 +688,17 @@ func (meal *cfgMeal) finishQueueSetReconfigsLocked() { var waitLimit int if qCfg := plState.pl.Spec.Limited.LimitResponse.Queuing; qCfg != nil { waitLimit = int(qCfg.Queues * qCfg.QueueLengthLimit) + + // Max seats allocatable from work estimator is calculated as MAX(1, MIN(0.15 * nominalCL, nominalCL/handSize)). + // This is to keep max seats relative to total available concurrency with a minimum value of 1. + // 15% of nominal concurrency was chosen since it preserved the previous max seats of 10 for default priority levels + // when using apiserver's default total server concurrency of 600 (--max-requests-inflight=400, --max-mutating-requests-inflight=200). + // This ensures that clusters with relatively high inflight requests will continue to use a max seats of 10 + // while clusters with lower inflight requests will use max seats no greater than nominalCL/handSize. + // Calculated max seats can return arbitrarily high values but work estimator currently limits max seats at 10. + handSize := plState.pl.Spec.Limited.LimitResponse.Queuing.HandSize + maxSeats := uint64(math.Max(1, math.Min(math.Ceil(float64(concurrencyLimit)*priorityLevelMaxSeatsPercent), float64(int32(concurrencyLimit)/handSize)))) + meal.cfgCtlr.MaxSeatsTracker.SetMaxSeats(plName, maxSeats) } meal.maxWaitingRequests += waitLimit diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go index 71f147867..38b88bec6 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go @@ -76,6 +76,10 @@ type Interface interface { // WatchTracker provides the WatchTracker interface. WatchTracker + + // MaxSeatsTracker is invoked from the work estimator to track max seats + // that can be occupied by a request for a priority level. + MaxSeatsTracker } // This request filter implements https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1040-priority-and-fairness/README.md diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go index faa670bf1..684e76da6 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go @@ -72,11 +72,19 @@ type queueSetCompleter struct { // described in this package's doc, and a pointer to one implements // the QueueSet interface. The fields listed before the lock // should not be changed; the fields listed after the -// lock must be accessed only while holding the lock. The methods of -// this type follow the naming convention that the suffix "Locked" -// means the caller must hold the lock; for a method whose name does -// not end in "Locked" either acquires the lock or does not care about -// locking. +// lock must be accessed only while holding the lock. +// +// The methods of this type follow the naming convention that the +// suffix "Locked" means the caller must hold the lock; for a method +// whose name does not end in "Locked" either acquires the lock or +// does not care about locking. +// +// The methods of this type also follow the convention that the suffix +// "ToBoundLocked" means that the caller may have to follow up with a +// call to `boundNextDispatchLocked`. This is so for a method that +// changes what request is oldest in a queue, because that change means +// that the anti-windup hack in boundNextDispatchLocked needs to be +// applied wrt the revised oldest request in the queue. type queueSet struct { clock eventclock.Interface estimatedServiceDuration time.Duration @@ -396,7 +404,9 @@ func (req *request) wait() (bool, bool) { // TODO(aaron-prindle) add metrics for this case klog.V(5).Infof("QS(%s): Ejecting request %#+v %#+v from its queue", qs.qCfg.Name, req.descr1, req.descr2) // remove the request from the queue as it has timed out + queue := req.queue if req.removeFromQueueLocked() != nil { + defer qs.boundNextDispatchLocked(queue) qs.totRequestsWaiting-- metrics.AddReject(req.ctx, qs.qCfg.Name, req.fsName, "cancelled") metrics.AddRequestsInQueues(req.ctx, qs.qCfg.Name, req.fsName, -1) @@ -521,7 +531,7 @@ func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Conte queueIdx := qs.shuffleShardLocked(hashValue, descr1, descr2) queue := qs.queues[queueIdx] // The next step is the logic to reject requests that have been waiting too long - qs.removeTimedOutRequestsFromQueueLocked(queue, fsName) + qs.removeTimedOutRequestsFromQueueToBoundLocked(queue, fsName) // NOTE: currently timeout is only checked for each new request. This means that there can be // requests that are in the queue longer than the timeout if there are no new requests // We prefer the simplicity over the promptness, at least for now. @@ -543,7 +553,7 @@ func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Conte queueNoteFn: queueNoteFn, workEstimate: qs.completeWorkEstimate(workEstimate), } - if ok := qs.rejectOrEnqueueLocked(req); !ok { + if ok := qs.rejectOrEnqueueToBoundLocked(req); !ok { return nil } metrics.ObserveQueueLength(ctx, qs.qCfg.Name, fsName, queue.requests.Length()) @@ -583,9 +593,9 @@ func (qs *queueSet) shuffleShardLocked(hashValue uint64, descr1, descr2 interfac return bestQueueIdx } -// removeTimedOutRequestsFromQueueLocked rejects old requests that have been enqueued +// removeTimedOutRequestsFromQueueToBoundLocked rejects old requests that have been enqueued // past the requestWaitLimit -func (qs *queueSet) removeTimedOutRequestsFromQueueLocked(queue *queue, fsName string) { +func (qs *queueSet) removeTimedOutRequestsFromQueueToBoundLocked(queue *queue, fsName string) { timeoutCount := 0 now := qs.clock.Now() reqs := queue.requests @@ -616,11 +626,11 @@ func (qs *queueSet) removeTimedOutRequestsFromQueueLocked(queue *queue, fsName s } } -// rejectOrEnqueueLocked rejects or enqueues the newly arrived +// rejectOrEnqueueToBoundLocked rejects or enqueues the newly arrived // request, which has been assigned to a queue. If up against the // queue length limit and the concurrency limit then returns false. // Otherwise enqueues and returns true. -func (qs *queueSet) rejectOrEnqueueLocked(request *request) bool { +func (qs *queueSet) rejectOrEnqueueToBoundLocked(request *request) bool { queue := request.queue curQueueLength := queue.requests.Length() // rejects the newly arrived request if resource criteria not met @@ -629,12 +639,12 @@ func (qs *queueSet) rejectOrEnqueueLocked(request *request) bool { return false } - qs.enqueueLocked(request) + qs.enqueueToBoundLocked(request) return true } // enqueues a request into its queue. -func (qs *queueSet) enqueueLocked(request *request) { +func (qs *queueSet) enqueueToBoundLocked(request *request) { queue := request.queue now := qs.clock.Now() if queue.requests.Length() == 0 && queue.requestsExecuting == 0 { @@ -693,7 +703,7 @@ func (qs *queueSet) dispatchSansQueueLocked(ctx context.Context, workEstimate *f // be false when either all queues are empty or the request at the head // of the next queue cannot be dispatched. func (qs *queueSet) dispatchLocked() bool { - queue, request := qs.findDispatchQueueLocked() + queue, request := qs.findDispatchQueueToBoundLocked() if queue == nil { return false } @@ -729,6 +739,11 @@ func (qs *queueSet) dispatchLocked() bool { request.workEstimate, queue.index, queue.nextDispatchR, queue.requests.Length(), queue.requestsExecuting, queue.seatsInUse, qs.totSeatsInUse) } // When a request is dequeued for service -> qs.virtualStart += G * width + if request.totalWork() > rDecrement/100 { // A single increment should never be so big + klog.Errorf("QS(%s) at t=%s R=%v: dispatching request %#+v %#+v with implausibly high work %v from queue %d with start R %v", + qs.qCfg.Name, request.startTime.Format(nsTimeFmt), qs.currentR, request.descr1, request.descr2, + request.workEstimate, queue.index, queue.nextDispatchR) + } queue.nextDispatchR += request.totalWork() return true } @@ -756,11 +771,12 @@ func (qs *queueSet) canAccommodateSeatsLocked(seats int) bool { return true } -// findDispatchQueueLocked examines the queues in round robin order and +// findDispatchQueueToBoundLocked examines the queues in round robin order and // returns the first one of those for which the virtual finish time of // the oldest waiting request is minimal, and also returns that request. -// Returns nils if the head of the selected queue can not be dispatched now. -func (qs *queueSet) findDispatchQueueLocked() (*queue, *request) { +// Returns nils if the head of the selected queue can not be dispatched now, +// in which case the caller does not need to follow up with`qs.boundNextDispatchLocked`. +func (qs *queueSet) findDispatchQueueToBoundLocked() (*queue, *request) { minVirtualFinish := fqrequest.MaxSeatSeconds sMin := fqrequest.MaxSeatSeconds dsMin := fqrequest.MaxSeatSeconds diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/max_seats.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/max_seats.go new file mode 100644 index 000000000..18f88ab3b --- /dev/null +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/max_seats.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flowcontrol + +import ( + "sync" +) + +// MaxSeatsTracker is used to track max seats allocatable per priority level from the work estimator +type MaxSeatsTracker interface { + // GetMaxSeats returns the maximum seats a request should occupy for a given priority level. + GetMaxSeats(priorityLevelName string) uint64 + + // SetMaxSeats configures max seats for a priority level. + SetMaxSeats(priorityLevelName string, maxSeats uint64) + + // ForgetPriorityLevel removes max seats tracking for a priority level. + ForgetPriorityLevel(priorityLevelName string) +} + +type maxSeatsTracker struct { + sync.RWMutex + + maxSeats map[string]uint64 +} + +func NewMaxSeatsTracker() MaxSeatsTracker { + return &maxSeatsTracker{ + maxSeats: make(map[string]uint64), + } +} + +func (m *maxSeatsTracker) GetMaxSeats(plName string) uint64 { + m.RLock() + defer m.RUnlock() + + return m.maxSeats[plName] +} + +func (m *maxSeatsTracker) SetMaxSeats(plName string, maxSeats uint64) { + m.Lock() + defer m.Unlock() + + m.maxSeats[plName] = maxSeats +} + +func (m *maxSeatsTracker) ForgetPriorityLevel(plName string) { + m.Lock() + defer m.Unlock() + + delete(m.maxSeats, plName) +} diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/config.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/config.go index b6db19209..c51435b15 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/config.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/config.go @@ -24,7 +24,7 @@ import ( const ( minimumSeats = 1 - maximumSeats = 10 + maximumSeatsLimit = 10 objectsPerSeat = 100.0 watchesPerSeat = 10.0 enableMutatingWorkEstimator = true @@ -39,12 +39,13 @@ type WorkEstimatorConfig struct { // MinimumSeats is the minimum number of seats a request must occupy. MinimumSeats uint64 `json:"minimumSeats,omitempty"` - // MaximumSeats is the maximum number of seats a request can occupy + + // MaximumSeatsLimit is an upper limit on the max seats a request can occupy. // // NOTE: work_estimate_seats_samples metric uses the value of maximumSeats // as the upper bound, so when we change maximumSeats we should also // update the buckets of the metric. - MaximumSeats uint64 `json:"maximumSeats,omitempty"` + MaximumSeatsLimit uint64 `json:"maximumSeatsLimit,omitempty"` } // ListWorkEstimatorConfig holds work estimator parameters related to list requests. @@ -66,7 +67,7 @@ type MutatingWorkEstimatorConfig struct { func DefaultWorkEstimatorConfig() *WorkEstimatorConfig { return &WorkEstimatorConfig{ MinimumSeats: minimumSeats, - MaximumSeats: maximumSeats, + MaximumSeatsLimit: maximumSeatsLimit, ListWorkEstimatorConfig: defaultListWorkEstimatorConfig(), MutatingWorkEstimatorConfig: defaultMutatingWorkEstimatorConfig(), } diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/list_work_estimator.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/list_work_estimator.go index 4771fcdec..e49e08109 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/list_work_estimator.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/list_work_estimator.go @@ -29,10 +29,11 @@ import ( "k8s.io/klog/v2" ) -func newListWorkEstimator(countFn objectCountGetterFunc, config *WorkEstimatorConfig) WorkEstimatorFunc { +func newListWorkEstimator(countFn objectCountGetterFunc, config *WorkEstimatorConfig, maxSeatsFn maxSeatsFunc) WorkEstimatorFunc { estimator := &listWorkEstimator{ config: config, countGetterFn: countFn, + maxSeatsFn: maxSeatsFn, } return estimator.estimate } @@ -40,14 +41,30 @@ func newListWorkEstimator(countFn objectCountGetterFunc, config *WorkEstimatorCo type listWorkEstimator struct { config *WorkEstimatorConfig countGetterFn objectCountGetterFunc + maxSeatsFn maxSeatsFunc } func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelName string) WorkEstimate { + minSeats := e.config.MinimumSeats + maxSeats := e.maxSeatsFn(priorityLevelName) + if maxSeats == 0 || maxSeats > e.config.MaximumSeatsLimit { + maxSeats = e.config.MaximumSeatsLimit + } + requestInfo, ok := apirequest.RequestInfoFrom(r.Context()) if !ok { // no RequestInfo should never happen, but to be on the safe side // let's return maximumSeats - return WorkEstimate{InitialSeats: e.config.MaximumSeats} + return WorkEstimate{InitialSeats: maxSeats} + } + + if requestInfo.Name != "" { + // Requests with metadata.name specified are usually executed as get + // requests in storage layer so their width should be 1. + // Example of such list requests: + // /apis/certificates.k8s.io/v1/certificatesigningrequests?fieldSelector=metadata.name%3Dcsr-xxs4m + // /api/v1/namespaces/test/configmaps?fieldSelector=metadata.name%3Dbig-deployment-1&limit=500&resourceVersion=0 + return WorkEstimate{InitialSeats: minSeats} } query := r.URL.Query() @@ -57,7 +74,7 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe // This request is destined to fail in the validation layer, // return maximumSeats for this request to be consistent. - return WorkEstimate{InitialSeats: e.config.MaximumSeats} + return WorkEstimate{InitialSeats: maxSeats} } isListFromCache := !shouldListFromStorage(query, &listOptions) @@ -68,7 +85,7 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe // be conservative here and allocate maximum seats to this list request. // NOTE: if a CRD is removed, its count will go stale first and then the // pruner will eventually remove the CRD from the cache. - return WorkEstimate{InitialSeats: e.config.MaximumSeats} + return WorkEstimate{InitialSeats: maxSeats} case err == ObjectCountNotFoundErr: // there are multiple scenarios in which we can see this error: // a. the type is truly unknown, a typo on the caller's part. @@ -82,12 +99,12 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe // when aggregated API calls are overestimated, we allocate the minimum // possible seats (see #109106 as an example when being more conservative // led to problems). - return WorkEstimate{InitialSeats: e.config.MinimumSeats} + return WorkEstimate{InitialSeats: minSeats} case err != nil: // we should never be here since Get returns either ObjectCountStaleErr or // ObjectCountNotFoundErr, return maximumSeats to be on the safe side. klog.ErrorS(err, "Unexpected error from object count tracker") - return WorkEstimate{InitialSeats: e.config.MaximumSeats} + return WorkEstimate{InitialSeats: maxSeats} } limit := numStored @@ -116,11 +133,11 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe seats := uint64(math.Ceil(float64(estimatedObjectsToBeProcessed) / e.config.ObjectsPerSeat)) // make sure we never return a seat of zero - if seats < e.config.MinimumSeats { - seats = e.config.MinimumSeats + if seats < minSeats { + seats = minSeats } - if seats > e.config.MaximumSeats { - seats = e.config.MaximumSeats + if seats > maxSeats { + seats = maxSeats } return WorkEstimate{InitialSeats: seats} } diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/mutating_work_estimator.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/mutating_work_estimator.go index 990aa6324..aa2807fd1 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/mutating_work_estimator.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/mutating_work_estimator.go @@ -25,25 +25,33 @@ import ( "k8s.io/apiserver/pkg/util/flowcontrol/metrics" ) -func newMutatingWorkEstimator(countFn watchCountGetterFunc, config *WorkEstimatorConfig) WorkEstimatorFunc { +func newMutatingWorkEstimator(countFn watchCountGetterFunc, config *WorkEstimatorConfig, maxSeatsFn maxSeatsFunc) WorkEstimatorFunc { estimator := &mutatingWorkEstimator{ - config: config, - countFn: countFn, + config: config, + countFn: countFn, + maxSeatsFn: maxSeatsFn, } return estimator.estimate } type mutatingWorkEstimator struct { - config *WorkEstimatorConfig - countFn watchCountGetterFunc + config *WorkEstimatorConfig + countFn watchCountGetterFunc + maxSeatsFn maxSeatsFunc } func (e *mutatingWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelName string) WorkEstimate { + minSeats := e.config.MinimumSeats + maxSeats := e.maxSeatsFn(priorityLevelName) + if maxSeats == 0 || maxSeats > e.config.MaximumSeatsLimit { + maxSeats = e.config.MaximumSeatsLimit + } + // TODO(wojtekt): Remove once we tune the algorithm to not fail // scalability tests. if !e.config.Enabled { return WorkEstimate{ - InitialSeats: 1, + InitialSeats: minSeats, } } @@ -52,11 +60,12 @@ func (e *mutatingWorkEstimator) estimate(r *http.Request, flowSchemaName, priori // no RequestInfo should never happen, but to be on the safe side // let's return a large value. return WorkEstimate{ - InitialSeats: 1, - FinalSeats: e.config.MaximumSeats, + InitialSeats: minSeats, + FinalSeats: maxSeats, AdditionalLatency: e.config.eventAdditionalDuration(), } } + watchCount := e.countFn(requestInfo) metrics.ObserveWatchCount(r.Context(), priorityLevelName, flowSchemaName, watchCount) @@ -117,8 +126,8 @@ func (e *mutatingWorkEstimator) estimate(r *http.Request, flowSchemaName, priori // // TODO: Confirm that the current cap of maximumSeats allow us to // achieve the above. - if finalSeats > e.config.MaximumSeats { - finalSeats = e.config.MaximumSeats + if finalSeats > maxSeats { + finalSeats = maxSeats } additionalLatency = finalWork.DurationPerSeat(float64(finalSeats)) } diff --git a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/width.go b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/width.go index 86f042584..e55cc714b 100644 --- a/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/width.go +++ b/vendor/k8s.io/apiserver/pkg/util/flowcontrol/request/width.go @@ -61,15 +61,19 @@ type objectCountGetterFunc func(string) (int64, error) // number of watchers potentially interested in a given request. type watchCountGetterFunc func(*apirequest.RequestInfo) int +// MaxSeatsFunc represents a function that returns the maximum seats +// allowed for the work estimator for a given priority level. +type maxSeatsFunc func(priorityLevelName string) uint64 + // NewWorkEstimator estimates the work that will be done by a given request, // if no WorkEstimatorFunc matches the given request then the default // work estimate of 1 seat is allocated to the request. -func NewWorkEstimator(objectCountFn objectCountGetterFunc, watchCountFn watchCountGetterFunc, config *WorkEstimatorConfig) WorkEstimatorFunc { +func NewWorkEstimator(objectCountFn objectCountGetterFunc, watchCountFn watchCountGetterFunc, config *WorkEstimatorConfig, maxSeatsFn maxSeatsFunc) WorkEstimatorFunc { estimator := &workEstimator{ minimumSeats: config.MinimumSeats, - maximumSeats: config.MaximumSeats, - listWorkEstimator: newListWorkEstimator(objectCountFn, config), - mutatingWorkEstimator: newMutatingWorkEstimator(watchCountFn, config), + maximumSeatsLimit: config.MaximumSeatsLimit, + listWorkEstimator: newListWorkEstimator(objectCountFn, config, maxSeatsFn), + mutatingWorkEstimator: newMutatingWorkEstimator(watchCountFn, config, maxSeatsFn), } return estimator.estimate } @@ -86,8 +90,8 @@ func (e WorkEstimatorFunc) EstimateWork(r *http.Request, flowSchemaName, priorit type workEstimator struct { // the minimum number of seats a request must occupy minimumSeats uint64 - // the maximum number of seats a request can occupy - maximumSeats uint64 + // the default maximum number of seats a request can occupy + maximumSeatsLimit uint64 // listWorkEstimator estimates work for list request(s) listWorkEstimator WorkEstimatorFunc // mutatingWorkEstimator calculates the width of mutating request(s) @@ -99,7 +103,7 @@ func (e *workEstimator) estimate(r *http.Request, flowSchemaName, priorityLevelN if !ok { klog.ErrorS(fmt.Errorf("no RequestInfo found in context"), "Failed to estimate work for the request", "URI", r.RequestURI) // no RequestInfo should never happen, but to be on the safe side let's return maximumSeats - return WorkEstimate{InitialSeats: e.maximumSeats} + return WorkEstimate{InitialSeats: e.maximumSeatsLimit} } switch requestInfo.Verb { diff --git a/vendor/k8s.io/client-go/rest/request.go b/vendor/k8s.io/client-go/rest/request.go index acf311361..a1f0a591d 100644 --- a/vendor/k8s.io/client-go/rest/request.go +++ b/vendor/k8s.io/client-go/rest/request.go @@ -34,6 +34,7 @@ import ( "time" "golang.org/x/net/http2" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -116,8 +117,11 @@ type Request struct { subresource string // output - err error - body io.Reader + err error + + // only one of body / bodyBytes may be set. requests using body are not retriable. + body io.Reader + bodyBytes []byte retryFn requestRetryFunc } @@ -443,12 +447,15 @@ func (r *Request) Body(obj interface{}) *Request { return r } glogBody("Request Body", data) - r.body = bytes.NewReader(data) + r.body = nil + r.bodyBytes = data case []byte: glogBody("Request Body", t) - r.body = bytes.NewReader(t) + r.body = nil + r.bodyBytes = t case io.Reader: r.body = t + r.bodyBytes = nil case runtime.Object: // callers may pass typed interface pointers, therefore we must check nil with reflection if reflect.ValueOf(t).IsNil() { @@ -465,7 +472,8 @@ func (r *Request) Body(obj interface{}) *Request { return r } glogBody("Request Body", data) - r.body = bytes.NewReader(data) + r.body = nil + r.bodyBytes = data r.SetHeader("Content-Type", r.c.content.ContentType) default: r.err = fmt.Errorf("unknown type used for body: %+v", obj) @@ -825,9 +833,7 @@ func (r *Request) Stream(ctx context.Context) (io.ReadCloser, error) { if err != nil { return nil, err } - if r.body != nil { - req.Body = ioutil.NopCloser(r.body) - } + resp, err := client.Do(req) updateURLMetrics(ctx, r, resp, err) retry.After(ctx, r, resp, err) @@ -889,8 +895,20 @@ func (r *Request) requestPreflightCheck() error { } func (r *Request) newHTTPRequest(ctx context.Context) (*http.Request, error) { + var body io.Reader + switch { + case r.body != nil && r.bodyBytes != nil: + return nil, fmt.Errorf("cannot set both body and bodyBytes") + case r.body != nil: + body = r.body + case r.bodyBytes != nil: + // Create a new reader specifically for this request. + // Giving each request a dedicated reader allows retries to avoid races resetting the request body. + body = bytes.NewReader(r.bodyBytes) + } + url := r.URL().String() - req, err := http.NewRequest(r.verb, url, r.body) + req, err := http.NewRequest(r.verb, url, body) if err != nil { return nil, err } diff --git a/vendor/k8s.io/client-go/rest/with_retry.go b/vendor/k8s.io/client-go/rest/with_retry.go index bdcc6f3a9..ab2694754 100644 --- a/vendor/k8s.io/client-go/rest/with_retry.go +++ b/vendor/k8s.io/client-go/rest/with_retry.go @@ -154,6 +154,11 @@ func (r *withRetry) IsNextRetry(ctx context.Context, restReq *Request, httpReq * return false } + if restReq.body != nil { + // we have an opaque reader, we can't safely reset it + return false + } + r.attempts++ r.retryAfter = &RetryAfter{Attempt: r.attempts} if r.attempts > r.maxRetries { @@ -210,18 +215,6 @@ func (r *withRetry) Before(ctx context.Context, request *Request) error { return nil } - // At this point we've made atleast one attempt, post which the response - // body should have been fully read and closed in order for it to be safe - // to reset the request body before we reconnect, in order for us to reuse - // the same TCP connection. - if seeker, ok := request.body.(io.Seeker); ok && request.body != nil { - if _, err := seeker.Seek(0, io.SeekStart); err != nil { - err = fmt.Errorf("failed to reset the request body while retrying a request: %v", err) - r.trackPreviousError(err) - return err - } - } - // if we are here, we have made attempt(s) at least once before. if request.backoff != nil { delay := request.backoff.CalculateBackoff(url) diff --git a/vendor/k8s.io/client-go/tools/cache/controller.go b/vendor/k8s.io/client-go/tools/cache/controller.go index 0762da3be..96005ff58 100644 --- a/vendor/k8s.io/client-go/tools/cache/controller.go +++ b/vendor/k8s.io/client-go/tools/cache/controller.go @@ -353,17 +353,6 @@ func NewIndexerInformer( return clientState, newInformer(lw, objType, resyncPeriod, h, clientState, nil) } -// TransformFunc allows for transforming an object before it will be processed -// and put into the controller cache and before the corresponding handlers will -// be called on it. -// TransformFunc (similarly to ResourceEventHandler functions) should be able -// to correctly handle the tombstone of type cache.DeletedFinalStateUnknown -// -// The most common usage pattern is to clean-up some parts of the object to -// reduce component memory usage if a given component doesn't care about them. -// given controller doesn't care for them -type TransformFunc func(interface{}) (interface{}, error) - // NewTransformingInformer returns a Store and a controller for populating // the store while also providing event notifications. You should only used // the returned Store for Get/List operations; Add/Modify/Deletes will cause @@ -411,19 +400,11 @@ func processDeltas( // Object which receives event notifications from the given deltas handler ResourceEventHandler, clientState Store, - transformer TransformFunc, deltas Deltas, ) error { // from oldest to newest for _, d := range deltas { obj := d.Object - if transformer != nil { - var err error - obj, err = transformer(obj) - if err != nil { - return err - } - } switch d.Type { case Sync, Replaced, Added, Updated: @@ -475,6 +456,7 @@ func newInformer( fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{ KnownObjects: clientState, EmitDeltaTypeReplaced: true, + Transformer: transformer, }) cfg := &Config{ @@ -486,7 +468,7 @@ func newInformer( Process: func(obj interface{}) error { if deltas, ok := obj.(Deltas); ok { - return processDeltas(h, clientState, transformer, deltas) + return processDeltas(h, clientState, deltas) } return errors.New("object given as Process argument is not Deltas") }, diff --git a/vendor/k8s.io/client-go/tools/cache/delta_fifo.go b/vendor/k8s.io/client-go/tools/cache/delta_fifo.go index 0c13a41f0..84f3ab9ca 100644 --- a/vendor/k8s.io/client-go/tools/cache/delta_fifo.go +++ b/vendor/k8s.io/client-go/tools/cache/delta_fifo.go @@ -51,6 +51,10 @@ type DeltaFIFOOptions struct { // When true, `Replaced` events will be sent for items passed to a Replace() call. // When false, `Sync` events will be sent instead. EmitDeltaTypeReplaced bool + + // If set, will be called for objects before enqueueing them. Please + // see the comment on TransformFunc for details. + Transformer TransformFunc } // DeltaFIFO is like FIFO, but differs in two ways. One is that the @@ -129,8 +133,32 @@ type DeltaFIFO struct { // emitDeltaTypeReplaced is whether to emit the Replaced or Sync // DeltaType when Replace() is called (to preserve backwards compat). emitDeltaTypeReplaced bool + + // Called with every object if non-nil. + transformer TransformFunc } +// TransformFunc allows for transforming an object before it will be processed. +// TransformFunc (similarly to ResourceEventHandler functions) should be able +// to correctly handle the tombstone of type cache.DeletedFinalStateUnknown. +// +// New in v1.27: In such cases, the contained object will already have gone +// through the transform object separately (when it was added / updated prior +// to the delete), so the TransformFunc can likely safely ignore such objects +// (i.e., just return the input object). +// +// The most common usage pattern is to clean-up some parts of the object to +// reduce component memory usage if a given component doesn't care about them. +// +// New in v1.27: unless the object is a DeletedFinalStateUnknown, TransformFunc +// sees the object before any other actor, and it is now safe to mutate the +// object in place instead of making a copy. +// +// Note that TransformFunc is called while inserting objects into the +// notification queue and is therefore extremely performance sensitive; please +// do not do anything that will take a long time. +type TransformFunc func(interface{}) (interface{}, error) + // DeltaType is the type of a change (addition, deletion, etc) type DeltaType string @@ -227,6 +255,7 @@ func NewDeltaFIFOWithOptions(opts DeltaFIFOOptions) *DeltaFIFO { knownObjects: opts.KnownObjects, emitDeltaTypeReplaced: opts.EmitDeltaTypeReplaced, + transformer: opts.Transformer, } f.cond.L = &f.lock return f @@ -411,6 +440,21 @@ func (f *DeltaFIFO) queueActionLocked(actionType DeltaType, obj interface{}) err if err != nil { return KeyError{obj, err} } + + // Every object comes through this code path once, so this is a good + // place to call the transform func. If obj is a + // DeletedFinalStateUnknown tombstone, then the containted inner object + // will already have gone through the transformer, but we document that + // this can happen. In cases involving Replace(), such an object can + // come through multiple times. + if f.transformer != nil { + var err error + obj, err = f.transformer(obj) + if err != nil { + return err + } + } + oldDeltas := f.items[id] newDeltas := append(oldDeltas, Delta{actionType, obj}) newDeltas = dedupDeltas(newDeltas) @@ -566,12 +610,11 @@ func (f *DeltaFIFO) Pop(process PopProcessFunc) (interface{}, error) { // using the Sync or Replace DeltaType and then (2) it does some deletions. // In particular: for every pre-existing key K that is not the key of // an object in `list` there is the effect of -// `Delete(DeletedFinalStateUnknown{K, O})` where O is current object -// of K. If `f.knownObjects == nil` then the pre-existing keys are -// those in `f.items` and the current object of K is the `.Newest()` -// of the Deltas associated with K. Otherwise the pre-existing keys -// are those listed by `f.knownObjects` and the current object of K is -// what `f.knownObjects.GetByKey(K)` returns. +// `Delete(DeletedFinalStateUnknown{K, O})` where O is the latest known +// object of K. The pre-existing keys are those in the union set of the keys in +// `f.items` and `f.knownObjects` (if not nil). The last known object for key K is +// the one present in the last delta in `f.items`. If there is no delta for K +// in `f.items`, it is the object in `f.knownObjects` func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { f.lock.Lock() defer f.lock.Unlock() @@ -595,51 +638,23 @@ func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { } } - if f.knownObjects == nil { - // Do deletion detection against our own list. - queuedDeletions := 0 - for k, oldItem := range f.items { - if keys.Has(k) { - continue - } - // Delete pre-existing items not in the new list. - // This could happen if watch deletion event was missed while - // disconnected from apiserver. - var deletedObj interface{} - if n := oldItem.Newest(); n != nil { - deletedObj = n.Object - } - queuedDeletions++ - if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { - return err - } - } - - if !f.populated { - f.populated = true - // While there shouldn't be any queued deletions in the initial - // population of the queue, it's better to be on the safe side. - f.initialPopulationCount = keys.Len() + queuedDeletions - } - - return nil - } - - // Detect deletions not already in the queue. - knownKeys := f.knownObjects.ListKeys() + // Do deletion detection against objects in the queue queuedDeletions := 0 - for _, k := range knownKeys { + for k, oldItem := range f.items { if keys.Has(k) { continue } - - deletedObj, exists, err := f.knownObjects.GetByKey(k) - if err != nil { - deletedObj = nil - klog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) - } else if !exists { - deletedObj = nil - klog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) + // Delete pre-existing items not in the new list. + // This could happen if watch deletion event was missed while + // disconnected from apiserver. + var deletedObj interface{} + if n := oldItem.Newest(); n != nil { + deletedObj = n.Object + + // if the previous object is a DeletedFinalStateUnknown, we have to extract the actual Object + if d, ok := deletedObj.(DeletedFinalStateUnknown); ok { + deletedObj = d.Obj + } } queuedDeletions++ if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { @@ -647,6 +662,32 @@ func (f *DeltaFIFO) Replace(list []interface{}, _ string) error { } } + if f.knownObjects != nil { + // Detect deletions for objects not present in the queue, but present in KnownObjects + knownKeys := f.knownObjects.ListKeys() + for _, k := range knownKeys { + if keys.Has(k) { + continue + } + if len(f.items[k]) > 0 { + continue + } + + deletedObj, exists, err := f.knownObjects.GetByKey(k) + if err != nil { + deletedObj = nil + klog.Errorf("Unexpected error %v during lookup of key %v, placing DeleteFinalStateUnknown marker without object", err, k) + } else if !exists { + deletedObj = nil + klog.Infof("Key %v does not exist in known objects store, placing DeleteFinalStateUnknown marker without object", k) + } + queuedDeletions++ + if err := f.queueActionLocked(Deleted, DeletedFinalStateUnknown{k, deletedObj}); err != nil { + return err + } + } + } + if !f.populated { f.populated = true f.initialPopulationCount = keys.Len() + queuedDeletions diff --git a/vendor/k8s.io/client-go/tools/cache/shared_informer.go b/vendor/k8s.io/client-go/tools/cache/shared_informer.go index 9f42782d1..35ebd396c 100644 --- a/vendor/k8s.io/client-go/tools/cache/shared_informer.go +++ b/vendor/k8s.io/client-go/tools/cache/shared_informer.go @@ -190,10 +190,7 @@ type SharedInformer interface { // // Must be set before starting the informer. // - // Note: Since the object given to the handler may be already shared with - // other goroutines, it is advisable to copy the object being - // transform before mutating it at all and returning the copy to prevent - // data races. + // Please see the comment on TransformFunc for more details. SetTransform(handler TransformFunc) error } @@ -404,6 +401,7 @@ func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) { fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{ KnownObjects: s.indexer, EmitDeltaTypeReplaced: true, + Transformer: s.transform, }) cfg := &Config{ @@ -568,7 +566,7 @@ func (s *sharedIndexInformer) HandleDeltas(obj interface{}) error { defer s.blockDeltas.Unlock() if deltas, ok := obj.(Deltas); ok { - return processDeltas(s, s.indexer, s.transform, deltas) + return processDeltas(s, s.indexer, deltas) } return errors.New("object given as Process argument is not Deltas") } diff --git a/vendor/k8s.io/client-go/tools/events/event_broadcaster.go b/vendor/k8s.io/client-go/tools/events/event_broadcaster.go index dd7e0aa12..7a9435637 100644 --- a/vendor/k8s.io/client-go/tools/events/event_broadcaster.go +++ b/vendor/k8s.io/client-go/tools/events/event_broadcaster.go @@ -181,22 +181,24 @@ func (e *eventBroadcasterImpl) recordToSink(event *eventsv1.Event, clock clock.C return nil } isomorphicEvent.Series = &eventsv1.EventSeries{ - Count: 1, + Count: 2, LastObservedTime: metav1.MicroTime{Time: clock.Now()}, } - return isomorphicEvent + // Make a copy of the Event to make sure that recording it + // doesn't mess with the object stored in cache. + return isomorphicEvent.DeepCopy() } e.eventCache[eventKey] = eventCopy - return eventCopy + // Make a copy of the Event to make sure that recording it doesn't + // mess with the object stored in cache. + return eventCopy.DeepCopy() }() if evToRecord != nil { - recordedEvent := e.attemptRecording(evToRecord) - if recordedEvent != nil { - recordedEventKey := getKey(recordedEvent) - e.mu.Lock() - defer e.mu.Unlock() - e.eventCache[recordedEventKey] = recordedEvent - } + // TODO: Add a metric counting the number of recording attempts + e.attemptRecording(evToRecord) + // We don't want the new recorded Event to be reflected in the + // client's cache because server-side mutations could mess with the + // aggregation mechanism used by the client. } }() } @@ -248,6 +250,14 @@ func recordEvent(sink EventSink, event *eventsv1.Event) (*eventsv1.Event, bool) return nil, false case *errors.StatusError: if errors.IsAlreadyExists(err) { + // If we tried to create an Event from an EventSerie, it means that + // the original Patch request failed because the Event we were + // trying to patch didn't exist. If the creation failed because the + // Event now exists, it is safe to retry. This occurs when a new + // Event is emitted twice in a very short period of time. + if isEventSeries { + return nil, true + } klog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err) } else { klog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err) diff --git a/vendor/k8s.io/client-go/transport/cache.go b/vendor/k8s.io/client-go/transport/cache.go index b4f8dab0c..477c22cde 100644 --- a/vendor/k8s.io/client-go/transport/cache.go +++ b/vendor/k8s.io/client-go/transport/cache.go @@ -109,7 +109,7 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) { // If we use are reloading files, we need to handle certificate rotation properly // TODO(jackkleeman): We can also add rotation here when config.HasCertCallback() is true - if config.TLS.ReloadTLSFiles { + if config.TLS.ReloadTLSFiles && tlsConfig != nil && tlsConfig.GetClientCertificate != nil { dynamicCertDialer := certRotatingDialer(tlsConfig.GetClientCertificate, dial) tlsConfig.GetClientCertificate = dynamicCertDialer.GetClientCertificate dial = dynamicCertDialer.connDialer.DialContext diff --git a/vendor/k8s.io/client-go/util/cert/cert.go b/vendor/k8s.io/client-go/util/cert/cert.go index 75143ec07..447c3013c 100644 --- a/vendor/k8s.io/client-go/util/cert/cert.go +++ b/vendor/k8s.io/client-go/util/cert/cert.go @@ -26,6 +26,7 @@ import ( "encoding/pem" "fmt" "io/ioutil" + "math" "math/big" "net" "path/filepath" @@ -44,6 +45,7 @@ type Config struct { Organization []string AltNames AltNames Usages []x509.ExtKeyUsage + NotBefore time.Time } // AltNames contains the domain names and IP addresses that will be added @@ -57,14 +59,24 @@ type AltNames struct { // NewSelfSignedCACert creates a CA certificate func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) { now := time.Now() + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) + notBefore := now.UTC() + if !cfg.NotBefore.IsZero() { + notBefore = cfg.NotBefore.UTC() + } tmpl := x509.Certificate{ - SerialNumber: new(big.Int).SetInt64(0), + SerialNumber: serial, Subject: pkix.Name{ CommonName: cfg.CommonName, Organization: cfg.Organization, }, DNSNames: []string{cfg.CommonName}, - NotBefore: now.UTC(), + NotBefore: notBefore, NotAfter: now.Add(duration365d * 10).UTC(), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, @@ -116,9 +128,14 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a if err != nil { return nil, nil, err } - + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) caTemplate := x509.Certificate{ - SerialNumber: big.NewInt(1), + SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()), }, @@ -144,9 +161,14 @@ func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, a if err != nil { return nil, nil, err } - + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err = cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) template := x509.Certificate{ - SerialNumber: big.NewInt(2), + SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()), }, diff --git a/vendor/k8s.io/component-base/metrics/metric.go b/vendor/k8s.io/component-base/metrics/metric.go index 2980a9723..e0a58b9b8 100644 --- a/vendor/k8s.io/component-base/metrics/metric.go +++ b/vendor/k8s.io/component-base/metrics/metric.go @@ -207,7 +207,6 @@ var noopCounterVec = &prometheus.CounterVec{} var noopHistogramVec = &prometheus.HistogramVec{} var noopTimingHistogramVec = &promext.TimingHistogramVec{} var noopGaugeVec = &prometheus.GaugeVec{} -var noopObserverVec = &noopObserverVector{} // just use a convenience struct for all the no-ops var noop = &noopMetric{} @@ -226,22 +225,3 @@ func (noopMetric) Desc() *prometheus.Desc { return nil } func (noopMetric) Write(*dto.Metric) error { return nil } func (noopMetric) Describe(chan<- *prometheus.Desc) {} func (noopMetric) Collect(chan<- prometheus.Metric) {} - -type noopObserverVector struct{} - -func (noopObserverVector) GetMetricWith(prometheus.Labels) (prometheus.Observer, error) { - return noop, nil -} -func (noopObserverVector) GetMetricWithLabelValues(...string) (prometheus.Observer, error) { - return noop, nil -} -func (noopObserverVector) With(prometheus.Labels) prometheus.Observer { return noop } -func (noopObserverVector) WithLabelValues(...string) prometheus.Observer { return noop } -func (noopObserverVector) CurryWith(prometheus.Labels) (prometheus.ObserverVec, error) { - return noopObserverVec, nil -} -func (noopObserverVector) MustCurryWith(prometheus.Labels) prometheus.ObserverVec { - return noopObserverVec -} -func (noopObserverVector) Describe(chan<- *prometheus.Desc) {} -func (noopObserverVector) Collect(chan<- prometheus.Metric) {} diff --git a/vendor/modules.txt b/vendor/modules.txt index 98870babf..7b80889fd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -140,7 +140,7 @@ github.com/google/go-cmp/cmp/internal/value # github.com/google/gofuzz v1.1.0 ## explicit; go 1.12 github.com/google/gofuzz -# github.com/google/uuid v1.2.0 +# github.com/google/uuid v1.3.0 ## explicit github.com/google/uuid # github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -555,7 +555,7 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# k8s.io/api v0.25.5 +# k8s.io/api v0.25.13 ## explicit; go 1.19 k8s.io/api/admission/v1 k8s.io/api/admission/v1beta1 @@ -605,12 +605,12 @@ k8s.io/api/scheduling/v1beta1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 -# k8s.io/apiextensions-apiserver v0.25.0 +# k8s.io/apiextensions-apiserver v0.25.13 ## explicit; go 1.19 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1 -# k8s.io/apimachinery v0.25.5 +# k8s.io/apimachinery v0.25.13 ## explicit; go 1.19 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -666,7 +666,7 @@ k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.25.0 +# k8s.io/apiserver v0.25.13 ## explicit; go 1.19 k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission/configuration @@ -798,7 +798,7 @@ k8s.io/apiserver/plugin/pkg/audit/truncate k8s.io/apiserver/plugin/pkg/audit/webhook k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook -# k8s.io/client-go v0.25.5 +# k8s.io/client-go v0.25.13 ## explicit; go 1.19 k8s.io/client-go/applyconfigurations/admissionregistration/v1 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 @@ -1083,7 +1083,7 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/jsonpath k8s.io/client-go/util/keyutil k8s.io/client-go/util/workqueue -# k8s.io/code-generator v0.25.0 +# k8s.io/code-generator v0.25.13 ## explicit; go 1.19 k8s.io/code-generator k8s.io/code-generator/cmd/client-gen @@ -1119,7 +1119,7 @@ k8s.io/code-generator/cmd/set-gen k8s.io/code-generator/pkg/namer k8s.io/code-generator/pkg/util k8s.io/code-generator/third_party/forked/golang/reflect -# k8s.io/component-base v0.25.5 +# k8s.io/component-base v0.25.13 ## explicit; go 1.19 k8s.io/component-base/cli/flag k8s.io/component-base/config @@ -1199,9 +1199,11 @@ k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/strings/slices k8s.io/utils/trace -# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 +# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 ## explicit; go 1.17 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client +sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics +sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client # sigs.k8s.io/controller-runtime v0.13.1 ## explicit; go 1.17 diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go index c191649a8..d9c151e98 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go @@ -24,10 +24,14 @@ import ( "math/rand" "net" "sync" + "sync/atomic" "time" "google.golang.org/grpc" "k8s.io/klog/v2" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics" + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -36,10 +40,13 @@ type Tunnel interface { // Dial connects to the address on the named network, similar to // what net.Dial does. The only supported protocol is tcp. DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) + // Done returns a channel that is closed when the tunnel is no longer serving any connections, + // and can no longer be used. + Done() <-chan struct{} } type dialResult struct { - err string + err *dialFailure connid int64 } @@ -50,17 +57,88 @@ type pendingDial struct { cancelCh <-chan struct{} } +// TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+ +type pendingDialManager struct { + pendingDials map[int64]pendingDial + mutex sync.RWMutex +} + +func (p *pendingDialManager) add(dialID int64, pd pendingDial) { + p.mutex.Lock() + defer p.mutex.Unlock() + p.pendingDials[dialID] = pd +} + +func (p *pendingDialManager) remove(dialID int64) { + p.mutex.Lock() + defer p.mutex.Unlock() + delete(p.pendingDials, dialID) +} + +func (p *pendingDialManager) get(dialID int64) (pendingDial, bool) { + p.mutex.RLock() + defer p.mutex.RUnlock() + pd, ok := p.pendingDials[dialID] + return pd, ok +} + +// TODO: Replace with a generic implementation once it is safe to assume the client is built with go1.18+ +type connectionManager struct { + conns map[int64]*conn + mutex sync.RWMutex +} + +func (cm *connectionManager) add(connID int64, c *conn) { + cm.mutex.Lock() + defer cm.mutex.Unlock() + cm.conns[connID] = c +} + +func (cm *connectionManager) remove(connID int64) { + cm.mutex.Lock() + defer cm.mutex.Unlock() + delete(cm.conns, connID) +} + +func (cm *connectionManager) get(connID int64) (*conn, bool) { + cm.mutex.RLock() + defer cm.mutex.RUnlock() + c, ok := cm.conns[connID] + return c, ok +} + +func (cm *connectionManager) closeAll() { + cm.mutex.Lock() + defer cm.mutex.Unlock() + for _, conn := range cm.conns { + close(conn.readCh) + } +} + // grpcTunnel implements Tunnel type grpcTunnel struct { - stream client.ProxyService_ProxyClient - pendingDial map[int64]pendingDial - conns map[int64]*conn - pendingDialLock sync.RWMutex - connsLock sync.RWMutex + stream client.ProxyService_ProxyClient + sendLock sync.Mutex + recvLock sync.Mutex + clientConn clientConn + pendingDial pendingDialManager + conns connectionManager // The tunnel will be closed if the caller fails to read via conn.Read() // more than readTimeoutSeconds after a packet has been received. readTimeoutSeconds int + + // The done channel is closed after the tunnel has cleaned up all connections and is no longer + // serving. + done chan struct{} + + // closing is an atomic bool represented as a 0 or 1, and set to true when the tunnel is being closed. + // closing should only be accessed through atomic methods. + // TODO: switch this to an atomic.Bool once the client is exclusively buit with go1.19+ + closing uint32 + + // Stores the current metrics.ClientConnectionStatus + prevStatus atomic.Value } type clientConn interface { @@ -69,6 +147,11 @@ type clientConn interface { var _ clientConn = &grpc.ClientConn{} +var ( + // Expose metrics for client to register. + Metrics = metrics.Metrics +) + // CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a // gRPC based proxy service. // Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated @@ -99,119 +182,201 @@ func CreateSingleUseGrpcTunnelWithContext(createCtx, tunnelCtx context.Context, return nil, err } - tunnel := &grpcTunnel{ + tunnel := newUnstartedTunnel(stream, c) + + go tunnel.serve(tunnelCtx) + + return tunnel, nil +} + +func newUnstartedTunnel(stream client.ProxyService_ProxyClient, c clientConn) *grpcTunnel { + t := grpcTunnel{ stream: stream, - pendingDial: make(map[int64]pendingDial), - conns: make(map[int64]*conn), + clientConn: c, + pendingDial: pendingDialManager{pendingDials: make(map[int64]pendingDial)}, + conns: connectionManager{conns: make(map[int64]*conn)}, readTimeoutSeconds: 10, + done: make(chan struct{}), + } + s := metrics.ClientConnectionStatusCreated + t.prevStatus.Store(s) + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(s)).Inc() + return &t +} + +func (t *grpcTunnel) updateMetric(status metrics.ClientConnectionStatus) { + select { + case <-t.Done(): + return + default: } - go tunnel.serve(tunnelCtx, c) + prevStatus := t.prevStatus.Swap(status).(metrics.ClientConnectionStatus) - return tunnel, nil + m := metrics.Metrics.GetClientConnectionsMetric() + m.WithLabelValues(string(prevStatus)).Dec() + m.WithLabelValues(string(status)).Inc() +} + +// closeMetric should be called exactly once to finalize client_connections metric. +func (t *grpcTunnel) closeMetric() { + select { + case <-t.Done(): + return + default: + } + prevStatus := t.prevStatus.Load().(metrics.ClientConnectionStatus) + + metrics.Metrics.GetClientConnectionsMetric().WithLabelValues(string(prevStatus)).Dec() } -func (t *grpcTunnel) serve(tunnelCtx context.Context, c clientConn) { +func (t *grpcTunnel) serve(tunnelCtx context.Context) { defer func() { - c.Close() + t.clientConn.Close() // A connection in t.conns after serve() returns means // we never received a CLOSE_RSP for it, so we need to // close any channels remaining for these connections. - t.connsLock.Lock() - for _, conn := range t.conns { - close(conn.readCh) - } - t.connsLock.Unlock() + t.conns.closeAll() + + t.closeMetric() + + close(t.done) }() for { - pkt, err := t.stream.Recv() + pkt, err := t.Recv() if err == io.EOF { return } + isClosing := t.isClosing() if err != nil || pkt == nil { - klog.ErrorS(err, "stream read failure") + if !isClosing { + klog.ErrorS(err, "stream read failure") + } + return + } + if isClosing { return } - klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type) switch pkt.Type { case client.PacketType_DIAL_RSP: resp := pkt.GetDialResponse() - t.pendingDialLock.RLock() - pendingDial, ok := t.pendingDial[resp.Random] - t.pendingDialLock.RUnlock() + pendingDial, ok := t.pendingDial.get(resp.Random) if !ok { - klog.V(1).InfoS("DialResp not recognized; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) + // If the DIAL_RSP does not match a pending dial, it means one of two things: + // 1. There was a second DIAL_RSP for the connection request (this is very unlikely but possible) + // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context + // + // In either scenario, we should return here and close the tunnel as it is no longer needed. + kvs := []interface{}{"dialID", resp.Random, "connectID", resp.ConnectID} + if resp.Error != "" { + kvs = append(kvs, "error", resp.Error) + } + klog.V(1).InfoS("DialResp not recognized; dropped", kvs...) return + } + + result := dialResult{connid: resp.ConnectID} + if resp.Error != "" { + result.err = &dialFailure{resp.Error, metrics.DialFailureEndpoint} + } else { + t.updateMetric(metrics.ClientConnectionStatusOk) + } + select { + // try to send to the result channel + case pendingDial.resultCh <- result: + // unblock if the cancel channel is closed + case <-pendingDial.cancelCh: + // Note: this condition can only be hit by a race condition where the + // DialContext() returns early (timeout) after the pendingDial is already + // fetched here, but before the result is sent. + klog.V(1).InfoS("Pending dial has been cancelled; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) + return + case <-tunnelCtx.Done(): + klog.V(1).InfoS("Tunnel has been closed; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) + return + } + + if resp.Error != "" { + // On dial error, avoid leaking serve goroutine. + return + } + + case client.PacketType_DIAL_CLS: + resp := pkt.GetCloseDial() + pendingDial, ok := t.pendingDial.get(resp.Random) + + if !ok { + // If the DIAL_CLS does not match a pending dial, it means one of two things: + // 1. There was a DIAL_CLS receieved after a DIAL_RSP (unlikely but possible) + // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context + // + // In either scenario, we should return here and close the tunnel as it is no longer needed. + klog.V(1).InfoS("DIAL_CLS after dial finished", "dialID", resp.Random) } else { result := dialResult{ - err: resp.Error, - connid: resp.ConnectID, + err: &dialFailure{"dial closed", metrics.DialFailureDialClosed}, } select { - // try to send to the result channel case pendingDial.resultCh <- result: - // unblock if the cancel channel is closed case <-pendingDial.cancelCh: - // If there are no readers of the pending dial channel above, it means one of two things: - // 1. There was a second DIAL_RSP for the connection request (this is very unlikely but possible) - // 2. grpcTunnel.DialContext() returned early due to a dial timeout or the client canceling the context - // - // In either scenario, we should return here as this tunnel is no longer needed. - klog.V(1).InfoS("Pending dial has been cancelled; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) - return + // Note: this condition can only be hit by a race condition where the + // DialContext() returns early (timeout) after the pendingDial is already + // fetched here, but before the result is sent. case <-tunnelCtx.Done(): - klog.V(1).InfoS("Tunnel has been closed; dropped", "connectionID", resp.ConnectID, "dialID", resp.Random) - return } } - - if resp.Error != "" { - // On dial error, avoid leaking serve goroutine. - return - } + return // Stop serving & close the tunnel. case client.PacketType_DATA: resp := pkt.GetData() + if resp.ConnectID == 0 { + klog.ErrorS(nil, "Received packet missing ConnectID", "packetType", "DATA") + continue + } // TODO: flow control - t.connsLock.RLock() - conn, ok := t.conns[resp.ConnectID] - t.connsLock.RUnlock() + conn, ok := t.conns.get(resp.ConnectID) - if ok { - timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second) - select { - case conn.readCh <- resp.Data: - timer.Stop() - case <-timer.C: - klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds) - return - case <-tunnelCtx.Done(): - klog.V(1).InfoS("Tunnel has been closed, the grpc connection to the proxy server will be closed", "connectionID", conn.connID) - } - } else { - klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID) + if !ok { + klog.ErrorS(nil, "Connection not recognized", "connectionID", resp.ConnectID, "packetType", "DATA") + t.Send(&client.Packet{ + Type: client.PacketType_CLOSE_REQ, + Payload: &client.Packet_CloseRequest{ + CloseRequest: &client.CloseRequest{ + ConnectID: resp.ConnectID, + }, + }, + }) + continue + } + timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second) + select { + case conn.readCh <- resp.Data: + timer.Stop() + case <-timer.C: + klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds) + return + case <-tunnelCtx.Done(): + klog.V(1).InfoS("Tunnel has been closed, the grpc connection to the proxy server will be closed", "connectionID", conn.connID) } + case client.PacketType_CLOSE_RSP: resp := pkt.GetCloseResponse() - t.connsLock.RLock() - conn, ok := t.conns[resp.ConnectID] - t.connsLock.RUnlock() - - if ok { - close(conn.readCh) - conn.closeCh <- resp.Error - close(conn.closeCh) - t.connsLock.Lock() - delete(t.conns, resp.ConnectID) - t.connsLock.Unlock() - return + conn, ok := t.conns.get(resp.ConnectID) + + if !ok { + klog.V(1).InfoS("Connection not recognized", "connectionID", resp.ConnectID, "packetType", "CLOSE_RSP") + continue } - klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID) + close(conn.readCh) + conn.closeCh <- resp.Error + close(conn.closeCh) + t.conns.remove(resp.ConnectID) + return } } } @@ -219,10 +384,27 @@ func (t *grpcTunnel) serve(tunnelCtx context.Context, c clientConn) { // Dial connects to the address on the named network, similar to // what net.Dial does. The only supported protocol is tcp. func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + conn, err := t.dialContext(requestCtx, protocol, address) + if err != nil { + _, reason := GetDialFailureReason(err) + metrics.Metrics.ObserveDialFailure(reason) + } + return conn, err +} + +func (t *grpcTunnel) dialContext(requestCtx context.Context, protocol, address string) (net.Conn, error) { + select { + case <-t.done: + return nil, errors.New("tunnel is closed") + default: // Tunnel is open, carry on. + } + if protocol != "tcp" { return nil, errors.New("protocol not supported") } + t.updateMetric(metrics.ClientConnectionStatusDialing) + random := rand.Int63() /* #nosec G404 */ // This channel is closed once we're returning and no longer waiting on resultCh @@ -232,14 +414,8 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s // This channel MUST NOT be buffered. The sender needs to know when we are not receiving things, so they can abort. resCh := make(chan dialResult) - t.pendingDialLock.Lock() - t.pendingDial[random] = pendingDial{resultCh: resCh, cancelCh: cancelCh} - t.pendingDialLock.Unlock() - defer func() { - t.pendingDialLock.Lock() - delete(t.pendingDial, random) - t.pendingDialLock.Unlock() - }() + t.pendingDial.add(random, pendingDial{resultCh: resCh, cancelCh: cancelCh}) + defer t.pendingDial.remove(random) req := &client.Packet{ Type: client.PacketType_DIAL_REQ, @@ -253,33 +429,115 @@ func (t *grpcTunnel) DialContext(requestCtx context.Context, protocol, address s } klog.V(5).InfoS("[tracing] send packet", "type", req.Type) - err := t.stream.Send(req) + err := t.Send(req) if err != nil { return nil, err } klog.V(5).Infoln("DIAL_REQ sent to proxy server") - c := &conn{stream: t.stream, random: random} + c := &conn{ + tunnel: t, + random: random, + closeTunnel: t.closeTunnel, + } select { case res := <-resCh: - if res.err != "" { - return nil, errors.New(res.err) + if res.err != nil { + return nil, res.err } c.connID = res.connid c.readCh = make(chan []byte, 10) c.closeCh = make(chan string, 1) - t.connsLock.Lock() - t.conns[res.connid] = c - t.connsLock.Unlock() + t.conns.add(res.connid, c) case <-time.After(30 * time.Second): klog.V(5).InfoS("Timed out waiting for DialResp", "dialID", random) - return nil, errors.New("dial timeout, backstop") + go t.closeDial(random) + return nil, &dialFailure{"dial timeout, backstop", metrics.DialFailureTimeout} case <-requestCtx.Done(): klog.V(5).InfoS("Context canceled waiting for DialResp", "ctxErr", requestCtx.Err(), "dialID", random) - return nil, errors.New("dial timeout, context") + go t.closeDial(random) + return nil, &dialFailure{"dial timeout, context", metrics.DialFailureContext} + case <-t.done: + klog.V(5).InfoS("Tunnel closed while waiting for DialResp", "dialID", random) + return nil, &dialFailure{"tunnel closed", metrics.DialFailureTunnelClosed} } return c, nil } + +func (t *grpcTunnel) Done() <-chan struct{} { + return t.done +} + +// Send a best-effort DIAL_CLS request for the given dial ID. +func (t *grpcTunnel) closeDial(dialID int64) { + req := &client.Packet{ + Type: client.PacketType_DIAL_CLS, + Payload: &client.Packet_CloseDial{ + CloseDial: &client.CloseDial{ + Random: dialID, + }, + }, + } + if err := t.Send(req); err != nil { + klog.V(5).InfoS("Failed to send DIAL_CLS", "err", err, "dialID", dialID) + } + t.closeTunnel() +} + +func (t *grpcTunnel) closeTunnel() { + atomic.StoreUint32(&t.closing, 1) + t.clientConn.Close() +} + +func (t *grpcTunnel) isClosing() bool { + return atomic.LoadUint32(&t.closing) != 0 +} + +func (t *grpcTunnel) Send(pkt *client.Packet) error { + t.sendLock.Lock() + defer t.sendLock.Unlock() + + const segment = commonmetrics.SegmentFromClient + metrics.Metrics.ObservePacket(segment, pkt.Type) + err := t.stream.Send(pkt) + if err != nil && err != io.EOF { + metrics.Metrics.ObserveStreamError(segment, err, pkt.Type) + } + return err +} + +func (t *grpcTunnel) Recv() (*client.Packet, error) { + t.recvLock.Lock() + defer t.recvLock.Unlock() + + const segment = commonmetrics.SegmentToClient + pkt, err := t.stream.Recv() + if err != nil { + if err != io.EOF { + metrics.Metrics.ObserveStreamErrorNoPacket(segment, err) + } + return nil, err + } + metrics.Metrics.ObservePacket(segment, pkt.Type) + return pkt, nil +} + +func GetDialFailureReason(err error) (isDialFailure bool, reason metrics.DialFailureReason) { + var df *dialFailure + if errors.As(err, &df) { + return true, df.reason + } + return false, metrics.DialFailureUnknown +} + +type dialFailure struct { + msg string + reason metrics.DialFailureReason +} + +func (df *dialFailure) Error() string { + return df.msg +} diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/conn.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/conn.go index 822831b10..f4d3f7886 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/conn.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/conn.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/klog/v2" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" ) @@ -35,12 +36,15 @@ var errConnCloseTimeout = errors.New("close timeout") // conn is an implementation of net.Conn, where the data is transported // over an established tunnel defined by a gRPC service ProxyService. type conn struct { - stream client.ProxyService_ProxyClient + tunnel *grpcTunnel connID int64 random int64 readCh chan []byte closeCh chan string rdata []byte + + // closeTunnel is an optional callback to close the underlying grpc connection. + closeTunnel func() } var _ net.Conn = &conn{} @@ -59,7 +63,7 @@ func (c *conn) Write(data []byte) (n int, err error) { klog.V(5).InfoS("[tracing] send req", "type", req.Type) - err = c.stream.Send(req) + err = c.tunnel.Send(req) if err != nil { return 0, err } @@ -116,6 +120,10 @@ func (c *conn) SetWriteDeadline(t time.Time) error { // proxy service to notify remote to drop the connection. func (c *conn) Close() error { klog.V(4).Infoln("closing connection") + if c.closeTunnel != nil { + defer c.closeTunnel() + } + var req *client.Packet if c.connID != 0 { req = &client.Packet{ @@ -140,7 +148,7 @@ func (c *conn) Close() error { klog.V(5).InfoS("[tracing] send req", "type", req.Type) - if err := c.stream.Send(req); err != nil { + if err := c.tunnel.Send(req); err != nil { return err } diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go new file mode 100644 index 000000000..03e9d94da --- /dev/null +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics/metrics.go @@ -0,0 +1,162 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + + commonmetrics "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics" + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +const ( + Namespace = "konnectivity_network_proxy" + Subsystem = "client" +) + +var ( + // Metrics provides access to all client metrics. The client + // application is responsible for registering (via Metrics.RegisterMetrics). + Metrics = newMetrics() +) + +// ClientMetrics includes all the metrics of the konnectivity-client. +type ClientMetrics struct { + registerOnce sync.Once + streamPackets *prometheus.CounterVec + streamErrors *prometheus.CounterVec + dialFailures *prometheus.CounterVec + clientConns *prometheus.GaugeVec +} + +type DialFailureReason string + +const ( + DialFailureUnknown DialFailureReason = "unknown" + // DialFailureTimeout indicates the hard 30 second timeout was hit. + DialFailureTimeout DialFailureReason = "timeout" + // DialFailureContext indicates that the context was cancelled or reached it's deadline before + // the dial response was returned. + DialFailureContext DialFailureReason = "context" + // DialFailureEndpoint indicates that the konnectivity-agent was unable to reach the backend endpoint. + DialFailureEndpoint DialFailureReason = "endpoint" + // DialFailureDialClosed indicates that the client received a CloseDial response, indicating the + // connection was closed before the dial could complete. + DialFailureDialClosed DialFailureReason = "dialclosed" + // DialFailureTunnelClosed indicates that the client connection was closed before the dial could + // complete. + DialFailureTunnelClosed DialFailureReason = "tunnelclosed" +) + +type ClientConnectionStatus string + +const ( + // The connection is created but has not yet been dialed. + ClientConnectionStatusCreated ClientConnectionStatus = "created" + // The connection is pending dial response. + ClientConnectionStatusDialing ClientConnectionStatus = "dialing" + // The connection is established. + ClientConnectionStatusOk ClientConnectionStatus = "ok" + // The connection is closing. + ClientConnectionStatusClosing ClientConnectionStatus = "closing" +) + +func newMetrics() *ClientMetrics { + // The denominator (total dials started) for both + // dial_failure_total and dial_duration_seconds is the + // stream_packets_total (common metric), where segment is + // "from_client" and packet_type is "DIAL_REQ". + dialFailures := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "dial_failure_total", + Help: "Number of dial failures observed, by reason (example: remote endpoint error)", + }, + []string{ + "reason", + }, + ) + clientConns := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "client_connections", + Help: "Number of open client connections, by status (Example: dialing)", + }, + []string{ + "status", + }, + ) + return &ClientMetrics{ + streamPackets: commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem), + streamErrors: commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem), + dialFailures: dialFailures, + clientConns: clientConns, + } +} + +// RegisterMetrics registers all metrics with the client application. +func (c *ClientMetrics) RegisterMetrics(r prometheus.Registerer) { + c.registerOnce.Do(func() { + r.MustRegister(c.streamPackets) + r.MustRegister(c.streamErrors) + r.MustRegister(c.dialFailures) + r.MustRegister(c.clientConns) + }) +} + +// LegacyRegisterMetrics registers all metrics via MustRegister func. +// TODO: remove this once https://github.com/kubernetes/kubernetes/pull/114293 is available. +func (c *ClientMetrics) LegacyRegisterMetrics(mustRegisterFn func(...prometheus.Collector)) { + c.registerOnce.Do(func() { + mustRegisterFn(c.streamPackets) + mustRegisterFn(c.streamErrors) + mustRegisterFn(c.dialFailures) + mustRegisterFn(c.clientConns) + }) +} + +// Reset resets the metrics. +func (c *ClientMetrics) Reset() { + c.streamPackets.Reset() + c.streamErrors.Reset() + c.dialFailures.Reset() + c.clientConns.Reset() +} + +func (c *ClientMetrics) ObserveDialFailure(reason DialFailureReason) { + c.dialFailures.WithLabelValues(string(reason)).Inc() +} + +func (c *ClientMetrics) GetClientConnectionsMetric() *prometheus.GaugeVec { + return c.clientConns +} + +func (c *ClientMetrics) ObservePacket(segment commonmetrics.Segment, packetType client.PacketType) { + commonmetrics.ObservePacket(c.streamPackets, segment, packetType) +} + +func (c *ClientMetrics) ObserveStreamErrorNoPacket(segment commonmetrics.Segment, err error) { + commonmetrics.ObserveStreamErrorNoPacket(c.streamErrors, segment, err) +} + +func (c *ClientMetrics) ObserveStreamError(segment commonmetrics.Segment, err error, packetType client.PacketType) { + commonmetrics.ObserveStreamError(c.streamErrors, segment, err, packetType) +} diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/metrics.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/metrics.go new file mode 100644 index 000000000..e8619f472 --- /dev/null +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics/metrics.go @@ -0,0 +1,78 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metrics provides metric definitions and helpers used +// across konnectivity client, server, and agent. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" + + "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client" +) + +// Segment identifies one of four tunnel segments (e.g. from server to agent). +type Segment string + +const ( + // SegmentFromClient indicates a packet from client to server. + SegmentFromClient Segment = "from_client" + // SegmentToClient indicates a packet from server to client. + SegmentToClient Segment = "to_client" + // SegmentFromAgent indicates a packet from agent to server. + SegmentFromAgent Segment = "from_agent" + // SegmentToAgent indicates a packet from server to agent. + SegmentToAgent Segment = "to_agent" +) + +func MakeStreamPacketsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_packets_total", + Help: "Count of packets processed, by segment and packet type (example: from_client, DIAL_REQ)", + }, + []string{"segment", "packet_type"}, + ) +} + +func MakeStreamErrorsTotalMetric(namespace, subsystem string) *prometheus.CounterVec { + return prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "stream_errors_total", + Help: "Count of gRPC stream errors, by segment, grpc Code, packet type. (example: from_agent, Code.Unavailable, DIAL_RSP)", + }, + []string{"segment", "code", "packet_type"}, + ) +} + +func ObservePacket(m *prometheus.CounterVec, segment Segment, packetType client.PacketType) { + m.WithLabelValues(string(segment), packetType.String()).Inc() +} + +func ObserveStreamErrorNoPacket(m *prometheus.CounterVec, segment Segment, err error) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), "Unknown").Inc() +} + +func ObserveStreamError(m *prometheus.CounterVec, segment Segment, err error, packetType client.PacketType) { + code := status.Code(err) + m.WithLabelValues(string(segment), code.String(), packetType.String()).Inc() +}