diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..c4ebe00 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-pc-windows-msvc] +linker = "rust-lld.exe" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..b6fffcb --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,161 @@ +name: "CI" +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + check: + timeout-minutes: 3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt, clippy + - name: cargo fetch + run: cargo fetch --verbose + - name: cargo check + run: cargo check --all-features --all-targets + if: always() + - name: cargo fmt + uses: actions-rust-lang/rustfmt@v1 + if: always() + - id: clippy + run: cargo clippy --all-features --all-targets + if: always() + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + kind_image: + - "v1.30.2" + - "v1.29.4" + - "v1.28.9" + - "v1.27.13" + - "v1.26.15" + - "v1.25.16" + - "v1.24.17" + env: + KIND_IMAGE: kindest/node:${{matrix.kind_image}} + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + - name: Setup kind + run: | + kind create cluster --image="$KIND_IMAGE" --name test-pgd + - run: cargo fetch --verbose + - run: cargo test --all-features + timeout-minutes: 5 + + tests-result: + runs-on: ubuntu-latest + needs: test + if: always() + steps: + - name: All tests ok + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + lint-helm: + timeout-minutes: 1 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: helm lint charts/pod-graceful-drain --strict + + build-image: + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + docker build --file docker/build.Dockerfile \ + --tag "pod-graceful-drain:latest" \ + . + - run: docker image save --output pod-graceful-drain.tar pod-graceful-drain + - uses: actions/upload-artifact@v4 + with: + name: pod-graceful-drain.tar + path: ./pod-graceful-drain.tar + + smoke-test: + timeout-minutes: 3 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + kind_image: + - "v1.30.2" + - "v1.29.4" + - "v1.28.9" + - "v1.27.13" + - "v1.26.15" + - "v1.25.16" + - "v1.24.17" + env: + KIND_IMAGE: kindest/node:${{matrix.kind_image}} + needs: + - build-image + steps: + - uses: actions/checkout@v4 + - name: Setup kind + run: | + kind create cluster --image="$KIND_IMAGE" + - uses: eifinger/setup-rye@v4 + with: + version: '0.37.0' + - uses: actions/download-artifact@v4 + with: + name: pod-graceful-drain.tar + - run: | + docker image load --input pod-graceful-drain.tar + kind load docker-image pod-graceful-drain:latest + - run: | + helm install pod-graceful-drain charts/pod-graceful-drain \ + --create-namespace --namespace pod-graceful-drain \ + --set image.repository=pod-graceful-drain --set image.tag=latest \ + --set experimentalGeneralIngress=true \ + --set logLevel=info\\,pod_graceful_drain=trace \ + --wait=true --timeout=1m + - run: rye sync + - run: rye test + - name: Dump + if: always() + run: | + KUBECTL="kubectl --namespace pod-graceful-drain" + echo "::group::kubectl get" + ${KUBECTL} get --ignore-not-found=true --output wide all + echo "::endgroup::" + + echo "::group::kubectl describe" + ${KUBECTL} describe all + echo "::endgroup::" + + for POD in $(${KUBECTL} get pod -o=name); do + echo "::group::kubectl logs ${POD}" + ${KUBECTL} logs --ignore-errors=true "${POD}" + echo "::endgroup::" + done + + smoke-tests-result: + runs-on: ubuntu-latest + needs: smoke-test + if: always() + steps: + - name: All tests ok + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 diff --git a/.github/workflows/release-chart.yaml b/.github/workflows/release-chart.yaml index 26dc7d5..3617a7c 100644 --- a/.github/workflows/release-chart.yaml +++ b/.github/workflows/release-chart.yaml @@ -6,7 +6,7 @@ jobs: name: Publish chart runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - id: package name: Package charts run: | @@ -33,4 +33,4 @@ jobs: git config user.email pod-graceful-drain-bot@users.noreply.github.com git config user.name pod-graceful-drain-bot git commit -m "Updated chart index: ${ADDED_FILE}" - - run: git push origin gh-pages \ No newline at end of file + - run: git push origin gh-pages diff --git a/.github/workflows/release-image.yaml b/.github/workflows/release-image.yaml index 026bb36..1eb6ddf 100644 --- a/.github/workflows/release-image.yaml +++ b/.github/workflows/release-image.yaml @@ -2,34 +2,35 @@ name: Release image on: push: tags: - - v* + - v* workflow_dispatch: jobs: build: name: Publish image runs-on: ubuntu-latest + env: + IMG_NAME: pod-graceful-drain steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: ^1.18 - - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - run: make docker-build - - name: Tag and push - run: | - GIT_VERSION=$(git describe --tags --dirty --always) - REGISTRY=ghcr.io/${{ github.repository_owner }} - IMG_NAME=pod-graceful-drain + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.CR_PAT }} + - run: | + docker build --file docker/build.Dockerfile \ + --tag "$IMG_NAME:latest" \ + . + - name: Tag and push + run: | + GIT_VERSION=$(git describe --tags --dirty --always) + REGISTRY=ghcr.io/${{ github.repository_owner }} - tag-and-push() { docker tag $1 $2 && docker push $2; } + tag-and-push() { docker tag $1 $2 && docker push $2; } - tag-and-push $IMG_NAME:latest $REGISTRY/$IMG_NAME:$GIT_VERSION + tag-and-push $IMG_NAME:latest $REGISTRY/$IMG_NAME:$GIT_VERSION - LATEST_TAG="$(git describe --tags --dirty --always --abbrev=0)" - if git diff --quiet "$LATEST_TAG"; then - tag-and-push $IMG_NAME:latest $REGISTRY/$IMG_NAME:latest - fi + LATEST_TAG="$(git describe --tags --dirty --always --abbrev=0)" + if git diff --quiet "$LATEST_TAG"; then + tag-and-push $IMG_NAME:latest $REGISTRY/$IMG_NAME:latest + fi diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index 6483b84..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Test -on: - push: - branches: - - main - pull_request: - branches: - - main -jobs: - build: - name: Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: ^1.18 - - run: make test manifests docker-build - - name: Manifests are up to date - run: git diff --exit-code -- config/ - - uses: codecov/codecov-action@v1 - with: - file: ./cover.out - helm: - name: Helm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install helm kubeval - run: helm plugin install https://github.com/instrumenta/helm-kubeval - - run: helm lint charts/pod-graceful-drain --strict - - run: helm kubeval charts/pod-graceful-drain - - uses: helm/kind-action@v1.2.0 - - run: helm install pod-graceful-drain charts/pod-graceful-drain --set image.tag=latest - kustomize: - name: Kustomize - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: ^1.18 - - uses: helm/kind-action@v1.2.0 - - name: Install certmanager - run: kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml - - run: make deploy diff --git a/.gitignore b/.gitignore index d97ffc5..089ff51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,7 @@ +.venv/ +*.egg-info/ +__pycache__/ -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib -bin +target/ -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Kubernetes Generated files - skip generated files, except for vendored files - -!vendor/**/zz_generated.* - -# editor and IDE paraphernalia -.idea -*.swp -*.swo -*~ +.idea/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..871f80a --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.3 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..292e930 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2985 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "async-broadcast" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-shutdown" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f13fa00d3d9ba01f4c6ee77a73d784281cfa3340a9e1d43948e439921d24e934" + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "async-trait" +version = "0.1.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-server" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" +dependencies = [ + "arc-swap", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tower-service", +] + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom", + "instant", + "rand", +] + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" + +[[package]] +name = "cc" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "clap" +version = "4.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c937d4061031a6d0c8da4b9a4f98a172fc2976dfb1c19213a9cf7d0d3c837e36" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85379ba512b21a328adf887e85f7742d12e96eb31f3ef077df4ffc26b506ffed" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "color-eyre" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.72", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "debounced" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d8b0346b9fa0aa01a3fa4bcce48d62f8738e9c2956e92f275bbf6cf9d6fab5" +dependencies = [ + "futures-timer", + "futures-util", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +dependencies = [ + "derive_builder_core", + "syn 2.0.72", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "filetime" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + +[[package]] +name = "fluent-uri" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "git+https://github.com/foriequal0/genawaiter#b7ea36d8fc6d6987db0edf499bbae8a99316eb56" +dependencies = [ + "futures-core", + "genawaiter-macro", + "genawaiter-proc-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "git+https://github.com/foriequal0/genawaiter#b7ea36d8fc6d6987db0edf499bbae8a99316eb56" + +[[package]] +name = "genawaiter-proc-macro" +version = "0.99.1" +source = "git+https://github.com/foriequal0/genawaiter#b7ea36d8fc6d6987db0edf499bbae8a99316eb56" +dependencies = [ + "proc-macro-error 0.4.12", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45727250e75cc04ff2846a66397da8ef2b3db8e40e0cef4df67950a07621eb9" +dependencies = [ + "proc-macro-error 1.0.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "headers" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +dependencies = [ + "base64 0.21.7", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" +dependencies = [ + "http", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "hostname" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" +dependencies = [ + "cfg-if", + "libc", + "windows", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-http-proxy" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d06dbdfbacf34d996c6fb540a71a684a7aae9056c71951163af8a8a4c07b9a4" +dependencies = [ + "bytes", + "futures-util", + "headers", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" +dependencies = [ + "jsonptr", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "jsonpath-rust" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" +dependencies = [ + "lazy_static", + "once_cell", + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror", +] + +[[package]] +name = "jsonptr" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" +dependencies = [ + "fluent-uri", + "serde", + "serde_json", +] + +[[package]] +name = "k8s-openapi" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19501afb943ae5806548bc3ebd7f3374153ca057a38f480ef30adfde5ef09755" +dependencies = [ + "base64 0.22.1", + "chrono", + "serde", + "serde-value", + "serde_json", +] + +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "kube" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0365920075af1a2d23619c1ca801c492f2400157de42627f041a061716e76416" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81336eb3a5b10a40c97a5a97ad66622e92bad942ce05ee789edd730aa4f8603" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "either", + "futures", + "home", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-http-proxy", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jsonpath-rust", + "k8s-openapi", + "kube-core", + "pem", + "rustls", + "rustls-pemfile", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cce373a74d787d439063cdefab0f3672860bd7bac01a38e39019177e764a0fe6" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch", + "k8s-openapi", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-runtime" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b84733c0fed6085c9210b43ffb96248676c1e800d0ba38d15043275a792ffa4" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "async-trait", + "backoff", + "derivative", + "futures", + "hashbrown", + "json-patch", + "jsonptr", + "k8s-openapi", + "kube-client", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "local-ip-address" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136ef34e18462b17bf39a7826f8f3bbc223341f8e83822beb8b77db9a3d49696" +dependencies = [ + "libc", + "neli", + "thiserror", + "windows-sys 0.48.0", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mio" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "neli" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43" +dependencies = [ + "byteorder", + "libc", + "log", + "neli-proc-macros", +] + +[[package]] +name = "neli-proc-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4" +dependencies = [ + "either", + "proc-macro2", + "quote", + "serde", + "syn 1.0.109", +] + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.6.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "pem" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +dependencies = [ + "base64 0.22.1", + "serde", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "pest_meta" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pod-graceful-drain" +version = "0.1.0-rc.1" +dependencies = [ + "anyhow", + "async-shutdown", + "axum", + "axum-server", + "backoff", + "base64 0.22.1", + "chrono", + "clap", + "color-eyre", + "debounced", + "either", + "eyre", + "futures", + "genawaiter", + "hostname", + "humantime", + "json-patch", + "jsonptr", + "k8s-openapi", + "kube", + "local-ip-address", + "notify", + "rand", + "rcgen", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "thiserror", + "tokio", + "tower", + "tracing", + "tracing-error", + "tracing-subscriber", + "uuid", + "vergen-gitcl", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-error" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +dependencies = [ + "proc-macro-error-attr 0.4.12", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr 1.0.4", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "syn-mid", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rcgen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-webpki" +version = "0.102.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.205" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.205" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "serde_json" +version = "1.0.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea305d57546cc8cd04feb14b62ec84bf17f50e3f7b12560d7bfa9265f39d9ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tokio" +version = "1.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio 1.0.1", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "base64 0.21.7", + "bitflags 2.6.0", + "bytes", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vergen" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c32e7318e93a9ac53693b6caccfb05ff22e04a44c7cf8a279051f24c09da286f" +dependencies = [ + "anyhow", + "derive_builder", + "rustc_version", + "rustversion", + "time", + "vergen-lib", +] + +[[package]] +name = "vergen-gitcl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbdc9746577cb4767f218d320ee0b623d415e8130332f8f562b910b61cc2c4e" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06bee42361e43b60f363bad49d63798d0f42fb1768091812270eca00c784720" +dependencies = [ + "anyhow", + "derive_builder", + "getset", + "rustversion", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..a8fad3d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "pod-graceful-drain" +description = "You don't need `lifecycle: { preStop: { exec: { command: [\"sleep\", \"30\"] } } }`" +authors = ["SeongChan Lee "] +version = "0.1.0-rc.1" +edition = "2021" +rust-version = "1.79" + +[dependencies] +# kubernetes libs +kube = { version = "0.93.1", features = ["runtime", "admission"] } +k8s-openapi = { version = "0.22.0", features = ["v1_30"] } + +# async runtime libs +tokio = "1.39.2" +futures = "0.3.30" + +# webhook libs +axum = "0.7.5" +axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } +tower = "0.4.13" +rustls = { version = "0.23.12", default-features = false, features = ["ring"] } +rustls-pemfile = "2.1.2" + +# observability libs +tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tracing-error = "0.2.0" +eyre = "0.6.12" +color-eyre = { version = "0.6.3", features = ["capture-spantrace"] } +thiserror = "1.0.63" + +# auxilary libs +clap = { version = "4.5.14", features = ["derive"] } +genawaiter = { git = "https://github.com/foriequal0/genawaiter", features = ["futures03"] } +humantime = "2.1.0" +async-shutdown = "0.2.2" +serde = "1.0.205" +serde_json = "1.0.122" +json-patch = "2.0.0" +jsonptr = "0.4.7" # json-patch uses 0.4.x +either = "1.13.0" +chrono = "0.4.38" +backoff = "0.4.0" +rand = "0.8.5" +notify = "6.1.1" +debounced = "0.1.0" +hostname = "0.4.0" +uuid = { version = "1.10.0", features = ["v4"] } + +[dev-dependencies] +tempfile = "3.12.0" +serde_yaml = "0.9.34-deprecated" +local-ip-address = "0.6.1" +base64 = "0.22.1" +rcgen = "0.13.1" + +[build-dependencies] +anyhow = "1.0.86" +vergen-gitcl = { version = "1.0.0", features = ["build", "rustc"] } diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1453a15..0000000 --- a/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:1.18 as base - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -FROM base as build -COPY . . -# Build the manager binary -RUN GIT_VERSION=$(git describe --tags --dirty --always) && \ - GIT_COMMIT=$(git rev-parse HEAD) && \ - BUILD_DATE=$(date +%Y-%m-%dT%H:%M:%S%z) && \ - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on \ - go build -ldflags="-X main.GitVersion=${GIT_VERSION} -X main.GitCommit=${GIT_COMMIT} -X main.BuildDate=${BUILD_DATE}" \ - -a -o /out/manager main.go - -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=build /out/manager . -USER nonroot:nonroot - -ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile deleted file mode 100644 index 72adf7a..0000000 --- a/Makefile +++ /dev/null @@ -1,79 +0,0 @@ - -# Image URL to use all building/pushing image targets -IMG ?= pod-graceful-drain:latest - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -GIT_VERSION=$(shell git describe --tags --dirty --always) -GIT_COMMIT=$(shell git rev-parse HEAD) -BUILD_DATE=$(shell date +%Y-%m-%dT%H:%M:%S%z) - -all: manager - -# Run tests -test: generate fmt vet manifests - go test ./... -coverprofile cover.out - -# Build manager binary -manager: generate fmt vet - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on \ - go build -ldflags="-X main.GitVersion=$(GIT_VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildDate=$(BUILD_DATE)" \ - -o bin/manager main.go - -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests - go run ./main.go - -# Install CRDs into a cluster -install: manifests - kustomize build config/crd | kubectl apply -f - - -# Uninstall CRDs from a cluster -uninstall: manifests - kustomize build config/crd | kubectl delete -f - - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests - cd config/manager && kustomize edit set image manager=${IMG} - kustomize build config/default | kubectl apply -f - - -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen - $(CONTROLLER_GEN) crd rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -# Run go fmt against code -fmt: - go fmt ./... - -# Run go vet against code -vet: - go vet ./... - -# Generate code -generate: controller-gen - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -# Build the docker image -docker-build: test - docker build . -t ${IMG} - -# Push the docker image -docker-push: - docker push ${IMG} - -# find or download controller-gen -# download controller-gen if necessary -controller-gen: -ifeq (, $(shell which controller-gen)) - @{ \ - go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0 ;\ - } -CONTROLLER_GEN=$(GOBIN)/controller-gen -else -CONTROLLER_GEN=$(shell which controller-gen) -endif diff --git a/PROJECT b/PROJECT deleted file mode 100644 index 3e74b88..0000000 --- a/PROJECT +++ /dev/null @@ -1,3 +0,0 @@ -domain: github.com/foriequal0/pod-graceful-drain -repo: github.com/foriequal0/pod-graceful-drain -version: "2" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..320b504 --- /dev/null +++ b/build.rs @@ -0,0 +1,16 @@ +use anyhow::Result; +use vergen_gitcl::{BuildBuilder, Emitter, GitclBuilder, RustcBuilder}; + +pub fn main() -> Result<()> { + let build = BuildBuilder::all_build()?; + let gitcl = GitclBuilder::all_git()?; + let rustc = RustcBuilder::all_rustc()?; + + Emitter::default() + .add_instructions(&build)? + .add_instructions(&gitcl)? + .add_instructions(&rustc)? + .emit()?; + + Ok(()) +} diff --git a/charts/pod-graceful-drain/Chart.yaml b/charts/pod-graceful-drain/Chart.yaml index 5fc1e07..2b6e6e0 100644 --- a/charts/pod-graceful-drain/Chart.yaml +++ b/charts/pod-graceful-drain/Chart.yaml @@ -3,5 +3,5 @@ name: pod-graceful-drain description: | You don't need `lifecycle: { preStop: { exec: { command: ["sleep", "30"] } } }` type: application -version: 0.0.11 -appVersion: "v0.0.8" +version: 0.1.0 +appVersion: "v0.1.0-rc.1" diff --git a/charts/pod-graceful-drain/templates/_helpers.tpl b/charts/pod-graceful-drain/templates/_helpers.tpl index 9cb31d1..2ef4c5e 100644 --- a/charts/pod-graceful-drain/templates/_helpers.tpl +++ b/charts/pod-graceful-drain/templates/_helpers.tpl @@ -73,3 +73,12 @@ caCert: {{ $ca.Cert | b64enc }} clientCert: {{ $cert.Cert | b64enc }} clientKey: {{ $cert.Key | b64enc }} {{- end -}} + +{{/* +Timeouts: +5s to deleteAfter +*/}} +{{- define "pod-graceful-drain.timeoutSeconds" -}} +{{- $now := now -}} +{{- $seconds := sub ($now | dateModify .Values.deleteAfter | unixEpoch) ($now | unixEpoch) -}} +{{- printf "%d" (add $seconds 5) -}} +{{- end }} diff --git a/charts/pod-graceful-drain/templates/assertions.yaml b/charts/pod-graceful-drain/templates/assertions.yaml new file mode 100644 index 0000000..5188c17 --- /dev/null +++ b/charts/pod-graceful-drain/templates/assertions.yaml @@ -0,0 +1,10 @@ +{{/* +Assertions +*/}} +{{- with .Values.deleteAfter -}} +{{- $now := now -}} +{{- $seconds := sub ($now | dateModify . | unixEpoch) ($now | unixEpoch) -}} +{{- if or (gt $seconds 25) (lt $seconds 1) -}} +{{- fail (printf "'deleteAfter' should be >= 1s, <= 25s, current: %s" .) -}} +{{- end -}} +{{- end -}} diff --git a/charts/pod-graceful-drain/templates/deployment.yaml b/charts/pod-graceful-drain/templates/deployment.yaml index 4813946..7893aa3 100644 --- a/charts/pod-graceful-drain/templates/deployment.yaml +++ b/charts/pod-graceful-drain/templates/deployment.yaml @@ -6,6 +6,9 @@ metadata: {{- include "pod-graceful-drain.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxUnavailable: 0 selector: matchLabels: {{- include "pod-graceful-drain.selectorLabels" . | nindent 6 }} @@ -38,15 +41,17 @@ spec: {{- with .Values.deleteAfter }} - --delete-after={{ . }} {{- end }} - {{- if .Values.noDenyAdmission }} - - --no-deny-admission - {{- end }} - {{- with .Values.logLevel }} - - --log-level={{ . }} - {{- end }} - {{- if .Values.ignoreError }} - - --ignore-error + {{- if .Values.experimentalGeneralIngress }} + - --experimental-general-ingress {{- end }} + env: + - name: RUST_LOG + value: {{ .Values.logLevel | quote }} + readinessProbe: + httpGet: + path: "/healthz" + port: 9443 + scheme: HTTPS ports: - containerPort: 9443 name: webhook-server @@ -74,7 +79,7 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ template "pod-graceful-drain.timeoutSeconds" . }} volumes: - name: cert secret: diff --git a/charts/pod-graceful-drain/templates/pdb.yaml b/charts/pod-graceful-drain/templates/pdb.yaml new file mode 100644 index 0000000..6e1af08 --- /dev/null +++ b/charts/pod-graceful-drain/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.Version }} +apiVersion: policy/v1beta1 +{{- else }} +apiVersion: policy/v1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ include "pod-graceful-drain.fullname" . }} +spec: + minAvailable: 1 + selector: + matchLabels: + {{- include "pod-graceful-drain.selectorLabels" . | nindent 6 }} diff --git a/charts/pod-graceful-drain/templates/rbac.yaml b/charts/pod-graceful-drain/templates/rbac.yaml index a0543cc..df28c51 100644 --- a/charts/pod-graceful-drain/templates/rbac.yaml +++ b/charts/pod-graceful-drain/templates/rbac.yaml @@ -5,18 +5,29 @@ metadata: labels: {{- include "pod-graceful-drain.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: [nodes] - verbs: [get, list, watch] - - apiGroups: [""] - resources: [pods] - verbs: [get, list, watch, patch, delete] - - apiGroups: [""] - resources: [services] - verbs: [get, list, watch] - - apiGroups: [elbv2.k8s.aws] - resources: [targetgroupbindings] - verbs: [list, watch] + - apiGroups: [ events.k8s.io ] + resources: [ events ] + verbs: [ create ] + - apiGroups: [ "" ] + resources: [ users, groups ] + verbs: [ impersonate ] + - apiGroups: [ "" ] + resources: [ nodes ] + verbs: [ get, list, watch ] + - apiGroups: [ "" ] + resources: [ pods ] + verbs: [ get, list, watch, patch, delete ] + - apiGroups: [ "" ] + resources: [ services ] + verbs: [ get, list, watch ] + - apiGroups: [ networking.k8s.io ] + resources: [ ingresses ] + verbs: [ list, watch ] +{{ if not .Values.experimentalGeneralIngress }} + - apiGroups: [ elbv2.k8s.aws ] + resources: [ targetgroupbindings ] + verbs: [ list, watch ] +{{ end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/charts/pod-graceful-drain/templates/webhook.yaml b/charts/pod-graceful-drain/templates/webhook.yaml index 02eb830..f658710 100644 --- a/charts/pod-graceful-drain/templates/webhook.yaml +++ b/charts/pod-graceful-drain/templates/webhook.yaml @@ -11,7 +11,7 @@ metadata: labels: {{- include "pod-graceful-drain.labels" . | nindent 4 }} webhooks: - - admissionReviewVersions: [v1beta1, v1] + - admissionReviewVersions: [ v1beta1, v1 ] clientConfig: {{- if not .Values.enableCertManager }} caBundle: {{ $tls.caCert }} @@ -19,16 +19,16 @@ webhooks: service: name: {{ template "pod-graceful-drain.fullname" . }}-webhook-service namespace: {{ .Release.Namespace }} - path: /validate-core-v1-pod - failurePolicy: {{ if .Values.ignoreError -}}Ignore{{- else -}}Fail{{end}} - name: vpod.pod-graceful-drain.io + path: /webhook/validate + failurePolicy: Ignore + name: validate.pod-graceful-drain.io rules: - - apiGroups: [""] - apiVersions: [v1] - operations: [DELETE] - resources: [pods] - sideEffects: NoneOnDryRun - timeoutSeconds: {{ .Values.webhookTimeoutSeconds }} + - apiGroups: [ "" ] + apiVersions: [ v1 ] + operations: [ DELETE ] + resources: [ pods ] + sideEffects: None + timeoutSeconds: {{ template "pod-graceful-drain.timeoutSeconds" . }} {{- with .Values.namespaceSelector }} namespaceSelector: {{- toYaml . | nindent 6 }} @@ -45,7 +45,7 @@ metadata: labels: {{- include "pod-graceful-drain.labels" . | nindent 4 }} webhooks: - - admissionReviewVersions: [v1beta1, v1] + - admissionReviewVersions: [ v1beta1, v1 ] clientConfig: {{- if not .Values.enableCertManager }} caBundle: {{ $tls.caCert }} @@ -53,14 +53,14 @@ webhooks: service: name: {{ template "pod-graceful-drain.fullname" . }}-webhook-service namespace: {{ .Release.Namespace }} - path: /mutate-core-v1-pod-eviction - failurePolicy: {{ if .Values.ignoreError -}}Ignore{{- else -}}Fail{{end}} - name: mpodseviction.pod-graceful-drain.io + path: /webhook/mutate + failurePolicy: Ignore + name: mutate.pod-graceful-drain.io rules: - - apiGroups: [""] - apiVersions: [v1] - operations: [CREATE] - resources: [pods/eviction] + - apiGroups: [ "" ] + apiVersions: [ v1 ] + operations: [ CREATE ] + resources: [ pods/eviction ] sideEffects: NoneOnDryRun {{- with .Values.namespaceSelector }} namespaceSelector: @@ -106,5 +106,5 @@ metadata: labels: {{- include "pod-graceful-drain.labels" . | nindent 4 }} spec: - selfSigned: {} + selfSigned: { } {{- end }} diff --git a/charts/pod-graceful-drain/values.yaml b/charts/pod-graceful-drain/values.yaml index 2329630..9f4f4cd 100644 --- a/charts/pod-graceful-drain/values.yaml +++ b/charts/pod-graceful-drain/values.yaml @@ -10,7 +10,7 @@ image: # Overrides the image tag whose default is the chart appVersion. tag: "" -imagePullSecrets: [] +imagePullSecrets: [ ] nameOverride: "" fullnameOverride: "" @@ -18,17 +18,17 @@ serviceAccount: # Specifies whether a service account should be created create: true # Annotations to add to the service account - annotations: {} + annotations: { } # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" -podAnnotations: {} +podAnnotations: { } -podLabels: {} +podLabels: { } -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: { } +# fsGroup: 2000 securityContext: readOnlyRootFilesystem: true @@ -39,10 +39,19 @@ securityContext: # drop: # - ALL -resources: {} -nodeSelector: {} -tolerations: [] -affinity: {} +resources: + requests: + cpu: "100m" + # 10MB at rest. + memory: "50Mi" + limits: + # main runtime is single-threaded. + cpu: "1000m" + memory: "500Mi" + +nodeSelector: { } +tolerations: [ ] +affinity: { } # Enable cert-manager enableCertManager: false @@ -52,16 +61,9 @@ metrics: # Set the manager log level: info, debug (default: info) logLevel: -# Amount of time that a pod is deleted after a denial of an admission (default: 90s) -deleteAfter: -# Delay a pod deletion by only delaying an admission without denying it (default: false) -noDenyAdmission: -# Allow pod deletion even if there were errors during the pod deletion interception (default: true) -ignoreError: true +# Amount of time that a pod is deleted after a denial of an admission (default: 20s, max: 25s) +deleteAfter: 20s +experimentalGeneralIngress: false -# Time period for the controller pod to do a graceful shutdown. It should be greater than deleteAfter -terminationGracePeriodSeconds: 100 -# Timeout for the ValidatingAdmissionWebhook. It should be long enough since admission delay cap is determined by this value -webhookTimeoutSeconds: 30 -# webhook's namespaceSelector to limit where the -namespaceSelector: {} +# webhook's namespaceSelector to limit where the pod-graceful-drain is applied +namespaceSelector: { } diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 21a3775..0000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned-issuer -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml -spec: - # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize - dnsNames: - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc - - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local - issuerRef: - kind: Issuer - name: selfsigned-issuer - secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a..0000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index 90d7c31..0000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml deleted file mode 100644 index 09663bd..0000000 --- a/config/default/kustomization.yaml +++ /dev/null @@ -1,56 +0,0 @@ -namespace: kube-system - -namePrefix: pod-graceful-drain- - -commonLabels: - app.kubernetes.io/name: pod-graceful-drain - -bases: -- ../rbac -- ../manager -- ../webhook -- ../certmanager -# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -#- ../prometheus - -patchesStrategicMerge: - # Protect the /metrics endpoint by putting it behind auth. - # If you want your controller-manager to expose the /metrics - # endpoint w/o any authn/z, please comment the following line. -- manager_webhook_patch.yaml -- webhookcainjection_patch.yaml - -patchesJson6902: -- target: - version: v1 - kind: Deployment - name: manager - path: manager_auth_proxy_patch.yaml - # the following config is for teaching kustomize how to do var substitution -vars: -- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml - fieldref: - fieldpath: metadata.namespace -- name: CERTIFICATE_NAME - objref: - kind: Certificate - group: cert-manager.io - version: v1 - name: serving-cert # this name should match the one in certificate.yaml -- name: SERVICE_NAMESPACE # namespace of the service - objref: - kind: Service - version: v1 - name: webhook-service - fieldref: - fieldpath: metadata.namespace -- name: SERVICE_NAME - objref: - kind: Service - version: v1 - name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4150cfb..0000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -- op: add - path: /spec/template/spec/containers/0/args/- - value: "--metrics-bind-address=127.0.0.1:8080" -- op: add - path: /spec/template/spec/containers/- - value: - name: kube-rbac-proxy - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=10" - ports: - - containerPort: 8443 - name: https \ No newline at end of file diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml deleted file mode 100644 index 5bf8ca3..0000000 --- a/config/default/manager_webhook_patch.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager -spec: - template: - spec: - containers: - - name: manager - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml deleted file mode 100644 index 3ff9d8f..0000000 --- a/config/default/webhookcainjection_patch.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This patch add annotation to admission webhook config and -# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml deleted file mode 100644 index b175699..0000000 --- a/config/manager/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -resources: -- manager.yaml - -patchesStrategicMerge: -- security_context_patch.yaml - -images: -- name: manager - newName: ghcr.io/foriequal0/pod-graceful-drain \ No newline at end of file diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml deleted file mode 100644 index d6dd46f..0000000 --- a/config/manager/manager.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: manager - labels: - app.kubernetes.io/component: manager ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager - labels: - app.kubernetes.io/component: manager -spec: - selector: - matchLabels: - app.kubernetes.io/component: manager - replicas: 1 - template: - metadata: - labels: - app.kubernetes.io/component: manager - spec: - containers: - - name: manager - image: manager:latest - command: - - /manager - args: - - --delete-after=90s - - --no-deny-admission=false - - --ignore-error - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - terminationGracePeriodSeconds: 100 # should be greater than --delete-after and --admission-delay - serviceAccountName: manager diff --git a/config/manager/security_context_patch.yaml b/config/manager/security_context_patch.yaml deleted file mode 100644 index a8f25f1..0000000 --- a/config/manager/security_context_patch.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager -spec: - template: - spec: - containers: - - name: manager - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - runAsNonRoot: true diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml deleted file mode 100644 index ed13716..0000000 --- a/config/prometheus/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml deleted file mode 100644 index 5d2efa7..0000000 --- a/config/prometheus/monitor.yaml +++ /dev/null @@ -1,15 +0,0 @@ - -# Prometheus Monitor Service (Metrics) -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - app.kubernetes.io/component: manager - name: manager-metrics-monitor -spec: - endpoints: - - path: /metrics - port: https - selector: - matchLabels: - app.kubernetes.io/component: manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index bd4af13..0000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: ["/metrics"] - verbs: ["get"] diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 618f5e4..0000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: ["authentication.k8s.io"] - resources: - - tokenreviews - verbs: ["create"] -- apiGroups: ["authorization.k8s.io"] - resources: - - subjectaccessreviews - verbs: ["create"] diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index 39b0871..0000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: manager diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 55e0876..0000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: manager - name: manager-metrics-service -spec: - ports: - - name: https - port: 8443 - targetPort: https - selector: - app.kubernetes.io/component: manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml deleted file mode 100644 index ef03319..0000000 --- a/config/rbac/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -resources: -- role.yaml -- role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml deleted file mode 100644 index 2f96864..0000000 --- a/config/rbac/role.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manager-role -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - delete - - get - - list - - patch - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - elbv2.k8s.aws - resources: - - targetgroupbindings - verbs: - - list - - watch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml deleted file mode 100644 index b14caec..0000000 --- a/config/rbac/role_binding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manager-role -subjects: -- kind: ServiceAccount - name: manager diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml deleted file mode 100644 index 96e798a..0000000 --- a/config/webhook/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -resources: -- manifests.yaml -- service.yaml - -configurations: -- kustomizeconfig.yaml - -patchesStrategicMerge: -- webhook_timeout_patch.yaml -- webhook_namespace_selector_patch.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml deleted file mode 100644 index 25e21e3..0000000 --- a/config/webhook/kustomizeconfig.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# the following config is for teaching kustomize where to look at when substituting vars. -# It requires kustomize v2.1.0 or newer to work properly. -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - - kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/name - -namespace: -- kind: MutatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true -- kind: ValidatingWebhookConfiguration - group: admissionregistration.k8s.io - path: webhooks/clientConfig/service/namespace - create: true - -varReference: -- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml deleted file mode 100644 index ffa6f8f..0000000 --- a/config/webhook/manifests.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - creationTimestamp: null - name: mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-core-v1-pod-eviction - failurePolicy: Ignore - name: mpodseviction.pod-graceful-drain.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods/eviction - sideEffects: NoneOnDryRun ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - creationTimestamp: null - name: validating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-core-v1-pod - failurePolicy: Ignore - name: vpod.pod-graceful-drain.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - pods - sideEffects: NoneOnDryRun diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml deleted file mode 100644 index 3de8350..0000000 --- a/config/webhook/service.yaml +++ /dev/null @@ -1,11 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: webhook-service -spec: - ports: - - port: 443 - targetPort: 9443 - selector: - app.kubernetes.io/component: manager diff --git a/config/webhook/webhook_namespace_selector_patch.yaml b/config/webhook/webhook_namespace_selector_patch.yaml deleted file mode 100644 index 6d8c130..0000000 --- a/config/webhook/webhook_namespace_selector_patch.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration -webhooks: - - name: mpodseviction.pod-graceful-drain.io - namespaceSelector: - matchExpressions: - - key: pod-graceful-drain - operator: In - values: - - enabled ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration -webhooks: - - name: vpod.pod-graceful-drain.io - namespaceSelector: - matchExpressions: - - key: pod-graceful-drain - operator: In - values: - - enabled \ No newline at end of file diff --git a/config/webhook/webhook_timeout_patch.yaml b/config/webhook/webhook_timeout_patch.yaml deleted file mode 100644 index 65cbc18..0000000 --- a/config/webhook/webhook_timeout_patch.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration -webhooks: - - name: vpod.pod-graceful-drain.io - timeoutSeconds: 30 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration -webhooks: - - name: mpodseviction.pod-graceful-drain.io - timeoutSeconds: 30 diff --git a/docker/build.Dockerfile b/docker/build.Dockerfile new file mode 100644 index 0000000..1887c7a --- /dev/null +++ b/docker/build.Dockerfile @@ -0,0 +1,11 @@ +FROM rust:1.79-slim-bookworm as builder +RUN apt-get update && apt-get install -y git +RUN mkdir /src +WORKDIR /src +COPY . /src/ +RUN cargo install --path . + +FROM debian:bookworm-slim +WORKDIR /app +COPY --from=builder /usr/local/cargo/bin/pod-graceful-drain /app/pod-graceful-drain +ENTRYPOINT ["/app/pod-graceful-drain"] diff --git a/docker/build.Dockerfile.dockerignore b/docker/build.Dockerfile.dockerignore new file mode 100644 index 0000000..9fea67c --- /dev/null +++ b/docker/build.Dockerfile.dockerignore @@ -0,0 +1,16 @@ +/.github/ +/charts/ +/tests/ +/.gitignore +/LICENSE +/README.md +/.python-version +/pyproject.toml +/requirements.lock +/requirements-dev.lock +/skaffold.yaml + +/target/ +/.venv/ + +/.idea/ diff --git a/docker/skaffold.Dockerfile b/docker/skaffold.Dockerfile new file mode 100644 index 0000000..b2970dd --- /dev/null +++ b/docker/skaffold.Dockerfile @@ -0,0 +1,26 @@ +FROM rust:1.79-slim-bookworm AS cache + +# prepare git cli +RUN apt-get update && apt-get install -y git + +# create /src dir +RUN mkdir /src +WORKDIR /src + +# warm-up dependencies build cache +COPY ./Cargo.* /src/ +COPY build.rs /src/ +RUN mkdir src && \ + echo 'fn main() { println!("Hello, world!"); }' > src/main.rs && \ + cargo build && \ + rm -rf src + +FROM cache AS build + +COPY . /src/ +RUN cargo build + +FROM debian:bookworm-slim +WORKDIR /app +COPY --from=build /src/target/debug/pod-graceful-drain /app/pod-graceful-drain +ENTRYPOINT ["/app/pod-graceful-drain"] diff --git a/docker/skaffold.Dockerfile.dockerignore b/docker/skaffold.Dockerfile.dockerignore new file mode 100644 index 0000000..59b7a58 --- /dev/null +++ b/docker/skaffold.Dockerfile.dockerignore @@ -0,0 +1,17 @@ +/.git/ +/.github/ +/charts/ +/tests/ +/.gitignore +/LICENSE +/README.md +/.python-version +/pyproject.toml +/requirements.lock +/requirements-dev.lock +/skaffold.yaml + +/.venv/ +/target/ + +/.idea/ diff --git a/go.mod b/go.mod deleted file mode 100644 index e806a8c..0000000 --- a/go.mod +++ /dev/null @@ -1,64 +0,0 @@ -module github.com/foriequal0/pod-graceful-drain - -go 1.18 - -require ( - github.com/go-logr/logr v1.2.3 - github.com/pkg/errors v0.9.1 - go.uber.org/zap v1.21.0 - gomodules.xyz/jsonpatch/v2 v2.2.0 - gotest.tools/v3 v3.1.0 - k8s.io/api v0.23.5 - k8s.io/apimachinery v0.23.5 - k8s.io/client-go v0.23.5 - sigs.k8s.io/aws-load-balancer-controller v0.0.0-20220316010148-c4471defda10 // v2.4.1 - sigs.k8s.io/controller-runtime v0.11.2 -) - -require ( - cloud.google.com/go v0.81.0 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/go-logr/zapr v1.2.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.6 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.2.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/prometheus/client_golang v1.11.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.28.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect - golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect - golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.27.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiextensions-apiserver v0.23.5 // indirect - k8s.io/component-base v0.23.5 // indirect - k8s.io/klog/v2 v2.30.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 2894d8a..0000000 --- a/go.sum +++ /dev/null @@ -1,951 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= -github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw= -github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 h1:M69LAlWZCshgp0QSzyDcSsSIejIEeuaCVpmwcKwyLMk= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= -gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= -k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= -k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/aws-load-balancer-controller v0.0.0-20220316010148-c4471defda10 h1:k00CjF5xvSxbAlzjITqh8rSKP8w0a4DiK6NtCYCfoks= -sigs.k8s.io/aws-load-balancer-controller v0.0.0-20220316010148-c4471defda10/go.mod h1:IgdiNk6qx80CRWsvY2tfmXrpxG6rhkb9p+8oTcaJXzs= -sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= -sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt deleted file mode 100644 index e2fe4d1..0000000 --- a/hack/boilerplate.go.txt +++ /dev/null @@ -1,13 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/internal/config.go b/internal/config.go deleted file mode 100644 index be81e93..0000000 --- a/internal/config.go +++ /dev/null @@ -1,40 +0,0 @@ -package internal - -import ( - "flag" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" -) - -type Config struct { - LogLevel string - MetricsBindAddress string - WebhookBindPort int - - PodGracefulDrain core.PodGracefulDrainConfig -} - -func (c *Config) BindFlags(fs *flag.FlagSet) { - fs.StringVar(&c.LogLevel, "log-level", "info", "Log level: info, debug") - fs.StringVar(&c.MetricsBindAddress, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - fs.IntVar(&c.WebhookBindPort, "webhook-bind-port", 9443, "The port the webhook server serves at.") - - c.PodGracefulDrain.BindFlags(fs) -} - -func (c *Config) BuildManagerOptions(scheme *runtime.Scheme) ctrl.Options { - return ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: c.MetricsBindAddress, - Port: c.WebhookBindPort, - } -} - -func (c *Config) Validate() error { - if err := c.PodGracefulDrain.Validate(); err != nil { - return err - } - - return nil -} diff --git a/internal/pkg/core/config.go b/internal/pkg/core/config.go deleted file mode 100644 index 43a2475..0000000 --- a/internal/pkg/core/config.go +++ /dev/null @@ -1,33 +0,0 @@ -package core - -import ( - "errors" - "flag" - "time" -) - -type PodGracefulDrainConfig struct { - DeleteAfter time.Duration - NoDenyAdmission bool - IgnoreError bool -} - -func (c *PodGracefulDrainConfig) BindFlags(fs *flag.FlagSet) { - fs.DurationVar(&c.DeleteAfter, "delete-after", 90*time.Second, "Amount of time that a pod is deleted after a denial of an admission") - fs.BoolVar(&c.NoDenyAdmission, "no-deny-admission", false, "Delay a pod deletion by only delaying an admission without denying it") - fs.BoolVar(&c.IgnoreError, "ignore-error", true, "Allow pod deletion even if there were errors during the pod deletion interception") -} - -func (c *PodGracefulDrainConfig) Validate() error { - if c.DeleteAfter < time.Duration(0) { - return errors.New("deletion delay cannot be less than 0 (time travelling?)") - } - - if !c.NoDenyAdmission { - if c.DeleteAfter == time.Duration(0) { - return errors.New("deletion delay cannot be 0 when you choose to deny admissions") - } - } - - return nil -} diff --git a/internal/pkg/core/delayer.go b/internal/pkg/core/delayer.go deleted file mode 100644 index 0b6d10b..0000000 --- a/internal/pkg/core/delayer.go +++ /dev/null @@ -1,164 +0,0 @@ -package core - -import ( - "context" - "github.com/go-logr/logr" - "sync" - "sync/atomic" - "time" -) - -type Delayer interface { - NewTask(duration time.Duration, task func(context.Context, bool) error) DelayedTask - Stop(drain time.Duration, cleanup time.Duration) -} - -type delayer struct { - logger logr.Logger - counter int64 - - tasksWaitGroup *sync.WaitGroup - interrupt chan struct{} - cleanup chan struct{} -} - -var _ Delayer = &delayer{} - -func NewDelayer(logger logr.Logger) Delayer { - return &delayer{ - logger: logger.WithName("delayer"), - - tasksWaitGroup: &sync.WaitGroup{}, - interrupt: make(chan struct{}), - cleanup: make(chan struct{}), - } -} - -func (d *delayer) NewTask(duration time.Duration, task func(context.Context, bool) error) DelayedTask { - id := atomic.AddInt64(&d.counter, 1) - - return &delayedTask{ - delayer: d, - logger: d.logger.WithValues("taskId", id), - id: DelayedTaskId(id), - duration: duration, - task: task, - } -} - -func (d *delayer) Stop(drain time.Duration, cleanup time.Duration) { - d.logger.Info("Stopping delayer") - - stopped := make(chan struct{}) - go func() { - d.tasksWaitGroup.Wait() - close(stopped) - }() - - select { - case <-stopped: - d.logger.Info("Drained all delayed tasks") - case <-time.After(drain): - d.logger.Info("Some delayed tasks are not finished in time. Interrupt and wait them to cleanup") - close(d.interrupt) - - select { - case <-stopped: - case <-time.After(cleanup): - } - } - close(d.cleanup) - d.logger.Info("Stopped delayer") -} - -type DelayedTaskId int64 - -type DelayedTask interface { - GetId() DelayedTaskId - GetDuration() time.Duration - RunWait(ctx context.Context) error - RunAsync() -} - -type delayedTask struct { - delayer *delayer - logger logr.Logger - id DelayedTaskId - duration time.Duration - task func(context.Context, bool) error -} - -var _ DelayedTask = &delayedTask{} - -func (t *delayedTask) GetId() DelayedTaskId { - return t.id -} - -func (t *delayedTask) GetDuration() time.Duration { - return t.duration -} - -func (t *delayedTask) RunWait(ctx context.Context) error { - t.delayer.tasksWaitGroup.Add(1) - defer t.delayer.tasksWaitGroup.Done() - - innerCtx, cancel := context.WithCancel(ctx) - defer cancel() - go func() { - select { - case <-innerCtx.Done(): - case <-t.delayer.cleanup: - cancel() - } - }() - - return t.run(innerCtx, t.duration) -} - -func (t *delayedTask) RunAsync() { - t.delayer.tasksWaitGroup.Add(1) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - select { - case <-ctx.Done(): - case <-t.delayer.cleanup: - cancel() - } - }() - - go func() { - defer t.delayer.tasksWaitGroup.Done() - defer cancel() - - err := t.run(ctx, t.duration) - _ = err - }() - - t.logger.V(1).Info("Scheduled delayed task") -} - -func (t *delayedTask) run(ctx context.Context, duration time.Duration) error { - t.logger.Info("Wait timer for", "duration", duration) - - var interrupted bool - select { - case <-ctx.Done(): - interrupted = true - case <-t.delayer.interrupt: - interrupted = true - case <-time.After(duration): - interrupted = false - } - - t.logger.V(1).Info("Start delayed task", "interrupted", interrupted) - - if t.task != nil { - newCtx := logr.NewContext(ctx, t.logger) - - if err := t.task(newCtx, interrupted); err != nil { - t.logger.Error(err, "Delayed task errored") - return err - } - } - return nil -} diff --git a/internal/pkg/core/delayer_test.go b/internal/pkg/core/delayer_test.go deleted file mode 100644 index f64bf90..0000000 --- a/internal/pkg/core/delayer_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package core_test - -import ( - "context" - "errors" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "gotest.tools/v3/assert" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "testing" - "time" -) - -func newDelayer() core.Delayer { - return core.NewDelayer(zap.New()) -} - -type taskProbe struct { - contextErr error - interrupted bool - time time.Time -} - -func newTaskProbe(ctx context.Context, interrupted bool) taskProbe { - return taskProbe{ - contextErr: ctx.Err(), - interrupted: interrupted, - time: time.Now(), - } -} - -const ( - shortDuration = 30 * time.Millisecond - halfDuration = 50 * time.Millisecond - duration = 100 * time.Millisecond - longDuration = 200 * time.Millisecond -) - -func TestDelayedTask_RunAfterWait_ShouldBlock(t *testing.T) { - delayer := newDelayer() - defer delayer.Stop(duration, duration) - probe := make(chan taskProbe, 1) - task := delayer.NewTask(duration, func(ctx context.Context, interrupted bool) error { - probe <- newTaskProbe(ctx, interrupted) - return nil - }) - - _ = task.RunWait(context.TODO()) - - select { - case <-probe: - default: - assert.Assert(t, false, "Task should've ran when RunWait returned") - } -} - -func TestDelayedTask_RunAfterWait_ShouldCancelledAfterTimeout(t *testing.T) { - delayer := newDelayer() - defer delayer.Stop(duration, duration) - probe := make(chan taskProbe, 1) - task := delayer.NewTask(duration, func(ctx context.Context, interrupted bool) error { - probe <- newTaskProbe(ctx, interrupted) - return nil - }) - - ctx, cancel := context.WithTimeout(context.TODO(), shortDuration) - defer cancel() - start := time.Now() - _ = task.RunWait(ctx) - - select { - case probeResult := <-probe: - assert.Equal(t, probeResult.interrupted, true, "Task should be interrupted") - assert.Equal(t, probeResult.contextErr, context.DeadlineExceeded, "Task should be timed out") - assert.Equal(t, start.Sub(probeResult.time) < duration, true, "Task should be started earlier") - default: - assert.Assert(t, false, "Task should've ran when RunWait returned") - } -} - -func TestDelayedTask_RunAfterWait_ShouldPassError(t *testing.T) { - tests := []struct { - name string - err error - }{{"nil", nil}, {"err", errors.New("error")}} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - delayer := newDelayer() - defer delayer.Stop(duration, duration) - err1 := errors.New("error") - task := delayer.NewTask(time.Duration(0), func(ctx context.Context, _ bool) error { - return err1 - }) - - err2 := task.RunWait(context.TODO()) - - assert.Equal(t, err1, err2, "RunWait should return what task have returned") - }) - } -} - -func TestDelayedTask_RunAfterWait_ShouldNotBlock(t *testing.T) { - delayer := newDelayer() - defer delayer.Stop(duration, duration) - probe := make(chan taskProbe, 1) - task := delayer.NewTask(duration, func(ctx context.Context, interrupted bool) error { - probe <- newTaskProbe(ctx, interrupted) - <-ctx.Done() - return nil - }) - - task.RunAsync() - - select { - case <-probe: - assert.Assert(t, false, "Task shouldn't have ran when RunAsync returned") - default: - } -} - -func TestDelayedTask_NoInterruptDrain_WhenDelayIsShortEnough(t *testing.T) { - givenDelay := duration - givenDrain := longDuration - - tests := []struct { - name string - runner func(task core.DelayedTask) - }{ - {"RunWait", func(task core.DelayedTask) { _ = task.RunWait(context.TODO()) }}, - {"RunAsync", func(task core.DelayedTask) { task.RunAsync() }}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - delayer := newDelayer() - probe := make(chan taskProbe, 1) - task := delayer.NewTask(givenDelay, func(ctx context.Context, interrupted bool) error { - probe <- newTaskProbe(ctx, interrupted) // probeResult - return nil - }) - - stopperChan := make(chan time.Time, 2) - go func() { - stopperChan <- time.Now() // stopperStart - delayer.Stop(givenDrain, duration) - stopperChan <- time.Now() // stopperEnd - }() - - start := time.Now() - tt.runner(task) - - stopperStart := <-stopperChan - probeResult := <-probe - stopperEnd := <-stopperChan - - delay := probeResult.time.Sub(start) - stop := stopperEnd.Sub(stopperStart) - - assert.Equal(t, probeResult.interrupted, false, "Task should not be interrupted") - assert.Assert(t, delay >= givenDelay, "Task should be delayed enough") - assert.Assert(t, stop <= givenDrain, "delayer should stop as soon as all tasks are drained") - }) - } -} - -func TestDelayedTask_InterruptedDrain_WhenDelayIsTooLong(t *testing.T) { - givenDelay := longDuration - givenDrain := duration - givenCleanup := halfDuration - givenStop := givenDrain + givenCleanup - - tests := []struct { - name string - runner func(task core.DelayedTask) - }{ - {"RunWait", func(task core.DelayedTask) { _ = task.RunWait(context.Background()) }}, - {"RunAsync", func(task core.DelayedTask) { task.RunAsync() }}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - delayer := newDelayer() - probe := make(chan taskProbe, 2) - task := delayer.NewTask(givenDelay, func(ctx context.Context, interrupted bool) error { - probe <- newTaskProbe(ctx, interrupted) // interrupted - <-ctx.Done() - probe <- newTaskProbe(ctx, interrupted) // cancelled - return nil - }) - - stopperChan := make(chan time.Time, 2) - go func() { - stopperChan <- time.Now() // stopperStart - delayer.Stop(givenDrain, givenCleanup) - stopperChan <- time.Now() // stopperEnd - }() - - start := time.Now() - tt.runner(task) - - stopperStart := <-stopperChan - interrupted := <-probe - cancelled := <-probe - stopperEnd := <-stopperChan - - delay := interrupted.time.Sub(start) - drain := interrupted.time.Sub(stopperStart) - cleanup := cancelled.time.Sub(interrupted.time) - stop := stopperEnd.Sub(stopperStart) - - assert.Equal(t, interrupted.interrupted, true, "Task should be interrupted") - assert.NilError(t, interrupted.contextErr, "Task should not be cancelled yet") - assert.Equal(t, cancelled.contextErr, context.Canceled, "Task should be cancelled") - - assert.Assert(t, delay < givenDelay, "Task should be started earlier") - assert.Assert(t, similar(drain, givenDrain, 0.1), "delayer should allow tasks delayed up to drain period") - assert.Assert(t, similar(cleanup, givenCleanup, 0.1), "delayer should wait task to finish") - - assert.Assert(t, similar(stop, givenStop, 0.1), "delayer should stop in time") - }) - } -} - -func TestDelayedTask_EmptyStop(t *testing.T) { - givenDelay := shortDuration - stopSleep := duration - - tests := []struct { - name string - runner func(task core.DelayedTask, delay time.Duration) - }{ - {"RunWait", func(task core.DelayedTask, delay time.Duration) { _ = task.RunWait(context.Background()) }}, - {"RunAsync", func(task core.DelayedTask, delay time.Duration) { task.RunAsync() }}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - delayer := newDelayer() - task := delayer.NewTask(givenDelay, nil) - - stopperChan := make(chan time.Time, 2) - go func() { - time.Sleep(stopSleep) - stopperChan <- time.Now() // stopperStart - delayer.Stop(duration, duration) - stopperChan <- time.Now() // stopperEnd - }() - - tt.runner(task, givenDelay) - - stopperStart := <-stopperChan - stopperEnd := <-stopperChan - - stop := stopperEnd.Sub(stopperStart) - - assert.Assert(t, stop < shortDuration, "delayer should stop immediately") - }) - } -} -func similar(x time.Duration, y time.Duration, toleration float64) bool { - xs := x.Seconds() - ys := y.Seconds() - return xs >= ys*(1.0-toleration) && xs <= ys*(1.0+toleration) -} diff --git a/internal/pkg/core/intercepted_admission_response.go b/internal/pkg/core/intercepted_admission_response.go deleted file mode 100644 index 59dd7d1..0000000 --- a/internal/pkg/core/intercepted_admission_response.go +++ /dev/null @@ -1,55 +0,0 @@ -package core - -import ( - "encoding/json" - "github.com/pkg/errors" - "gomodules.xyz/jsonpatch/v2" - policyv1 "k8s.io/api/policy/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -type InterceptedAdmissionResponse interface { - GetAdmissionResponse() admission.Response -} - -type AdmissionResponse struct { - Allow bool - Reason string -} - -func (r AdmissionResponse) GetAdmissionResponse() admission.Response { - if r.Allow { - return admission.Allowed(r.Reason) - } - return admission.Denied(r.Reason) -} - -type EvictionResponse struct { - Operations []jsonpatch.Operation -} - -func NewEvictionResponse(eviction *policyv1.Eviction) (EvictionResponse, error) { - oldJson, err := json.Marshal(eviction) - if err != nil { - return EvictionResponse{}, errors.Wrap(err, "unable to marshal old eviction") - } - modified := eviction.DeepCopy() - if modified.DeleteOptions == nil { - modified.DeleteOptions = &metav1.DeleteOptions{} - } - modified.DeleteOptions.DryRun = append(modified.DeleteOptions.DryRun, "All") - modifiedJson, err := json.Marshal(modified) - if err != nil { - return EvictionResponse{}, errors.Wrap(err, "unable to marshal old eviction") - } - operations, err := jsonpatch.CreatePatch(oldJson, modifiedJson) - if err != nil { - return EvictionResponse{}, errors.Wrap(err, "unable to create jsonpatch") - } - return EvictionResponse{Operations: operations}, nil -} - -func (r EvictionResponse) GetAdmissionResponse() admission.Response { - return admission.Patched("Pod eviction is intercepted", r.Operations...) -} diff --git a/internal/pkg/core/interceptor.go b/internal/pkg/core/interceptor.go deleted file mode 100644 index 6588821..0000000 --- a/internal/pkg/core/interceptor.go +++ /dev/null @@ -1,50 +0,0 @@ -package core - -import ( - "context" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -type Interceptor struct { - drain *PodGracefulDrain - k8sClient client.Client -} - -func NewInterceptor(drain *PodGracefulDrain, k8sClient client.Client) Interceptor { - return Interceptor{ - drain: drain, - k8sClient: k8sClient, - } -} - -func (i *Interceptor) InterceptPodDeletion(ctx context.Context, req *admission.Request, pod *corev1.Pod) (InterceptedAdmissionResponse, error) { - if req.DryRun != nil && *req.DryRun == true { - return AdmissionResponse{Allow: true, Reason: "dry-run"}, nil - } - - interceptedResponse, err := i.drain.DelayPodDeletion(ctx, pod) - if err != nil { - return nil, err - } - return interceptedResponse, nil -} - -func (i *Interceptor) InterceptPodEviction(ctx context.Context, req *admission.Request, eviction *policyv1.Eviction) (InterceptedAdmissionResponse, error) { - if req.DryRun != nil && *req.DryRun == true { - return AdmissionResponse{Allow: true, Reason: "dry-run"}, nil - } - - intercepted, err := i.drain.DelayPodEviction(ctx, eviction) - if err != nil || !intercepted { - return nil, err - } - - response, err := NewEvictionResponse(eviction) - if err != nil { - return nil, err - } - return response, nil -} diff --git a/internal/pkg/core/pod_graceful_drain.go b/internal/pkg/core/pod_graceful_drain.go deleted file mode 100644 index e00917c..0000000 --- a/internal/pkg/core/pod_graceful_drain.go +++ /dev/null @@ -1,399 +0,0 @@ -package core - -import ( - "context" - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" - "time" -) - -const ( - fallbackAdmissionDelayTimeout = 30 * time.Second - admissionDelayOverhead = 2 * time.Second - defaultPodGracefulDrainCleanupTimeout = 10 * time.Second -) - -type PodGracefulDrain struct { - client client.Client - logger logr.Logger - config *PodGracefulDrainConfig - delayer Delayer -} - -var _ manager.Runnable = &PodGracefulDrain{} - -func NewPodGracefulDrain(k8sClient client.Client, logger logr.Logger, config *PodGracefulDrainConfig) PodGracefulDrain { - return PodGracefulDrain{ - client: k8sClient, - logger: logger.WithName("pod-graceful-drain"), - config: config, - delayer: NewDelayer(logger), - } -} - -func (d *PodGracefulDrain) DelayPodDeletion(ctx context.Context, pod *corev1.Pod) (InterceptedAdmissionResponse, error) { - now := time.Now() - logger := d.getLoggerFor(pod) - spec, err := d.getDelayedPodDeletionSpec(ctx, pod, now) - if err != nil || spec == nil { - return nil, err - } - - spec.log(logger) - - if err := spec.execute(ctx, NewPodMutator(d.client, pod).WithLogger(logger)); err != nil { - return nil, err - } - return spec.admission, nil -} - -type delayedPodDeletionSpec struct { - isolate bool - deleteAt time.Time - asyncDeleteTask DelayedTask - sleepTask DelayedTask - reason string - admission AdmissionResponse -} - -func (d *PodGracefulDrain) getDelayedPodDeletionSpec(ctx context.Context, pod *corev1.Pod, now time.Time) (spec *delayedPodDeletionSpec, err error) { - if !IsPodReady(pod) { - return nil, nil - } - - delayInfo, err := GetPodDeletionDelayInfo(pod) - if err != nil { - return nil, errors.Wrapf(err, "unable to get pod deletion info") - } else if delayInfo.Isolated { - spec, err := d.getReentrySpec(ctx, pod, delayInfo, now) - if err != nil { - return nil, errors.Wrapf(err, "unable to getPodDelayedRemoveSpec pod deletion reentry") - } - return spec, nil - } - - hadServiceTargetTypeIP, err := DidPodHaveServicesTargetTypeIP(ctx, d.client, pod) - if err != nil { - return nil, errors.Wrapf(err, "unable to determine whether the pod had service with ip target-type") - } else if !hadServiceTargetTypeIP { - return nil, nil - } - - canDeny, reason, err := d.canDenyAdmission(ctx, pod) - if err != nil { - return nil, errors.Wrap(err, "unable to determine whether it can be denied") - } else if canDeny { - spec = &delayedPodDeletionSpec{ - isolate: true, - deleteAt: now.Add(d.config.DeleteAfter), - asyncDeleteTask: d.getDelayedPodDeletionTask(pod, d.config.DeleteAfter), - reason: reason, - admission: AdmissionResponse{ - Allow: false, - Reason: "Pod cannot be removed immediately. It will be eventually removed after waiting for the load balancer to start", - }, - } - } else { - deleteAfter := getAdmissionDelayTimeout(ctx, now) - spec = &delayedPodDeletionSpec{ - isolate: true, - deleteAt: now.Add(deleteAfter), - sleepTask: d.getSleepTask(deleteAfter), - reason: reason, - admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough", - }, - } - } - return -} - -func getAdmissionDelayTimeout(ctx context.Context, now time.Time) time.Duration { - timeout := fallbackAdmissionDelayTimeout - if deadline, ok := ctx.Deadline(); ok { - timeout = deadline.Sub(now) - admissionDelayOverhead - if timeout < 0 { - timeout = time.Duration(0) - } - } - return timeout -} - -func (s *delayedPodDeletionSpec) log(logger logr.Logger) { - details := map[string]interface{}{} - if s.isolate { - details["isolate"] = map[string]interface{}{ - "deleteAt": s.deleteAt, - } - } - if s.asyncDeleteTask != nil { - details["asyncDelete"] = map[string]interface{}{ - "taskId": s.asyncDeleteTask.GetId(), - "duration": s.asyncDeleteTask.GetDuration().Seconds(), - } - } - if s.sleepTask != nil { - details["sleep"] = map[string]interface{}{ - "taskId": s.sleepTask.GetId(), - "duration": s.sleepTask.GetDuration().Seconds(), - } - } - - logger.Info("delayed pod remove spec", - "details", details, - "reason", s.reason, - "admission", s.admission.Allow) -} - -func (s *delayedPodDeletionSpec) execute(ctx context.Context, m *PodMutator) error { - if s.isolate { - if err := m.Isolate(ctx, s.deleteAt); err != nil { - return errors.Wrap(err, "unable to isolate the pod") - } - } - - if s.asyncDeleteTask != nil { - s.asyncDeleteTask.RunAsync() - } - - if s.sleepTask != nil { - if err := s.sleepTask.RunWait(ctx); err != nil { - return err - } - } - return nil -} - -// getReentrySpec handles these cases: -// * apiserver immediately retried the deletion when we patched the pod and denied the admission -// since it is indistinguishable from the collision. So it should keep deny. -// * We disabled wait sentinel label and deleted the pod, but the patch hasn't been propagated fast enough -// so ValidatingAdmissionWebhook read the wait label of the old version -// => deletePodAfter will retry with back-offs, so we keep denying the admission. -// * Users and controllers manually tries to delete the pod before deleteAt. -// => User can see the admission report message. Controller should getDelayedPodDeletionSpec admission failures. -func (d *PodGracefulDrain) getReentrySpec(ctx context.Context, pod *corev1.Pod, info PodDeletionDelayInfo, now time.Time) (spec *delayedPodDeletionSpec, err error) { - remainingTime := info.GetRemainingTime(now) - if remainingTime == time.Duration(0) { - return nil, nil - } - - canDeny, reason, err := d.canDenyAdmission(ctx, pod) - if err != nil { - return nil, errors.Wrap(err, "cannot determine whether it should be denied") - } else if canDeny { - spec = &delayedPodDeletionSpec{ - reason: reason, - admission: AdmissionResponse{ - Allow: false, - Reason: "Pod cannot be removed immediately. It will be eventually removed after waiting for the load balancer to start (reentry)", - }, - } - } else { - timeout := getAdmissionDelayTimeout(ctx, now) - if remainingTime > timeout { - remainingTime = timeout - } - // All admissions should be delayed. Pods will be deleted if any of admissions is finished. - spec = &delayedPodDeletionSpec{ - sleepTask: d.getSleepTask(remainingTime), - reason: reason, - admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough (reentry)", - }, - } - } - return -} - -// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch - -func (d *PodGracefulDrain) canDenyAdmission(ctx context.Context, pod *corev1.Pod) (bool, string, error) { - if d.config.NoDenyAdmission { - return false, "no-deny-admission config", nil - } - - // `kubectl drain` will fail and stop if it meets the first pod that cannot be deleted. - // It'll cordon a node before draining, so we detect it, and try not to deny the admission. - draining, err := IsPodInDrainingNode(ctx, d.client, pod) - if err != nil { - return false, "", nil - } else if draining { - return false, "node might be draining", nil - } - return true, "default", nil -} - -// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch - -func (d *PodGracefulDrain) DelayPodEviction(ctx context.Context, eviction *policyv1.Eviction) (bool, error) { - now := time.Now() - logger := d.getLoggerFor(eviction) - - podKey := types.NamespacedName{ - Namespace: eviction.Namespace, - Name: eviction.Name, - } - pod := &corev1.Pod{} - if err := d.client.Get(ctx, podKey, pod); err != nil { - return false, errors.Wrapf(err, "unable to get the pod") - } - - spec, err := d.getDelayedPodEvictionSpec(ctx, pod, now) - if err != nil || spec == nil { - return false, err - } - - spec.log(logger) - - if err := spec.execute(ctx, NewPodMutator(d.client, pod).WithLogger(logger)); err != nil { - return false, err - } - - return true, nil -} - -type delayedPodEvictionSpec struct { - isolate bool - deleteAt time.Time - asyncDeleteTask DelayedTask -} - -func (d *PodGracefulDrain) getDelayedPodEvictionSpec(ctx context.Context, pod *corev1.Pod, now time.Time) (spec *delayedPodEvictionSpec, err error) { - if !IsPodReady(pod) { - return nil, nil - } - - delayInfo, err := GetPodDeletionDelayInfo(pod) - if err != nil { - return nil, errors.Wrapf(err, "unable to get pod deletion info") - } else if delayInfo.Isolated { - remainingTime := delayInfo.GetRemainingTime(now) - if remainingTime == time.Duration(0) { - return nil, nil - } - - // reentry - return &delayedPodEvictionSpec{}, nil - } - - hadServiceTargetTypeIP, err := DidPodHaveServicesTargetTypeIP(ctx, d.client, pod) - if err != nil { - return nil, errors.Wrapf(err, "unable to determine whether the pod had service with ip target-type") - } else if !hadServiceTargetTypeIP { - return nil, nil - } - - spec = &delayedPodEvictionSpec{ - isolate: true, - deleteAt: now.Add(d.config.DeleteAfter), - asyncDeleteTask: d.getDelayedPodDeletionTask(pod, d.config.DeleteAfter), - } - return -} - -func (s *delayedPodEvictionSpec) log(logger logr.Logger) { - details := map[string]interface{}{} - if s.isolate { - details["isolate"] = map[string]interface{}{ - "deleteAt": s.deleteAt, - } - } - if s.asyncDeleteTask != nil { - details["asyncDelete"] = map[string]interface{}{ - "taskId": s.asyncDeleteTask.GetId(), - "duration": s.asyncDeleteTask.GetDuration().Seconds(), - } - } - - logger.Info("delayed pod eviction spec", - "details", details) -} - -func (s *delayedPodEvictionSpec) execute(ctx context.Context, m *PodMutator) error { - if s.isolate { - if err := m.Isolate(ctx, s.deleteAt); err != nil { - return errors.Wrap(err, "unable to isolate the pod") - } - } - - if s.asyncDeleteTask != nil { - s.asyncDeleteTask.RunAsync() - } - - return nil -} - -func (d *PodGracefulDrain) Start(ctx context.Context) error { - d.logger.Info("starting pod-graceful-drain") - if err := d.cleanupPreviousRun(ctx); err != nil { - d.logger.Error(err, "error while cleaning pods up that are not removed in the previous run") - } - - <-ctx.Done() - - d.logger.Info("stopping pod-graceful-drain") - - drainTimeout := fallbackAdmissionDelayTimeout - if drainTimeout < d.config.DeleteAfter { - drainTimeout = d.config.DeleteAfter - } - - d.delayer.Stop(drainTimeout, defaultPodGracefulDrainCleanupTimeout) - d.logger.V(1).Info("stopped pod-graceful-drain") - return nil -} - -// +kubebuilder:rbac:groups="",resources=pods,verbs=list;watch - -func (d *PodGracefulDrain) cleanupPreviousRun(ctx context.Context) error { - podList := &corev1.PodList{} - // select all pods regardless of its value. These pods were about to be deleted anyway when its value is empty. - if err := d.client.List(ctx, podList, client.HasLabels{WaitLabelKey}); err != nil { - return errors.Wrapf(err, "cannot list pods with wait sentinel label") - } - - now := time.Now() - for idx := range podList.Items { - pod := &podList.Items[idx] - - deleteAfter := d.config.DeleteAfter - delayInfo, err := GetPodDeletionDelayInfo(pod) - if err != nil { - d.getLoggerFor(pod).Error(err, "cannot get pod deletion delay info, but it has wait sentinel label") - } else { - deleteAfter = delayInfo.GetRemainingTime(now) - } - - d.getDelayedPodDeletionTask(pod, deleteAfter).RunAsync() - } - return nil -} - -func (d *PodGracefulDrain) getLoggerFor(obj client.Object) logr.Logger { - namespacedName := types.NamespacedName{ - Namespace: obj.GetNamespace(), - Name: obj.GetName(), - } - - return d.logger.WithValues(obj.GetObjectKind().GroupVersionKind().Kind, namespacedName.String()) -} - -func (d *PodGracefulDrain) getDelayedPodDeletionTask(pod *corev1.Pod, duration time.Duration) DelayedTask { - return d.delayer.NewTask(duration, func(ctx context.Context, _ bool) error { - return NewPodMutator(d.client, pod). - WithLogger(logr.FromContextOrDiscard(ctx)). - DisableWaitLabelAndDelete(ctx) - }) -} - -func (d *PodGracefulDrain) getSleepTask(duration time.Duration) DelayedTask { - return d.delayer.NewTask(duration, nil) -} diff --git a/internal/pkg/core/pod_graceful_drain_test.go b/internal/pkg/core/pod_graceful_drain_test.go deleted file mode 100644 index 7ed8ae7..0000000 --- a/internal/pkg/core/pod_graceful_drain_test.go +++ /dev/null @@ -1,625 +0,0 @@ -package core - -import ( - "context" - "fmt" - "gotest.tools/v3/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - elbv2 "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "testing" - "time" -) - -var ( - deleteAfter = 60 * time.Second - contextTimeout = 10 * time.Second -) - -var ( - defaultConfig = PodGracefulDrainConfig{ - DeleteAfter: deleteAfter, - NoDenyAdmission: false, - } - noDenyConfig = PodGracefulDrainConfig{ - NoDenyAdmission: true, - } -) - -func TestDelayedPodDeletionSpec(t *testing.T) { - now := time.Now().Truncate(time.Second) - deleteAt := now.UTC().Add(deleteAfter).Format(time.RFC3339) - - readyStatus := corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - } - boundPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "selector-label": "selector-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - readinessGatePod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "irrelevant-label": "irrelevant-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - ReadinessGates: []corev1.PodReadinessGate{ - {ConditionType: "target-health.elbv2.k8s.aws"}, - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - {Type: corev1.PodConditionType("target-health.elbv2.k8s.aws"), Status: corev1.ConditionTrue}, - }, - }, - } - unboundPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "irrelevant-label": "irrelevant-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - notReadyPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "selector-label": "selector-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - } - isolatedPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAt, - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - nowaitPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": "2006-01-02T15:04:05Z", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - - service := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc", - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "selector-label": "selector-value", - }, - }, - } - - targetTypeIP := elbv2.TargetTypeIP - tgbIP := elbv2.TargetGroupBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tgb", - }, - Spec: elbv2.TargetGroupBindingSpec{ - TargetType: &targetTypeIP, - ServiceRef: elbv2.ServiceReference{Name: "svc"}, - }, - } - targetTypeInstance := elbv2.TargetTypeInstance - tgbInstance := elbv2.TargetGroupBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tgb", - }, - Spec: elbv2.TargetGroupBindingSpec{ - TargetType: &targetTypeInstance, - ServiceRef: elbv2.ServiceReference{Name: "svc"}, - }, - } - - normalNode := corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node", - }, - } - unschedulableNode := corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node", - }, - Spec: corev1.NodeSpec{ - Unschedulable: true, - }, - } - taintedNode := corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - {Key: corev1.TaintNodeUnschedulable}, - }, - }, - } - - type wantedSpec struct { - Isolate bool - DeleteAt time.Time - AsyncDeleteTaskDuration time.Duration - SleepTaskDuration time.Duration - Reason string - Admission AdmissionResponse - } - - tests := []struct { - name string - existing []runtime.Object - config []PodGracefulDrainConfig - given *corev1.Pod - timeout *time.Duration - want *wantedSpec - }{ - { - name: "bound pod should be delayed", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig}, - given: &boundPod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(deleteAfter), - AsyncDeleteTaskDuration: deleteAfter, - Reason: "default", - Admission: AdmissionResponse{ - Allow: false, - Reason: "Pod cannot be removed immediately. It will be eventually removed after waiting for the load balancer to start", - }, - }, - }, { - name: "bound pod should be delayed with no-deny", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{noDenyConfig}, - timeout: &contextTimeout, - given: &boundPod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(contextTimeout - admissionDelayOverhead), - SleepTaskDuration: contextTimeout - admissionDelayOverhead, - Reason: "no-deny-admission config", - Admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough", - }, - }, - }, - { - name: "pod with readiness gate should be delayed", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig}, - given: &readinessGatePod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(deleteAfter), - AsyncDeleteTaskDuration: deleteAfter, - Reason: "default", - Admission: AdmissionResponse{ - Allow: false, - Reason: "Pod cannot be removed immediately. It will be eventually removed after waiting for the load balancer to start", - }, - }, - }, - { - name: "pod with readiness gate should be delayed with no-deny", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{noDenyConfig}, - timeout: &contextTimeout, - given: &readinessGatePod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(contextTimeout - admissionDelayOverhead), - SleepTaskDuration: contextTimeout - admissionDelayOverhead, - Reason: "no-deny-admission config", - Admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough", - }, - }, - }, - { - name: "unbound pod is deleted immediately", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig, noDenyConfig}, - given: &unboundPod, - want: nil, - }, - { - name: "Isolated pod should be delayed, again", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig}, - given: &isolatedPod, - want: &wantedSpec{ - Reason: "default", - Admission: AdmissionResponse{ - Allow: false, - Reason: "Pod cannot be removed immediately. It will be eventually removed after waiting for the load balancer to start (reentry)", - }, - }, - }, - { - name: "Isolated pod should be delayed, again with no-deny", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{noDenyConfig}, - timeout: &contextTimeout, - given: &isolatedPod, - want: &wantedSpec{ - SleepTaskDuration: contextTimeout - admissionDelayOverhead, - Reason: "no-deny-admission config", - Admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough (reentry)", - }, - }, - }, - { - name: "not ready pod should be deleted immediately", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig, noDenyConfig}, - given: ¬ReadyPod, - want: nil, - }, - { - name: "pod that deleted wait label should be deleted immediately", - existing: []runtime.Object{&normalNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig, noDenyConfig}, - given: &nowaitPod, - want: nil, - }, - { - name: "pod of instance type service is removed immediately", - existing: []runtime.Object{&normalNode, &tgbInstance, &service}, - config: []PodGracefulDrainConfig{defaultConfig, noDenyConfig}, - given: &boundPod, - want: nil, - }, - { - name: "pod in unschedulable node is delayed, but without async delete", - existing: []runtime.Object{&unschedulableNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig}, - timeout: &contextTimeout, - given: &boundPod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(contextTimeout - admissionDelayOverhead), - SleepTaskDuration: contextTimeout - admissionDelayOverhead, - Reason: "node might be draining", - Admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough", - }, - }, - }, - { - name: "pod in tainted node is delayed, but without async delete", - existing: []runtime.Object{&taintedNode, &tgbIP, &service}, - config: []PodGracefulDrainConfig{defaultConfig}, - timeout: &contextTimeout, - given: &boundPod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(contextTimeout - admissionDelayOverhead), - SleepTaskDuration: contextTimeout - admissionDelayOverhead, - Reason: "node might be draining", - Admission: AdmissionResponse{ - Allow: true, - Reason: "Pod deletion is delayed enough", - }, - }, - }, - } - - for _, tt := range tests { - for i, config := range tt.config { - t.Run(fmt.Sprintf("%v - config %v", tt.name, i), func(t *testing.T) { - ctx := context.Background() - if tt.timeout != nil { - var cancel context.CancelFunc - ctx, cancel = context.WithDeadline(ctx, now.Add(*tt.timeout)) - defer cancel() - } - k8sSchema := runtime.NewScheme() - assert.NilError(t, clientgoscheme.AddToScheme(k8sSchema)) - assert.NilError(t, elbv2.AddToScheme(k8sSchema)) - builder := fake.NewClientBuilder().WithScheme(k8sSchema) - for _, existing := range tt.existing { - builder = builder.WithRuntimeObjects(existing.DeepCopyObject()) - } - k8sClient := builder.WithRuntimeObjects(tt.given).Build() - - drain := NewPodGracefulDrain(k8sClient, zap.New(), &config) - spec, err := drain.getDelayedPodDeletionSpec(ctx, tt.given.DeepCopy(), now) - assert.NilError(t, err) - var convertedSpec *wantedSpec - if spec != nil { - convertedSpec = &wantedSpec{ - Isolate: spec.isolate, - DeleteAt: spec.deleteAt, - Reason: spec.reason, - Admission: spec.admission, - } - if spec.asyncDeleteTask != nil { - convertedSpec.AsyncDeleteTaskDuration = spec.asyncDeleteTask.GetDuration() - } - if spec.sleepTask != nil { - convertedSpec.SleepTaskDuration = spec.sleepTask.GetDuration() - } - } - assert.DeepEqual(t, convertedSpec, tt.want) - }) - } - } -} - -func TestDelayedPodEvictionSpec(t *testing.T) { - now := time.Now().Truncate(time.Second) - deleteAt := now.UTC().Add(deleteAfter).Format(time.RFC3339) - - readyStatus := corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - } - boundPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "selector-label": "selector-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - readinessGatePod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "irrelevant-label": "irrelevant-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - ReadinessGates: []corev1.PodReadinessGate{ - {ConditionType: "target-health.elbv2.k8s.aws"}, - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - {Type: corev1.PodConditionType("target-health.elbv2.k8s.aws"), Status: corev1.ConditionTrue}, - }, - }, - } - unboundPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "irrelevant-label": "irrelevant-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - notReadyPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "selector-label": "selector-value", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - } - isolatedPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAt, - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - nowaitPod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": "2006-01-02T15:04:05Z", - }, - }, - Spec: corev1.PodSpec{ - NodeName: "node", - }, - Status: readyStatus, - } - - service := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "svc", - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "selector-label": "selector-value", - }, - }, - } - - targetTypeIP := elbv2.TargetTypeIP - tgbIP := elbv2.TargetGroupBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tgb", - }, - Spec: elbv2.TargetGroupBindingSpec{ - TargetType: &targetTypeIP, - ServiceRef: elbv2.ServiceReference{Name: "svc"}, - }, - } - targetTypeInstance := elbv2.TargetTypeInstance - tgbInstance := elbv2.TargetGroupBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tgb", - }, - Spec: elbv2.TargetGroupBindingSpec{ - TargetType: &targetTypeInstance, - ServiceRef: elbv2.ServiceReference{Name: "svc"}, - }, - } - - node := corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node", - }, - } - - type wantedSpec struct { - Isolate bool - DeleteAt time.Time - AsyncDeleteTaskDuration time.Duration - } - - tests := []struct { - name string - existing []runtime.Object - given *corev1.Pod - timeout *time.Duration - want *wantedSpec - }{ - { - name: "bound pod should be delayed", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: &boundPod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(deleteAfter), - AsyncDeleteTaskDuration: deleteAfter, - }, - }, - { - name: "pod with readiness gate should be delayed", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: &readinessGatePod, - want: &wantedSpec{ - Isolate: true, - DeleteAt: now.Add(deleteAfter), - AsyncDeleteTaskDuration: deleteAfter, - }, - }, - { - name: "unbound pod is deleted immediately", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: &unboundPod, - want: nil, - }, - { - name: "Isolated pod should be delayed, again", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: &isolatedPod, - want: &wantedSpec{}, - }, - { - name: "not ready pod should be deleted immediately", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: ¬ReadyPod, - want: nil, - }, - { - name: "pod that deleted wait label should be deleted immediately", - existing: []runtime.Object{&node, &tgbIP, &service}, - given: &nowaitPod, - want: nil, - }, - { - name: "pod of instance type service is removed immediately", - existing: []runtime.Object{&node, &tgbInstance, &service}, - given: &boundPod, - want: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - if tt.timeout != nil { - var cancel context.CancelFunc - ctx, cancel = context.WithDeadline(ctx, now.Add(*tt.timeout)) - defer cancel() - } - k8sSchema := runtime.NewScheme() - assert.NilError(t, clientgoscheme.AddToScheme(k8sSchema)) - assert.NilError(t, elbv2.AddToScheme(k8sSchema)) - builder := fake.NewClientBuilder().WithScheme(k8sSchema) - for _, existing := range tt.existing { - builder = builder.WithRuntimeObjects(existing.DeepCopyObject()) - } - k8sClient := builder.WithRuntimeObjects(tt.given).Build() - - drain := NewPodGracefulDrain(k8sClient, zap.New(), &defaultConfig) - spec, err := drain.getDelayedPodEvictionSpec(ctx, tt.given.DeepCopy(), now) - assert.NilError(t, err) - var convertedSpec *wantedSpec - if spec != nil { - convertedSpec = &wantedSpec{ - Isolate: spec.isolate, - DeleteAt: spec.deleteAt, - } - if spec.asyncDeleteTask != nil { - convertedSpec.AsyncDeleteTaskDuration = spec.asyncDeleteTask.GetDuration() - } - } - assert.DeepEqual(t, convertedSpec, tt.want) - }) - } -} diff --git a/internal/pkg/core/pod_mutate.go b/internal/pkg/core/pod_mutate.go deleted file mode 100644 index 6f4ff8d..0000000 --- a/internal/pkg/core/pod_mutate.go +++ /dev/null @@ -1,198 +0,0 @@ -package core - -import ( - "context" - "encoding/json" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" - "time" -) - -type PodMutator struct { - client client.Client - logger logr.Logger - pod *corev1.Pod -} - -func NewPodMutator(client client.Client, pod *corev1.Pod) *PodMutator { - return &PodMutator{ - client: client, - logger: logr.Discard(), - pod: pod, - } -} - -func (m *PodMutator) WithLogger(logger logr.Logger) *PodMutator { - return &PodMutator{ - client: m.client, - logger: logger.WithValues("pod", types.NamespacedName{ - Namespace: m.pod.Namespace, - Name: m.pod.Name, - }), - pod: m.pod, - } -} - -func (m *PodMutator) Isolate(ctx context.Context, deleteAt time.Time) error { - m.logger.Info("isolating") - if err := m.isolate(ctx, deleteAt); err != nil { - return err - } - m.logger.V(1).Info("isolated") - return nil -} - -func (m *PodMutator) DisableWaitLabelAndDelete(ctx context.Context) error { - m.logger.Info("disabling wait label") - if err := m.disableWaitLabel(ctx); err != nil { - return err - } - m.logger.V(1).Info("disabled wait label") - - m.logger.Info("deleting") - if err := m.delete(ctx); err != nil { - return err - } - m.logger.V(1).Info("deleted") - return nil -} - -func (m *PodMutator) isolate(ctx context.Context, deleteAt time.Time) error { - patchCond := func(pod *corev1.Pod) bool { - delayInfo, _ := GetPodDeletionDelayInfo(pod) - return delayInfo.Isolated - } - patchMutate := func(pod *corev1.Pod) error { - oldLabels, err := json.Marshal(pod.Labels) - if err != nil { - return err - } - - pod.Labels = map[string]string{ - WaitLabelKey: "true", - } - if pod.Annotations == nil { - pod.Annotations = map[string]string{} - } - pod.Annotations[DeleteAtAnnotationKey] = deleteAt.UTC().Format(time.RFC3339) - pod.Annotations[OriginalLabelsAnnotationKey] = string(oldLabels) - - var newOwnerReferences []metav1.OwnerReference - // To stop the GC kicking in, we cut the OwnerReferences. - for _, item := range pod.OwnerReferences { - newItem := item.DeepCopy() - newItem.Controller = nil - newOwnerReferences = append(newOwnerReferences, *newItem) - } - pod.OwnerReferences = newOwnerReferences - - return nil - } - - return m.patchPod(ctx, patchCond, patchMutate) -} - -func (m *PodMutator) disableWaitLabel(ctx context.Context) error { - patchCond := func(pod *corev1.Pod) bool { - existingLabel := pod.Labels[WaitLabelKey] - return len(existingLabel) == 0 - } - patchMutate := func(pod *corev1.Pod) error { - // set empty rather than removing it. It helps to manually find delayed pods. - pod.Labels[WaitLabelKey] = "" - return nil - } - - return m.patchPod(ctx, patchCond, patchMutate) -} - -// +kubebuilder:rbac:groups="",resources=pods,verbs=patch - -func (m *PodMutator) patchPod(ctx context.Context, desired func(*corev1.Pod) bool, mutate func(*corev1.Pod) error) error { - needUpdate := false - if len(m.pod.ResourceVersion) == 0 { - needUpdate = true - } - - for { - if needUpdate { - if err := m.reloadPod(ctx); err != nil { - return err - } - } - - if desired(m.pod) { - return nil - } - - oldPod := m.pod.DeepCopy() - oldPod.UID = "" // only put the uid in the new object to ensure it appears in the patch as a precondition - - if err := mutate(m.pod); err != nil { - return nil - } - - podMergeOption := client.MergeFromWithOptions(oldPod, client.MergeFromWithOptimisticLock{}) - if err := m.client.Patch(ctx, m.pod, podMergeOption); err != nil { - if apierrors.IsConflict(err) { - needUpdate = false - continue - } - return err - } - - // see https://github.com/kubernetes-sigs/controller-runtime/issues/1257 - return wait.ExponentialBackoff(retry.DefaultBackoff, func() (bool, error) { - if desired(m.pod) { - return true, nil - } - err := m.reloadPod(ctx) - return false, err - }) - } -} - -// +kubebuilder:rbac:groups="",resources=pods,verbs=get;watch - -func (m *PodMutator) reloadPod(ctx context.Context) error { - podUID := m.pod.UID - podKey := types.NamespacedName{ - Namespace: m.pod.Namespace, - Name: m.pod.Name, - } - - var freshPod corev1.Pod - if err := m.client.Get(ctx, podKey, &freshPod); err != nil { - return err - } - if freshPod.UID != podUID { - // UID conflict -> pod is gone - return apierrors.NewNotFound(corev1.Resource(string(corev1.ResourcePods)), m.pod.Name) - } - - *m.pod = freshPod - return nil -} - -// +kubebuilder:rbac:groups="",resources=pods,verbs=delete - -func (m *PodMutator) delete(ctx context.Context) error { - return wait.ExponentialBackoff(retry.DefaultBackoff, func() (bool, error) { - if err := m.client.Delete(ctx, m.pod, client.Preconditions{UID: &m.pod.UID}); err != nil { - if apierrors.IsNotFound(err) || apierrors.IsConflict(err) { - // The pod is already deleted. Okay to ignore - return true, nil - } - // Intercept might deny the deletion as too early until DisableWaitLabel patch is propagated. - // TODO: error is actually admission denial - return false, nil - } - return true, nil - }) -} diff --git a/internal/pkg/core/pod_mutate_test.go b/internal/pkg/core/pod_mutate_test.go deleted file mode 100644 index 282472e..0000000 --- a/internal/pkg/core/pod_mutate_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package core - -import ( - "context" - "gotest.tools/v3/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" - "time" -) - -func TestIsolate(t *testing.T) { - deleteAt := time.Now().UTC().Truncate(time.Second) - deleteAtLabel := deleteAt.Format(time.RFC3339) - normalPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "label1": "value1", - }, - }, - } - isolatedPod1 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAtLabel, - }, - }, - } - isolatedPod2 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAtLabel, - }, - }, - } - isolatedPod3 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAtLabel, - }, - }, - } - - tests := []struct { - name string - existing []runtime.Object - given *corev1.Pod - want *corev1.Pod - }{ - { - name: "pod should be isolated by attaching wait sentinel label, must have deleteAt annotation", - existing: []runtime.Object{normalPod}, - given: normalPod, - want: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": deleteAtLabel, - "pod-graceful-drain/originalLabels": `{"label1":"value1"}`, - }, - }, - }, - }, { - name: "already isolated pod shouldn't be modified (1)", - existing: []runtime.Object{isolatedPod1}, - given: normalPod, - want: isolatedPod1, - }, { - name: "already isolated pod shouldn't be modified (2)", - existing: []runtime.Object{isolatedPod2}, - given: normalPod, - want: isolatedPod2, - }, { - name: "already isolated pod shouldn't be modified (3)", - existing: []runtime.Object{isolatedPod3}, - given: normalPod, - want: isolatedPod3, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - k8sSchema := runtime.NewScheme() - assert.NilError(t, clientgoscheme.AddToScheme(k8sSchema)) - builder := fake.NewClientBuilder().WithScheme(k8sSchema) - for _, existing := range tt.existing { - builder = builder.WithRuntimeObjects(existing.DeepCopyObject()) - } - k8sClient := builder.Build() - - pod := tt.given.DeepCopy() - err := NewPodMutator(k8sClient, pod).isolate(ctx, deleteAt) - - assert.NilError(t, err) - assert.DeepEqual(t, pod.Labels, tt.want.Labels) - assert.DeepEqual(t, pod.Annotations, tt.want.Annotations) - }) - } -} - -func TestDisableWaitLabel(t *testing.T) { - waitingPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - }, - } - disabledPod1 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "", - }, - }, - } - disabledPod2 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - }, - } - disabledPod3 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "label1": "value1", - }, - }, - } - - tests := []struct { - name string - existing []runtime.Object - given *corev1.Pod - want *corev1.Pod - }{ - { - name: "waiting pod should be disabled by setting empty string on the wait sentinel label", - existing: []runtime.Object{waitingPod}, - given: waitingPod, - want: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "", - }, - }, - }, - }, { - name: "already disabled pod shouldn't be modified (1)", - existing: []runtime.Object{disabledPod1}, - given: waitingPod, - want: disabledPod1, - }, { - name: "already disabled pod shouldn't be modified (2)", - existing: []runtime.Object{disabledPod2}, - given: waitingPod, - want: disabledPod2, - }, { - name: "already disabled pod shouldn't be modified (3)", - existing: []runtime.Object{disabledPod3}, - given: waitingPod, - want: disabledPod3, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - k8sSchema := runtime.NewScheme() - assert.NilError(t, clientgoscheme.AddToScheme(k8sSchema)) - builder := fake.NewClientBuilder().WithScheme(k8sSchema) - for _, existing := range tt.existing { - builder = builder.WithRuntimeObjects(existing.DeepCopyObject()) - } - k8sClient := builder.Build() - - pod := tt.given.DeepCopy() - err := NewPodMutator(k8sClient, pod).disableWaitLabel(ctx) - - assert.NilError(t, err) - assert.DeepEqual(t, pod.Labels, tt.want.Labels) - }) - } -} - -func TestDelete(t *testing.T) { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod", - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - }, - } - - tests := []struct { - name string - existing []runtime.Object - given *corev1.Pod - }{ - { - name: "delete", - existing: []runtime.Object{pod}, - given: pod, - }, - { - name: "delete gone", - existing: []runtime.Object{}, - given: pod, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - k8sSchema := runtime.NewScheme() - assert.NilError(t, clientgoscheme.AddToScheme(k8sSchema)) - builder := fake.NewClientBuilder().WithScheme(k8sSchema) - for _, existing := range tt.existing { - builder = builder.WithRuntimeObjects(existing.DeepCopyObject()) - } - k8sClient := builder.Build() - - pod := tt.given.DeepCopy() - err := NewPodMutator(k8sClient, pod).delete(ctx) - - assert.NilError(t, err) - }) - } -} diff --git a/internal/pkg/core/pod_state.go b/internal/pkg/core/pod_state.go deleted file mode 100644 index 38dcfa3..0000000 --- a/internal/pkg/core/pod_state.go +++ /dev/null @@ -1,108 +0,0 @@ -package core - -import ( - "context" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "time" -) - -const ( - GracefulDrainPrefix = "pod-graceful-drain" - WaitLabelKey = GracefulDrainPrefix + "/wait" - DeleteAtAnnotationKey = GracefulDrainPrefix + "/deleteAt" - OriginalLabelsAnnotationKey = GracefulDrainPrefix + "/originalLabels" -) - -func IsPodReady(pod *corev1.Pod) bool { - err, condition := getPodCondition(&pod.Status, corev1.PodReady) - if err == -1 || condition.Status != corev1.ConditionTrue { - return false - } - - for _, rg := range pod.Spec.ReadinessGates { - _, condition := getPodCondition(&pod.Status, rg.ConditionType) - if condition == nil || condition.Status != corev1.ConditionTrue { - return false - } - } - return true -} - -func getPodCondition(status *corev1.PodStatus, conditionType corev1.PodConditionType) (int, *corev1.PodCondition) { - if status == nil { - return -1, nil - } - - if status.Conditions == nil { - return -1, nil - } - - for i := range status.Conditions { - if status.Conditions[i].Type == conditionType { - return i, &status.Conditions[i] - } - } - return -1, nil -} - -type PodDeletionDelayInfo struct { - Isolated bool - Wait bool - DeleteAtUTC time.Time -} - -func GetPodDeletionDelayInfo(pod *corev1.Pod) (PodDeletionDelayInfo, error) { - result := PodDeletionDelayInfo{} - - waitLabelValue, hasWaitLabel := pod.Labels[WaitLabelKey] - deleteAtAnnotationValue, hasDeleteAtLabel := pod.Annotations[DeleteAtAnnotationKey] - - result.Isolated = hasWaitLabel || hasDeleteAtLabel - result.Wait = len(waitLabelValue) > 0 - - if hasWaitLabel && !hasDeleteAtLabel { - return result, errors.New("deleteAt annotation does not exits") - } - - if !result.Wait { - return result, nil - } - - deleteAt, err := time.Parse(time.RFC3339, deleteAtAnnotationValue) - if err != nil { - return result, errors.Wrapf(err, "deleteAt annotation is not RFC3339 format") - } - result.DeleteAtUTC = deleteAt - - return result, nil -} - -func (i *PodDeletionDelayInfo) GetRemainingTime(now time.Time) time.Duration { - nowUTC := now.UTC() - if !i.Isolated || !i.Wait || nowUTC.After(i.DeleteAtUTC) { - return time.Duration(0) - } else { - return i.DeleteAtUTC.Sub(nowUTC) - } -} - -func IsPodInDrainingNode(ctx context.Context, client client.Client, pod *corev1.Pod) (bool, error) { - nodeName := pod.Spec.NodeName - var node corev1.Node - if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil { - return false, errors.Wrapf(err, "cannot get node %v", nodeName) - } - - if node.Spec.Unschedulable { - return true, nil - } - for _, taint := range node.Spec.Taints { - if taint.Key == corev1.TaintNodeUnschedulable { - return true, nil - } - } - return false, nil -} diff --git a/internal/pkg/core/pod_state_test.go b/internal/pkg/core/pod_state_test.go deleted file mode 100644 index c85ac36..0000000 --- a/internal/pkg/core/pod_state_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package core_test - -import ( - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "gotest.tools/v3/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" - "time" -) - -func TestIsPodReady(t *testing.T) { - tests := []struct { - name string - given *corev1.Pod - want bool - }{ - { - name: "pod is not ready if ready is not exist", - given: &corev1.Pod{}, - want: false, - }, { - name: "pod is not ready if PodReady is false", - given: &corev1.Pod{ - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionFalse, - }, - }, - }, - }, - want: false, - }, { - name: "pod is ready if PodReady is true", - given: &corev1.Pod{ - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - }, - }, - }, - want: true, - }, { - name: "pod is not ready if ReadinessGate condition is not exists", - given: &corev1.Pod{ - Spec: corev1.PodSpec{ - ReadinessGates: []corev1.PodReadinessGate{ - {ConditionType: "readiness-gate"}, - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - }, - }, - }, - want: false, - }, { - name: "pod is not ready if ReadinessGate condition is false", - given: &corev1.Pod{ - Spec: corev1.PodSpec{ - ReadinessGates: []corev1.PodReadinessGate{ - {ConditionType: "readiness-gate"}, - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - { - Type: corev1.PodConditionType("readiness-gate"), - Status: corev1.ConditionFalse, - }, - }, - }, - }, - want: false, - }, { - name: "pod is not ready if ReadinessGate condition is false", - given: &corev1.Pod{ - Spec: corev1.PodSpec{ - ReadinessGates: []corev1.PodReadinessGate{ - {ConditionType: "readiness-gate"}, - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, - { - Type: corev1.PodConditionType("readiness-gate"), - Status: corev1.ConditionTrue, - }, - }, - }, - }, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := core.IsPodReady(tt.given) - assert.Equal(t, result, tt.want) - }) - } -} - -func TestGetPodDeletionDelayInfo(t *testing.T) { - deleteAt := time.Now().UTC().Truncate(time.Second) - correctDeleteAtLabel := deleteAt.Format(time.RFC3339) - incorrectDeleteAtLabel := deleteAt.Format(time.ANSIC) - - tests := []struct { - name string - given *corev1.Pod - want core.PodDeletionDelayInfo - errwant string - }{ - { - name: "plain pod", - given: &corev1.Pod{}, - want: core.PodDeletionDelayInfo{ - Isolated: false, - Wait: false, - }, - }, { - name: "pod has non-empty wait sentinel label and deleteAt annotation", - given: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": correctDeleteAtLabel, - }, - }, - }, - want: core.PodDeletionDelayInfo{ - Isolated: true, - Wait: true, - DeleteAtUTC: deleteAt, - }, - }, { - name: "pod has empty wait sentinel label", - given: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "pod-graceful-drain/wait": "", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": correctDeleteAtLabel, - }, - }, - }, - want: core.PodDeletionDelayInfo{ - Isolated: true, - Wait: false, - }, - }, { - name: "pod only has deleteAt annotation", - given: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": correctDeleteAtLabel, - }, - }, - }, - want: core.PodDeletionDelayInfo{ - Isolated: true, - Wait: false, - }, - }, { - name: "pod doesn't have deleteAt label", - given: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - }, - }, - errwant: "deleteAt annotation does not exits", - }, { - name: "pod has incorrect deleteAt label", - given: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "pod-graceful-drain/wait": "true", - }, - Annotations: map[string]string{ - "pod-graceful-drain/deleteAt": incorrectDeleteAtLabel, - }, - }, - }, - errwant: "deleteAt annotation is not RFC3339 format", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := core.GetPodDeletionDelayInfo(tt.given) - if err != nil { - assert.ErrorContains(t, err, tt.errwant) - } else { - assert.Equal(t, result, tt.want) - } - }) - } -} - -func TestPodDeletionDelayInfo_GetRemainingTime(t *testing.T) { - deleteAt := time.Now().UTC().Truncate(time.Second) - offset := 30 * time.Second - delayInfo := core.PodDeletionDelayInfo{ - Isolated: true, - Wait: true, - DeleteAtUTC: deleteAt, - } - - tests := []struct { - name string - now time.Time - want time.Duration - }{ - { - name: "before deleteAt", - now: deleteAt.Add(-offset), - want: offset, - }, { - name: "after deleteAt", - now: deleteAt.Add(offset), - want: time.Duration(0), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := delayInfo.GetRemainingTime(tt.now) - assert.Equal(t, result, tt.want) - }) - } -} diff --git a/internal/pkg/core/service.go b/internal/pkg/core/service.go deleted file mode 100644 index c0d5a45..0000000 --- a/internal/pkg/core/service.go +++ /dev/null @@ -1,74 +0,0 @@ -package core - -import ( - "context" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" - "sigs.k8s.io/controller-runtime/pkg/client" - "strings" -) - -const ( - // Prefix for TargetHealth pod condition type. - TargetHealthPodConditionTypePrefix = "target-health.elbv2.k8s.aws" -) - -// +kubebuilder:rbac:groups=elbv2.k8s.aws,resources=targetgroupbindings,verbs=list;watch -// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch - -func DidPodHaveServicesTargetTypeIP(ctx context.Context, client client.Client, pod *corev1.Pod) (bool, error) { - svcs, err := getServicesTargetTypeIP(client, ctx, pod) - if err != nil { - return false, err - } - - if len(svcs) == 0 { - for _, item := range pod.Spec.ReadinessGates { - if strings.HasPrefix(string(item.ConditionType), TargetHealthPodConditionTypePrefix) { - // The pod once had TargetGroupBindings, but it is somehow gone. - // We don't know whether its TargetType is IP, it's target group, etc. - // It might be worth to to give some time to ELB. - return true, nil - } - } - return false, nil - } - return true, nil -} - -func getServicesTargetTypeIP(k8sClient client.Client, ctx context.Context, pod *corev1.Pod) ([]corev1.Service, error) { - tgbList := &elbv2api.TargetGroupBindingList{} - if err := k8sClient.List(ctx, tgbList, client.InNamespace(pod.Namespace)); err != nil { - return nil, errors.Wrapf(err, "unable to list TargetGroupBindings in namespace %v", pod.Namespace) - } - var svcs []corev1.Service - for _, tgb := range tgbList.Items { - if tgb.Spec.TargetType == nil || (*tgb.Spec.TargetType) != elbv2api.TargetTypeIP { - continue - } - - svcKey := types.NamespacedName{Namespace: tgb.Namespace, Name: tgb.Spec.ServiceRef.Name} - svc := corev1.Service{} - if err := k8sClient.Get(ctx, svcKey, &svc); err != nil { - // If the service is not found, ignore - if apierrors.IsNotFound(err) { - continue - } - return nil, err - } - var svcSelector labels.Selector - if len(svc.Spec.Selector) == 0 { - svcSelector = labels.Nothing() - } else { - svcSelector = labels.SelectorFromSet(svc.Spec.Selector) - } - if svcSelector.Matches(labels.Set(pod.Labels)) { - svcs = append(svcs, svc) - } - } - return svcs, nil -} diff --git a/internal/pkg/webhooks/eviction_mutator.go b/internal/pkg/webhooks/eviction_mutator.go deleted file mode 100644 index bacdd69..0000000 --- a/internal/pkg/webhooks/eviction_mutator.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhooks - -import ( - "context" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "github.com/go-logr/logr" - admissionv1 "k8s.io/api/admission/v1" - policyv1 "k8s.io/api/policy/v1" - "k8s.io/apimachinery/pkg/types" - "net/http" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -type EvictionValidator struct { - interceptor *core.Interceptor - logger logr.Logger - config *core.PodGracefulDrainConfig - - decoder *admission.Decoder -} - -var _ admission.DecoderInjector = &PodValidator{} -var _ admission.Handler = &PodValidator{} - -func NewEvictionValidator(interceptor *core.Interceptor, logger logr.Logger, config *core.PodGracefulDrainConfig) EvictionValidator { - return EvictionValidator{ - interceptor: interceptor, - logger: logger.WithName("pod-eviction-validation-webhook"), - config: config, - } -} - -func (v *EvictionValidator) InjectDecoder(decoder *admission.Decoder) error { - v.decoder = decoder - return nil -} - -func (v *EvictionValidator) Handle(ctx context.Context, req admission.Request) admission.Response { - switch req.Operation { - case admissionv1.Create: - return v.handleCreate(ctx, req) - default: - return admission.Allowed("") - } -} - -func (v *EvictionValidator) handleCreate(ctx context.Context, req admission.Request) admission.Response { - ctx, cancel := context.WithTimeout(ctx, TimeoutFromContext(ctx)) - defer cancel() - - eviction := policyv1.Eviction{} - if err := v.decoder.DecodeRaw(req.Object, &eviction); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - logger := v.logger.WithValues("eviction", types.NamespacedName{Namespace: eviction.Namespace, Name: eviction.Name}) - logger.Info("Handling pod eviction") - - intercepted, err := v.interceptor.InterceptPodEviction(ctx, &req, &eviction) - if err != nil { - logger.Error(err, "errored while intercepting pod eviction") - if v.config.IgnoreError { - return admission.Allowed("ignore error during intercepting pod eviction") - } else { - return admission.Errored(1, err) - } - } else if intercepted != nil { - logger.Info("intercepted", "admission", intercepted) - return intercepted.GetAdmissionResponse() - } - - logger.V(1).Info("Pod eviction is not intercepted") - return admission.Allowed("") -} - -// +kubebuilder:webhook:admissionReviewVersions=v1,webhookVersions=v1,verbs=create,path=/mutate-core-v1-pod-eviction,mutating=true,failurePolicy=ignore,sideEffects=noneOnDryRun,groups="",resources=pods/eviction,versions=v1,name=mpodseviction.pod-graceful-drain.io - -func (v *EvictionValidator) SetupWebhookWithManager(mgr ctrl.Manager) error { - mgr.GetWebhookServer().Register("/mutate-core-v1-pod-eviction", &admission.Webhook{ - Handler: v, - WithContextFunc: NewContextFromRequest, - }) - return nil -} diff --git a/internal/pkg/webhooks/pod_validator.go b/internal/pkg/webhooks/pod_validator.go deleted file mode 100644 index 586e885..0000000 --- a/internal/pkg/webhooks/pod_validator.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhooks - -import ( - "context" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "github.com/go-logr/logr" - admissionv1 "k8s.io/api/admission/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "net/http" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -type PodValidator struct { - logger logr.Logger - interceptor *core.Interceptor - config *core.PodGracefulDrainConfig - - decoder *admission.Decoder -} - -var _ admission.DecoderInjector = &PodValidator{} -var _ admission.Handler = &PodValidator{} - -func NewPodValidator(interceptor *core.Interceptor, logger logr.Logger, config *core.PodGracefulDrainConfig) PodValidator { - return PodValidator{ - interceptor: interceptor, - logger: logger.WithName("pod-validation-webhook"), - config: config, - } -} - -func (v *PodValidator) InjectDecoder(decoder *admission.Decoder) error { - v.decoder = decoder - return nil -} - -func (v *PodValidator) Handle(ctx context.Context, req admission.Request) admission.Response { - switch req.Operation { - case admissionv1.Delete: - return v.handleDelete(ctx, req) - default: - return admission.Allowed("") - } -} - -func (v *PodValidator) handleDelete(ctx context.Context, req admission.Request) admission.Response { - ctx, cancel := context.WithTimeout(ctx, TimeoutFromContext(ctx)) - defer cancel() - - pod := v1.Pod{} - if err := v.decoder.DecodeRaw(req.OldObject, &pod); err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - - logger := v.logger.WithValues("pod", types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}) - logger.Info("Handling pod deletion") - - intercepted, err := v.interceptor.InterceptPodDeletion(ctx, &req, &pod) - if err != nil { - logger.Error(err, "errored while intercepting pod deletion") - if v.config.IgnoreError { - return admission.Allowed("ignore error during intercepting pod deletion") - } else { - return admission.Errored(1, err) - } - } else if intercepted != nil { - logger.Info("intercepted", "admission", intercepted) - return intercepted.GetAdmissionResponse() - } - - logger.V(1).Info("Pod deletion is not intercepted") - return admission.Allowed("") -} - -// +kubebuilder:webhook:admissionReviewVersions=v1,webhookVersions=v1,verbs=delete,path=/validate-core-v1-pod,mutating=false,failurePolicy=ignore,sideEffects=noneOnDryRun,groups=core,resources=pods,versions=v1,name=vpod.pod-graceful-drain.io - -func (v *PodValidator) SetupWebhookWithManager(mgr ctrl.Manager) error { - mgr.GetWebhookServer().Register("/validate-core-v1-pod", &admission.Webhook{ - Handler: v, - WithContextFunc: NewContextFromRequest, - }) - return nil -} diff --git a/internal/pkg/webhooks/timeout.go b/internal/pkg/webhooks/timeout.go deleted file mode 100644 index 2bdf1c0..0000000 --- a/internal/pkg/webhooks/timeout.go +++ /dev/null @@ -1,37 +0,0 @@ -package webhooks - -import ( - "context" - "net/http" - ctrl "sigs.k8s.io/controller-runtime" - "time" -) - -const ( - webhookDefaultTimeout = 10 * time.Second -) - -type contextKey struct{} - -func NewContextFromRequest(ctx context.Context, req *http.Request) context.Context { - query := req.URL.Query() - timeout := query.Get("timeout") - if len(timeout) == 0 { - return ctx - } - duration, err := time.ParseDuration(timeout) - if err != nil { - ctrl.Log.Error(err, "unable to parse timeout") - } - - return context.WithValue(ctx, contextKey{}, &duration) -} - -func TimeoutFromContext(ctx context.Context) time.Duration { - timeout := ctx.Value(contextKey{}).(*time.Duration) - if timeout != nil { - return *timeout - } else { - return webhookDefaultTimeout - } -} diff --git a/main.go b/main.go deleted file mode 100644 index ab49240..0000000 --- a/main.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "github.com/foriequal0/pod-graceful-drain/internal" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/core" - "github.com/foriequal0/pod-graceful-drain/internal/pkg/webhooks" - "github.com/go-logr/logr" - "github.com/pkg/errors" - "go.uber.org/zap/zapcore" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "os" - elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - // +kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - - GitVersion string - GitCommit string - BuildDate string -) - -func init() { - _ = clientgoscheme.AddToScheme(scheme) - _ = elbv2api.AddToScheme(scheme) - // +kubebuilder:scaffold:scheme -} - -func main() { - cfg, err := parseConfig() - if err != nil { - setupLog.Error(err, "unable to parse controller config") - os.Exit(1) - } - - logger, err := createLogger(cfg.LogLevel) - if err != nil { - setupLog.Error(err, "unable to create logger") - os.Exit(1) - } - ctrl.SetLogger(logger) - - setupLog.Info("version", - "GitVersion", GitVersion, - "GitCommit", GitCommit, - "BuildDate", BuildDate, - ) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), cfg.BuildManagerOptions(scheme)) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - // +kubebuilder:scaffold:builder - - drain := core.NewPodGracefulDrain(mgr.GetClient(), ctrl.Log, &cfg.PodGracefulDrain) - if err := mgr.Add(&drain); err != nil { - setupLog.Error(err, "unable to setup pod-graceful-drain") - os.Exit(1) - } - interceptor := core.NewInterceptor(&drain, mgr.GetClient()) - - podValidationWebhook := webhooks.NewPodValidator(&interceptor, ctrl.Log, &cfg.PodGracefulDrain) - if err := podValidationWebhook.SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "pod-validation-webhook") - os.Exit(1) - } - - evictionValidationWebhook := webhooks.NewEvictionValidator(&interceptor, ctrl.Log, &cfg.PodGracefulDrain) - if err := evictionValidationWebhook.SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "pod-eviction-validation-webhook") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func parseConfig() (internal.Config, error) { - fs := flag.NewFlagSet("", flag.ExitOnError) - cfg := internal.Config{} - cfg.BindFlags(fs) - err := fs.Parse(os.Args) - return cfg, err -} - -func createLogger(logLevel string) (logr.Logger, error) { - var zapcoreLevel zapcore.Level - switch logLevel { - case "info": - zapcoreLevel = zapcore.InfoLevel - case "debug": - zapcoreLevel = zapcore.DebugLevel - default: - return logr.Logger{}, errors.New("Invalid log level") - } - - logger := zap.New(zap.UseDevMode(false), - zap.Level(zapcoreLevel), - zap.StacktraceLevel(zapcore.FatalLevel)) - return logger, nil -} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..98b7b10 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "pod-graceful-drain-integ-test" +version = "0.1.0" +description = "Integration test for pod-graceful-drain" +authors = [ + { name = "SeongChan Lee", email = "foriequal@gmail.com" } +] +dependencies = [] +readme = "README.md" +requires-python = ">= 3.8" + +[tool.rye] +managed = true +dev-dependencies = [ + "pytest>=8.3.1", +] diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 0000000..2ee8d19 --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,20 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false + +-e file:. +colorama==0.4.6 + # via pytest +iniconfig==2.0.0 + # via pytest +packaging==24.1 + # via pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.1 diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..d795436 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,11 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false + +-e file:. diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..3be6644 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,35 @@ +# skaffold is just to help tighten the local development/test loop +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: pod-graceful-drain +build: + artifacts: + - image: ghcr.io/foriequal0/pod-graceful-drain + docker: + dockerfile: docker/skaffold.Dockerfile + context: . + local: + push: false + useBuildkit: true + concurrency: 0 +deploy: + helm: + releases: + - name: pod-graceful-drain + chartPath: charts/pod-graceful-drain + valuesFiles: + - charts/pod-graceful-drain/values.yaml + setValues: + experimentalGeneralIngress: true + # Skaffold sends this parameter to Helm as a command line parameter + # '--set=logLevel=info,pod_graceful_drain=trace'. If we omit the backslash, + # Helm interprets a comma in the parameter as a delimiter for key=value pair, + # and it tries to set non-existent 'pod_graceful_drain' value to 'trace'. + # This escaping backslash is why I RIIR'd everything. + # Whenever I see Google-related products, I see this kind of practice. + # They implement some features in seemingly-unharmful broken ways because it is quick and simple. + # When they find features are broken, they don't fix it for compatibility reasons. + # Instead, they cover them up with still-broken ad-hoc mechanisms in obscure, undocumented ways. + # And they leak throughout their entire layers of abstractions like this. + logLevel: "info\\,pod_graceful_drain=trace" diff --git a/src/api_resolver.rs b/src/api_resolver.rs new file mode 100644 index 0000000..44e5ff7 --- /dev/null +++ b/src/api_resolver.rs @@ -0,0 +1,84 @@ +use k8s_openapi::NamespaceResourceScope; +use kube::{Api, Client, Config, Resource, ResourceExt}; + +#[derive(Clone)] +pub struct ApiResolver { + pub client: Client, + config: Config, + + /// For namespace isolated test. + namespace: Option, +} + +impl ApiResolver { + pub fn try_new(config: Config) -> kube::Result { + let client = Client::try_from(config.clone())?; + Ok(Self { + client, + config, + namespace: None, + }) + } + + pub fn try_new_within(config: Config, ns: &str) -> kube::Result { + let client = Client::try_from(config.clone())?; + Ok(Self { + client, + config, + namespace: Some(String::from(ns)), + }) + } + + pub fn impersonate_as( + &self, + user: Option, + group: Option>, + ) -> kube::Result { + let mut config = self.config.clone(); + config.auth_info.impersonate = user; + config.auth_info.impersonate_groups = group; + let client = Client::try_from(config.clone())?; + + Ok(Self { + client, + config, + namespace: self.namespace.clone(), + }) + } + + pub fn all(&self) -> Api + where + K: Resource, + K::DynamicType: Default, + { + if let Some(ns) = self.namespace.as_ref() { + Api::namespaced(self.client.clone(), ns) + } else { + Api::all(self.client.clone()) + } + } + + pub fn default_namespaced(&self) -> Api + where + K: Resource, + K::DynamicType: Default, + { + if let Some(ns) = self.namespace.as_ref() { + Api::namespaced(self.client.clone(), ns) + } else { + Api::default_namespaced(self.client.clone()) + } + } + + pub fn api_for(&self, res: &K) -> Api + where + K: Resource, + K::DynamicType: Default, + { + if let Some(ns) = res.namespace() { + Api::namespaced(self.client.clone(), &ns) + } else { + Api::all(self.client.clone()) + } + } +} diff --git a/src/bin/pod-graceful-drain.rs b/src/bin/pod-graceful-drain.rs new file mode 100644 index 0000000..4041bc7 --- /dev/null +++ b/src/bin/pod-graceful-drain.rs @@ -0,0 +1,124 @@ +use clap::Parser; +use color_eyre::config::Frame; +use eyre::Result; +use std::process::ExitCode; +use std::time::Duration; +use tokio::select; +use tracing::{debug, error, info, Level}; +use tracing_error::ErrorLayer; +use tracing_subscriber::prelude::*; +use tracing_subscriber::{filter::Directive, EnvFilter}; +use uuid::Uuid; + +use pod_graceful_drain::{ + start_controller, start_reflectors, start_webhook, ApiResolver, Config, LoadBalancingConfig, + ServiceRegistry, Shutdown, WebhookConfig, +}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result { + let config = Config::parse(); + + init_tracing_subscriber()?; + install_color_eyre()?; + + print_build_info(); + + let shutdown = Shutdown::new(); + if let Err(err) = try_main(config, &shutdown).await { + error!(?err, "Failed to start server"); + shutdown.trigger_shutdown(); + } + + shutdown.wait_shutdown_triggered().await; + + select! { + _ = shutdown.wait_shutdown_complete() => {}, + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => { + info!("Waiting for graceful shutdown"); + shutdown.wait_shutdown_complete().await; + } + } + + info!("Bye!"); + Ok(ExitCode::from(1)) +} + +async fn try_main(config: Config, shutdown: &Shutdown) -> Result<()> { + let instance_id = Uuid::new_v4(); + info!(%instance_id, "Starting"); + let api_resolver = ApiResolver::try_new(kube::Config::infer().await?)?; + let service_registry = ServiceRegistry::default(); + let loadbalancing = LoadBalancingConfig::new(instance_id); + start_controller(&api_resolver, &service_registry, &loadbalancing, shutdown)?; + let reflectors = start_reflectors(&api_resolver, &config, &service_registry, shutdown)?; + start_webhook( + &api_resolver, + config, + WebhookConfig::controller_runtime_default(), + reflectors, + &service_registry, + &loadbalancing, + shutdown, + ) + .await?; + + info!("Services started"); + loop { + let not_ready = service_registry.get_not_ready_services(); + if not_ready.is_empty() { + info!("Service ready"); + break; + } + + select! { + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + _ = shutdown.wait_shutdown_triggered() => { + break + }, + } + } + + Ok(()) +} + +fn selfish_frame_filter(frames: &mut Vec<&Frame>) { + frames.retain(|frame| { + matches!(frame.name.as_ref(), + Some(name) if name == "pod_graceful_drain" + || name.starts_with("pod_graceful_drain::")) + }); +} + +fn init_tracing_subscriber() -> Result<()> { + let filter = EnvFilter::builder() + .with_default_directive(Directive::from(Level::INFO)) + .from_env()?; + + let fmt = tracing_subscriber::fmt::layer().with_filter(filter); + + tracing_subscriber::registry() + .with(fmt) + .with(ErrorLayer::default()) + .try_init()?; + + Ok(()) +} + +fn install_color_eyre() -> Result<()> { + color_eyre::config::HookBuilder::new() + .capture_span_trace_by_default(true) + .add_frame_filter(Box::new(selfish_frame_filter)) + .install()?; + Ok(()) +} + +fn print_build_info() { + info!("tag: {}", env!("VERGEN_GIT_DESCRIBE")); + debug!("branch: {}", env!("VERGEN_GIT_BRANCH")); + debug!("commit: {}", env!("VERGEN_GIT_SHA")); + debug!("commit date: {}", env!("VERGEN_GIT_COMMIT_DATE")); + + debug!("rustc: {}", env!("VERGEN_RUSTC_SEMVER")); + debug!("build date: {}", env!("VERGEN_BUILD_TIMESTAMP")); +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..5e222a3 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,24 @@ +use std::time::Duration; + +use clap::Parser; +use eyre::{eyre, Result}; +use humantime::parse_duration; + +#[derive(Clone, Debug, Parser)] +#[command(version, about)] +pub struct Config { + #[arg(long, default_value = "25s", value_parser = parse_delete_after)] + pub delete_after: Duration, + + #[arg(long, default_value = "false")] + pub experimental_general_ingress: bool, +} + +fn parse_delete_after(input: &str) -> Result { + let duration = parse_duration(input)?; + if duration > Duration::from_secs(25) { + return Err(eyre!("delete-after should be >=1s, <= 25s")); + } + + Ok(duration) +} diff --git a/src/consts.rs b/src/consts.rs new file mode 100644 index 0000000..6e575c1 --- /dev/null +++ b/src/consts.rs @@ -0,0 +1,8 @@ +pub const CONTROLLER_NAME: &str = "pod-graceful-drain"; + +pub const DRAINING_LABEL_KEY: &str = "pod-graceful-drain/draining"; + +pub const DRAIN_UNTIL_ANNOTATION_KEY: &str = "pod-graceful-drain/drain-until"; +pub const ORIGINAL_LABELS_ANNOTATION_KEY: &str = "pod-graceful-drain/original-labels"; +pub const DRAIN_CONTROLLER_ANNOTATION_KEY: &str = "pod-graceful-drain/controller"; +pub const DELETE_OPTIONS_ANNOTATION_KEY: &str = "pod-graceful-drain/delete-options"; diff --git a/src/controller.rs b/src/controller.rs new file mode 100644 index 0000000..72c3ce7 --- /dev/null +++ b/src/controller.rs @@ -0,0 +1,204 @@ +use std::ops::Add; +use std::sync::Arc; +use std::time::Duration; + +use chrono::Utc; +use eyre::Result; +use futures::StreamExt; +use k8s_openapi::api::core::v1::Pod; +use kube::api::{DeleteParams, EvictParams, Preconditions}; +use kube::runtime::controller::Action; +use kube::runtime::reflector::ObjectRef; +use kube::runtime::watcher::Config; +use kube::runtime::{controller, watcher, Controller}; +use kube::{Api, ResourceExt}; +use rand::Rng; +use thiserror::Error; +use tracing::{debug, error, info, span, trace, Level}; + +use crate::api_resolver::ApiResolver; +use crate::consts::DRAINING_LABEL_KEY; +use crate::loadbalancing::LoadBalancingConfig; +use crate::pod_draining_info::{get_pod_draining_info, PodDrainingInfo}; +use crate::pod_evict_params::get_pod_evict_params; +use crate::shutdown::Shutdown; +use crate::spawn_service::spawn_service; +use crate::status::{ + is_404_not_found_error, is_409_conflict_error, is_410_gone_error, is_transient_error, +}; +use crate::{instrumented, ServiceRegistry}; + +/// Start a controller that deletes deregistered pods. +pub fn start_controller( + api_resolver: &ApiResolver, + service_registry: &ServiceRegistry, + loadbalancing: &LoadBalancingConfig, + shutdown: &Shutdown, +) -> Result<()> { + let api_resolver = api_resolver.clone(); + + let context = Arc::new(ReconcilerContext { + api_resolver: api_resolver.clone(), + loadbalancing: loadbalancing.clone(), + }); + + let pods: Api = api_resolver.all(); + let controller = Controller::new(pods, Config::default().labels(DRAINING_LABEL_KEY)) + .graceful_shutdown_on(shutdown.wait_shutdown_triggered()); + + let signal = service_registry.register("controller"); + spawn_service(shutdown, "controller", { + let shutdown = shutdown.clone(); + async move { + signal.ready(); + controller + .run(reconcile, error_policy, context) + .take_until(shutdown.wait_shutdown_triggered()) + .for_each(log_reconcile_result) + .await + } + })?; + + Ok(()) +} + +struct ReconcilerContext { + api_resolver: ApiResolver, + loadbalancing: LoadBalancingConfig, +} + +#[derive(Error, Debug)] +enum ReconcileError { + #[error("kube error: {0}")] + KubeError(#[from] kube::Error), +} + +const CONTROLLER_EXCLUSIVE_DURATION: Duration = Duration::from_secs(10); +const CONTROLLER_TIMEOUT_JITTER: Duration = Duration::from_secs(10); +const DEFAULT_TRANSIENT_ERROR_RECONCILE: Duration = Duration::from_secs(5); +const DEFAULT_RECONCILE_DURATION: Duration = Duration::from_secs(3600); + +async fn reconcile( + pod: Arc, + context: Arc, +) -> Result { + let span = span!(Level::ERROR, "reconciler", object_ref = %ObjectRef::from_obj(pod.as_ref())); + instrumented!(span, async move { + if let PodDrainingInfo::DrainUntil(drain_until) = get_pod_draining_info(&pod) { + let remaining = drain_until - Utc::now(); + if let Ok(remaining) = remaining.to_std() { + return Ok(Action::requeue(remaining)); + } + + let expire = (-remaining).to_std().expect("should be expired"); + if expire < CONTROLLER_EXCLUSIVE_DURATION && !context.loadbalancing.controls(&pod) { + // Let the original controller handle first. + let requeue_duration = rand::thread_rng().gen_range( + CONTROLLER_EXCLUSIVE_DURATION + ..CONTROLLER_EXCLUSIVE_DURATION.add(CONTROLLER_TIMEOUT_JITTER), + ); + + return Ok(Action::requeue(requeue_duration)); + } + + // TODO: possible bottleneck of the reconciler. + let result = if let Some(evict_params) = get_pod_evict_params(&pod) { + evict_pod(&context.api_resolver, &pod, &evict_params).await + } else { + delete_pod(&context.api_resolver, &pod).await + }; + + if let Err(err) = result { + if is_transient_error(&err) { + return Ok(Action::requeue(DEFAULT_TRANSIENT_ERROR_RECONCILE)); + } + } + }; + + Ok(Action::requeue(DEFAULT_RECONCILE_DURATION)) + }) +} + +fn error_policy(_pod: Arc, err: &ReconcileError, _context: Arc) -> Action { + match err { + ReconcileError::KubeError(err) => { + if is_409_conflict_error(err) { + return Action::requeue(Duration::from_secs(1)); + } + } + } + + Action::requeue(Duration::from_secs(5)) +} + +async fn log_reconcile_result( + result: Result<(ObjectRef, Action), controller::Error>, +) { + let span = span!(Level::ERROR, "reconciler"); + instrumented!(span, async move { + match result { + Ok((object_ref, action)) => { + trace!(%object_ref, ?action, "success"); + } + Err(controller::Error::ReconcilerFailed(err, object_ref)) => match err { + ReconcileError::KubeError(err) if is_409_conflict_error(&err) => { + debug!(%object_ref, ?err, "conflict"); + } + _ => error!(%object_ref, ?err, "error"), + }, + Err(err) => { + error!(?err, "error"); + } + } + }) +} + +async fn delete_pod(api_resolver: &ApiResolver, pod: &Pod) -> kube::Result<()> { + let api = api_resolver.api_for(pod); + let name = pod.name_any(); + + let delete_params = DeleteParams { + preconditions: Some(Preconditions { + uid: pod.uid(), + ..Preconditions::default() + }), + ..DeleteParams::default() + }; + + info!("deleting pod"); + let result = api.delete(&name, &delete_params).await; + match result { + Ok(_) => { + debug!("pod is deleted"); + Ok(()) + } + Err(err) if is_404_not_found_error(&err) || is_410_gone_error(&err) => { + debug!("pod is gone anyway"); // This is what we desired. + Ok(()) + } + Err(err) => Err(err), + } +} + +async fn evict_pod( + api_resolver: &ApiResolver, + pod: &Pod, + evict_params: &EvictParams, +) -> kube::Result<()> { + let api = api_resolver.api_for(pod); + let name = pod.name_any(); + + info!("evicting pod"); + let result = api.evict(&name, evict_params).await; + match result { + Ok(_) => { + debug!("pod is evicted"); + Ok(()) + } + Err(err) if is_404_not_found_error(&err) || is_410_gone_error(&err) => { + debug!("pod is gone anyway"); // This is what we desired. + Ok(()) + } + Err(err) => Err(err), + } +} diff --git a/src/elbv2/apis.rs b/src/elbv2/apis.rs new file mode 100644 index 0000000..26ebb0b --- /dev/null +++ b/src/elbv2/apis.rs @@ -0,0 +1,68 @@ +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use k8s_openapi::serde::{Deserialize, Serialize}; +use k8s_openapi::{Metadata, NamespaceResourceScope, Resource}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetGroupBinding { + pub metadata: ObjectMeta, + #[serde(skip_serializing_if = "Option::is_none")] + pub spec: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetGroupBindingSpec { + #[serde(rename = "targetGroupARN")] + pub target_group_arn: String, + pub target_type: Option, + pub service_ref: Option, + // not needed for our scenario + // pub networking: Option, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TargetType { + Instance, + Ip, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ServiceReference { + pub name: String, + pub port: IntOrString, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TargetGroupBindingStatus { + #[serde(skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, +} + +impl Resource for TargetGroupBinding { + const API_VERSION: &'static str = "elbv2.k8s.aws/v1beta1"; + const GROUP: &'static str = "elbv2.k8s.aws"; + const KIND: &'static str = "TargetGroupBinding"; + const VERSION: &'static str = "v1beta1"; + const URL_PATH_SEGMENT: &'static str = "targetgroupbindings"; + + type Scope = NamespaceResourceScope; +} + +impl Metadata for TargetGroupBinding { + type Ty = ObjectMeta; + + fn metadata(&self) -> &Self::Ty { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut Self::Ty { + &mut self.metadata + } +} diff --git a/src/elbv2/mod.rs b/src/elbv2/mod.rs new file mode 100644 index 0000000..e1724ce --- /dev/null +++ b/src/elbv2/mod.rs @@ -0,0 +1,3 @@ +pub mod apis; + +pub const TARGET_HEALTH_POD_CONDITION_TYPE_PREFIX: &str = "target-health.elbv2.k8s.aws"; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..2c0561c --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,28 @@ +mod api_resolver; +mod config; +mod consts; +mod controller; +mod elbv2; +mod loadbalancing; +mod pod_draining_info; +mod pod_evict_params; +mod pod_state; +mod reflector; +mod service_registry; +mod shutdown; +mod spawn_service; +mod status; +mod utils; +pub mod webhooks; + +pub use crate::api_resolver::ApiResolver; +pub use crate::config::Config; +pub use crate::controller::start_controller; +pub use crate::loadbalancing::LoadBalancingConfig; +pub use crate::reflector::{start_reflectors, Stores}; +pub use crate::service_registry::ServiceRegistry; +pub use crate::shutdown::Shutdown; +pub use crate::webhooks::{start_webhook, WebhookConfig}; + +#[cfg(test)] +pub use crate::webhooks::patch_pod_isolate; diff --git a/src/loadbalancing.rs b/src/loadbalancing.rs new file mode 100644 index 0000000..bd14534 --- /dev/null +++ b/src/loadbalancing.rs @@ -0,0 +1,30 @@ +use k8s_openapi::api::core::v1::Pod; +use kube::ResourceExt; +use uuid::Uuid; + +use crate::consts; + +#[derive(Clone, Debug)] +pub struct LoadBalancingConfig { + instance_id: Uuid, +} + +impl LoadBalancingConfig { + pub fn new(instance_id: Uuid) -> Self { + Self { instance_id } + } + + pub fn get_id(&self) -> String { + self.instance_id.to_string() + } + + pub fn controls(&self, pod: &Pod) -> bool { + let annotation = pod + .annotations() + .get(consts::DRAIN_CONTROLLER_ANNOTATION_KEY); + + matches!( + annotation.map(|controller| Uuid::try_parse(controller)), + Some(Ok(uuid)) if uuid == self.instance_id) + } +} diff --git a/src/pod_draining_info.rs b/src/pod_draining_info.rs new file mode 100644 index 0000000..3db2b5d --- /dev/null +++ b/src/pod_draining_info.rs @@ -0,0 +1,158 @@ +use chrono::{DateTime, Utc}; +use k8s_openapi::api::core::v1::Pod; +use kube::ResourceExt; + +use crate::consts::{DRAINING_LABEL_KEY, DRAIN_UNTIL_ANNOTATION_KEY}; + +#[derive(Debug)] +pub enum PodDrainingInfo { + None, + DrainUntil(DateTime), + Deleted, + DrainDisabled, + AnnotationParseError { message: String }, +} + +pub fn get_pod_draining_info(pod: &Pod) -> PodDrainingInfo { + if pod.metadata.deletion_timestamp.is_some() { + return PodDrainingInfo::Deleted; + } + + if let Some(label) = pod.labels().get(DRAINING_LABEL_KEY) { + if !label.eq_ignore_ascii_case("true") || label == "0" || label.is_empty() { + return PodDrainingInfo::DrainDisabled; + } + } else { + return PodDrainingInfo::None; + } + + let Some(str) = pod.annotations().get(DRAIN_UNTIL_ANNOTATION_KEY) else { + return PodDrainingInfo::AnnotationParseError { + message: format!("annotation '{DRAIN_UNTIL_ANNOTATION_KEY}' not exists"), + }; + }; + + match DateTime::parse_from_rfc3339(str) { + Ok(datetime) => { + let utc = datetime.with_timezone(&Utc); + PodDrainingInfo::DrainUntil(utc) + } + Err(err) => PodDrainingInfo::AnnotationParseError { + message: format!( + "annotation '{DRAIN_UNTIL_ANNOTATION_KEY}' has invalid format: {}", + err + ), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::assert_matches; + + macro_rules! from_json { + ($($json:tt)+) => { + ::serde_json::from_value(::serde_json::json!($($json)+)).expect("Invalid json") + }; + } + + #[test] + fn should_return_some_drain_until() { + let pod: Pod = from_json! ({ + "metadata": { + "labels": { + "pod-graceful-drain/draining": "true", + }, + "annotations": { + "pod-graceful-drain/drain-until": "2023-02-09T15:30:45Z", + }, + } + }); + + let info = get_pod_draining_info(&pod); + let expected = DateTime::parse_from_rfc3339("2023-02-09T15:30:45Z") + .unwrap() + .with_timezone(&Utc); + assert_matches!(info, PodDrainingInfo::DrainUntil(value) if value == expected); + } + + #[test] + fn should_return_none_with_draining_false() { + let pod: Pod = from_json! ({ + "metadata": { + "labels": { + "pod-graceful-drain/draining": "false", + }, + "annotations": { + "pod-graceful-drain/drain-until": "2023-02-09T15:30:45Z", + }, + } + }); + + let info = get_pod_draining_info(&pod); + assert_matches!(info, PodDrainingInfo::DrainDisabled); + } + + #[test] + fn should_return_none_when_no_label() { + let pod: Pod = from_json! ({ + "metadata": { + "labels": {}, + } + }); + + let info = get_pod_draining_info(&pod); + assert_matches!(info, PodDrainingInfo::None); + } + + #[test] + fn should_return_none_when_no_annotation() { + let pod: Pod = from_json! ({ + "metadata": { + "labels": { + "pod-graceful-drain/draining": "true", + }, + "annotations": {}, + } + }); + + let info = get_pod_draining_info(&pod); + assert_matches!(info, PodDrainingInfo::AnnotationParseError { message: _ }); + } + + #[test] + fn should_return_some_error_when_invalid_annotation() { + let pod: Pod = from_json! ({ + "metadata": { + "labels": { + "pod-graceful-drain/draining": "true", + }, + "annotations": { + "pod-graceful-drain/drain-until": "INVALID", + }, + } + }); + + let info = get_pod_draining_info(&pod); + assert_matches!(info, PodDrainingInfo::AnnotationParseError { message: _ }); + } + + #[test] + fn should_return_some_deleted_when_deletion_timestamp_on_it() { + let pod: Pod = from_json! ({ + "metadata": { + "deletionTimestamp": "2023-02-09T15:30:45Z", + "labels": { + "pod-graceful-drain/draining": "true", + }, + "annotations": { + "pod-graceful-drain/drain-until": "2023-02-09T15:30:45Z", + }, + } + }); + + let info = get_pod_draining_info(&pod); + assert_matches!(info, PodDrainingInfo::Deleted); + } +} diff --git a/src/pod_evict_params.rs b/src/pod_evict_params.rs new file mode 100644 index 0000000..df6d222 --- /dev/null +++ b/src/pod_evict_params.rs @@ -0,0 +1,34 @@ +use k8s_openapi::api::core::v1::Pod; +use kube::api::{DeleteParams, EvictParams, Preconditions}; +use kube::ResourceExt; + +use crate::consts::DELETE_OPTIONS_ANNOTATION_KEY; +use crate::utils::to_delete_params; + +pub fn get_pod_evict_params(pod: &Pod) -> Option { + let annotation = pod.annotations().get(DELETE_OPTIONS_ANNOTATION_KEY)?; + + let Ok(delete_options) = serde_json::from_str(annotation) else { + // TODO : propagate error + return None; + }; + + let Ok(delete_params) = to_delete_params(delete_options, false) else { + // TODO : propagate error + return None; + }; + + Some(EvictParams { + delete_options: Some(DeleteParams { + dry_run: false, + preconditions: Some(Preconditions { + uid: pod.uid(), + // it'll cause conflict + resource_version: None, + }), + grace_period_seconds: delete_params.grace_period_seconds, + propagation_policy: delete_params.propagation_policy, + }), + ..EvictParams::default() + }) +} diff --git a/src/pod_state.rs b/src/pod_state.rs new file mode 100644 index 0000000..dee1150 --- /dev/null +++ b/src/pod_state.rs @@ -0,0 +1,558 @@ +use genawaiter::{rc::gen, yield_}; +use k8s_openapi::api::core::v1::{Pod, Service}; +use kube::runtime::reflector::ObjectRef; +use kube::{Resource, ResourceExt}; +use std::collections::{HashMap, HashSet}; + +use crate::elbv2::apis::TargetType; +use crate::elbv2::TARGET_HEALTH_POD_CONDITION_TYPE_PREFIX; +use crate::reflector::Stores; +use crate::utils::get_object_ref_from_name; +use crate::{try_some, Config}; + +pub fn is_pod_ready(pod: &Pod) -> bool { + let readiness_gates = { + let mut result = HashSet::new(); + // "Ready" is required even if not listed in readiness gate + result.insert("Ready"); + + if let Some(readiness_gates) = try_some!(pod.spec?.readiness_gates?) { + for readiness_gate in readiness_gates { + result.insert(readiness_gate.condition_type.as_str()); + } + } + + result + }; + + let conditions = { + let mut result = HashMap::new(); + if let Some(conditions) = try_some!(pod.status?.conditions?) { + for condition in conditions { + result.insert(condition.type_.as_str(), condition.status.as_str()); + } + } + + result + }; + + for readiness_gate in readiness_gates { + if !matches!(conditions.get(readiness_gate), Some(&"True")) { + return false; + } + } + + true +} + +pub fn is_pod_exposed(config: &Config, stores: &Stores, pod: &Pod) -> bool { + // TODO: Find better way to determine whether a pod is exposed. + // e.g. Examine EndpointSlice, etc. + if config.experimental_general_ingress { + is_exposed_by_ingress(stores, pod) + } else { + is_exposed_by_target_group_binding(stores, pod) + } +} + +fn is_exposed_by_ingress(stores: &Stores, pod: &Pod) -> bool { + // TODO: Build inverted index in reconciler incrementally? + let ingress_exposed_services = gen!({ + let mut seen = HashSet::new(); + let pod_namespace = pod.metadata.namespace.as_ref(); + for ingress in stores.ingresses() { + if ingress.meta().namespace.as_ref() != pod_namespace { + continue; + } + + if let Some(default_service_name) = + try_some!(&ingress.spec?.default_backend?.service?.name) + { + let service_ref = get_object_ref_from_name::( + &default_service_name, + ingress.namespace().as_ref(), + ); + if !seen.insert(service_ref.clone()) { + continue; + } + yield_!(service_ref); + } + + for rule in try_some!(ingress.spec?.rules?).unwrap_or(&vec![]) { + for path in try_some!(&rule.http?.paths).unwrap_or(&vec![]) { + if let Some(service_name) = try_some!(&path.backend.service?.name) { + let service_ref = get_object_ref_from_name::( + &service_name, + ingress.namespace().as_ref(), + ); + if !seen.insert(service_ref.clone()) { + continue; + } + yield_!(service_ref); + } + } + } + } + }); + + ingress_exposed_services + .into_iter() + .any(|service_ref| is_exposing_service(stores, pod, service_ref)) +} + +fn is_exposed_by_target_group_binding(stores: &Stores, pod: &Pod) -> bool { + // TODO: Build inverted index in reconciler incrementally? + let tgb_exposed_service = gen!({ + let mut seen = HashSet::new(); + let pod_namespace = pod.metadata.namespace.as_ref(); + for tgb in stores.target_group_bindings() { + if tgb.meta().namespace.as_ref() != pod_namespace { + continue; + } + + if try_some!(tgb.spec?.target_type?) != Some(&TargetType::Ip) { + continue; + } + + if let Some(service_name) = try_some!(&tgb.spec?.service_ref?.name) { + let service_ref = + get_object_ref_from_name::(&service_name, tgb.namespace().as_ref()); + if !seen.insert(service_ref.clone()) { + continue; + } + + yield_!(service_ref); + } + } + }); + + let is_exposed_by_tgb = tgb_exposed_service + .into_iter() + .any(|service_ref| is_exposing_service(stores, pod, service_ref)); + if is_exposed_by_tgb { + return true; + } + + // The pod once had corresponding TargetGroupBinding, but it is somehow gone. + // We don't know whether its TargetType was IP or not. + // But, true is more conservative than false. + try_some!(pod.spec?.readiness_gates?) + .unwrap_or(&vec![]) + .iter() + .any(|readiness_gate| { + readiness_gate + .condition_type + .starts_with(TARGET_HEALTH_POD_CONDITION_TYPE_PREFIX) + }) +} + +fn is_exposing_service(stores: &Stores, pod: &Pod, service_ref: ObjectRef) -> bool { + let Some(service) = stores.get_service(&service_ref) else { + return false; + }; + + let Some(selector) = try_some!(service.spec?.selector?) else { + return false; + }; + + for (key, value) in selector.iter() { + if pod.labels().get(key) != Some(value) { + return false; + } + } + + true +} + +#[cfg(test)] +mod tests { + use super::*; + use std::hash::Hash; + use std::time::Duration; + + use kube::runtime::reflector::{store, Store}; + use kube::runtime::watcher::Event; + + macro_rules! from_json { + ($($json:tt)+) => { + ::serde_json::from_value(::serde_json::json!($($json)+)).expect("Invalid json") + }; + } + + fn store_from(iter: impl IntoIterator) -> Store + where + K: 'static + Resource + Clone, + K::DynamicType: Hash + Eq + Clone + Default, + { + let (reader, mut writer) = store(); + writer.apply_watcher_event(&Event::Init); + for item in iter.into_iter() { + writer.apply_watcher_event(&Event::InitApply(item)); + } + writer.apply_watcher_event(&Event::InitDone); + reader + } + + fn get_test_experimental_general_ingress_config() -> Config { + Config { + experimental_general_ingress: true, + delete_after: Duration::from_secs(30), + } + } + + #[test] + fn pod_is_ready() { + assert!(is_pod_ready(&from_json!({ + "status": { + "conditions": [ + { + "status": "True", + "type": "Ready" + }, + ], + } + }))); + + assert!(!is_pod_ready(&from_json!({ + "status": { + "conditions": [ + { + "status": "False", + "type": "Ready" + }, + ], + } + }))); + + assert!(is_pod_ready(&from_json!({ + "status": { + "conditions": [ + { + "status": "False", + "type": "some-unknown-condition" + }, + { + "status": "True", + "type": "Ready" + }, + ], + } + }))); + + assert!(!is_pod_ready(&from_json!({ + "spec": { + "readinessGates": [ + { + "conditionType": "some-readiness-gate-condition" + }, + ], + }, + "status": { + "conditions": [ + { + "status": "False", + "type": "some-readiness-gate-condition" + }, + { + "status": "True", + "type": "Ready" + }, + ], + } + }))); + + assert!(is_pod_ready(&from_json!({ + "spec": { + "readinessGates": [ + { + "conditionType": "some-readiness-gate-condition" + }, + ], + }, + "status": { + "conditions": [ + { + "status": "True", + "type": "some-readiness-gate-condition" + }, + { + "status": "True", + "type": "Ready" + }, + ], + } + }))); + } + + #[test] + fn pod_is_exposed() { + let pod: Pod = from_json!({ + "metadata": { + "name": "pod", + "namespace": "ns", + "labels": { + "app": "test" + } + }, + }); + + let service = from_json!({ + "metadata": { + "name": "svc", + "namespace": "ns", + }, + "spec": { + "selector": { + "app": "test", + }, + }, + }); + + let ingress = from_json!({ + "metadata": { + "name": "ig", + "namespace": "ns", + }, + "spec": { + "rules": [{ + "http": { + "paths": [{ + "backend": { + "service": { + "name": "svc", + }, + }, + }], + }, + }], + } + }); + + let stores = Stores::new( + store_from([pod.clone()]), + store_from([service]), + store_from([ingress]), + store_from([]), + ); + + assert!(is_pod_exposed( + &get_test_experimental_general_ingress_config(), + &stores, + &pod + )) + } + + #[test] + fn pod_is_exposed_by_tgb() { + let pod: Pod = from_json!({ + "metadata": { + "name": "pod", + "namespace": "ns", + "labels": { + "app": "test" + } + }, + }); + + let service = from_json!({ + "metadata": { + "name": "svc", + "namespace": "ns", + }, + "spec": { + "selector": { + "app": "test", + }, + }, + }); + + let tgb = from_json!({ + "metadata": { + "name": "tgb", + "namespace": "ns", + }, + "spec": { + "networking": { + // snip + }, + "serviceRef": { + "name": "svc", + "port": "http" + }, + "targetGroupARN": "some-target-group-arn", + "targetType": "ip" + } + }); + + let stores = Stores::new( + store_from([pod.clone()]), + store_from([service]), + store_from([]), + store_from([tgb]), + ); + + assert!(is_pod_exposed( + &Config { + delete_after: Duration::from_secs(30), + experimental_general_ingress: false, + }, + &stores, + &pod + )) + } + + #[test] + fn pod_is_not_exposed_when_no_ingress() { + let pod: Pod = from_json!({ + "metadata": { + "name": "pod", + "namespace": "ns", + "labels": { + "app": "test" + } + }, + }); + + let service = from_json!({ + "metadata": { + "name": "svc", + "namespace": "ns", + }, + "spec": { + "selector": { + "app": "test", + }, + }, + }); + + let stores = Stores::new( + store_from([pod.clone()]), + store_from([service]), + store_from([]), + store_from([]), + ); + + assert!(!is_pod_exposed( + &get_test_experimental_general_ingress_config(), + &stores, + &pod + )) + } + + #[test] + fn pod_is_not_exposed_when_selector_not_match() { + let pod: Pod = from_json!({ + "metadata": { + "name": "pod", + "namespace": "ns", + "labels": { + "app": "test" + } + }, + }); + + let service = from_json!({ + "metadata": { + "name": "svc", + "namespace": "ns", + }, + "spec": { + "selector": { + "app": "test", + "another": "another", + }, + }, + }); + + let ingress = from_json!({ + "metadata": { + "name": "ig", + "namespace": "ns", + }, + "spec": { + "rules": [{ + "http": { + "paths": [{ + "backend": { + "service": { + "name": "svc", + }, + }, + }], + }, + }], + } + }); + + let stores = Stores::new( + store_from([pod.clone()]), + store_from([service]), + store_from([ingress]), + store_from([]), + ); + + assert!(!is_pod_exposed( + &get_test_experimental_general_ingress_config(), + &stores, + &pod + )) + } + + #[test] + fn pod_is_not_exposed_namespace_differ() { + let pod: Pod = from_json!({ + "metadata": { + "name": "pod", + "namespace": "ns2", + "labels": { + "app": "test" + } + }, + }); + + let service = from_json!({ + "metadata": { + "name": "svc", + "namespace": "ns", + }, + "spec": { + "selector": { + "app": "test", + }, + }, + }); + + let ingress = from_json!({ + "metadata": { + "name": "ig", + "namespace": "ns", + }, + "spec": { + "rules": [{ + "http": { + "paths": [{ + "backend": { + "service": { + "name": "svc", + }, + }, + }], + }, + }], + } + }); + + let stores = Stores::new( + store_from([pod.clone()]), + store_from([service]), + store_from([ingress]), + store_from([]), + ); + + assert!(!is_pod_exposed( + &get_test_experimental_general_ingress_config(), + &stores, + &pod + )) + } +} diff --git a/src/reflector.rs b/src/reflector.rs new file mode 100644 index 0000000..f9d519b --- /dev/null +++ b/src/reflector.rs @@ -0,0 +1,229 @@ +use std::default::Default; +use std::future::Future; +use std::hash::Hash; +use std::sync::Arc; + +use eyre::Result; +use futures::{Stream, StreamExt, TryStreamExt}; +use k8s_openapi::api::core::v1::{PodSpec, PodStatus}; +use k8s_openapi::api::{ + core::v1::{Pod, Service}, + networking::v1::Ingress, +}; +use kube::runtime::reflector::store::Writer; +use kube::runtime::reflector::{store, ObjectRef, Store}; +use kube::runtime::watcher; +use kube::runtime::watcher::Event; +use kube::{Api, Resource}; +use tracing::{error, span, trace, Level}; + +use crate::api_resolver::ApiResolver; +use crate::elbv2::apis::TargetGroupBinding; +use crate::service_registry::ServiceSignal; +use crate::shutdown::Shutdown; +use crate::spawn_service::spawn_service; +use crate::{instrumented, try_some, Config, ServiceRegistry}; + +#[derive(Clone)] +pub struct Stores { + inner: Arc, +} + +pub struct StoresInner { + pods: Store, + services: Store, + ingresses: Store, + tgbs: Store, +} + +impl Stores { + pub(crate) fn new( + pods: Store, + services: Store, + ingresses: Store, + tgbs: Store, + ) -> Self { + Self { + inner: Arc::new(StoresInner { + pods, + services, + ingresses, + tgbs, + }), + } + } +} + +pub fn start_reflectors( + api_resolver: &ApiResolver, + config: &Config, + service_registry: &ServiceRegistry, + shutdown: &Shutdown, +) -> Result { + let api_proivder = api_resolver.clone(); + + // TODO : clear unnecessary fields to reduce memory usage + + let (pod_reader, pod_writer) = store(); + spawn_service(shutdown, "reflector:Pod", { + let api: Api = api_proivder.all(); + let stream = watcher(api, Default::default()).map_ok(|event| { + event.modify(|pod| { + if let Some(spec) = try_some!(mut pod.spec?) { + *spec = PodSpec { + readiness_gates: spec.readiness_gates.clone(), + ..PodSpec::default() + } + } + if let Some(spec) = try_some!(mut pod.status?) { + *spec = PodStatus { + conditions: spec.conditions.clone(), + ..PodStatus::default() + } + } + }) + }); + let signal = service_registry.register("reflector:Pod"); + run_reflector(shutdown, pod_writer, stream, signal) + })?; + + let (service_reader, service_writer) = store(); + spawn_service(shutdown, "reflector:Service", { + let api: Api = api_proivder.all(); + let stream = watcher(api, Default::default()).map_ok(|ev| { + ev.modify(|service| { + service.metadata.annotations = None; + service.metadata.labels = None; + service.status = None; + }) + }); + let signal = service_registry.register("reflector:Service"); + run_reflector(shutdown, service_writer, stream, signal) + })?; + + let (ingress_reader, ingress_writer) = store(); + spawn_service(shutdown, "reflector:Ingress", { + let api: Api = api_proivder.all(); + let stream = watcher(api, Default::default()).map_ok(|ev| { + ev.modify(|ingress| { + ingress.metadata.annotations = None; + ingress.metadata.labels = None; + ingress.status = None; + }) + }); + let signal = service_registry.register("reflector:Ingress"); + run_reflector(shutdown, ingress_writer, stream, signal) + })?; + + let (tgb_reader, tgb_writer) = store(); + if !config.experimental_general_ingress { + spawn_service(shutdown, "reflector:TargetGroupBinding", { + let api: Api = api_proivder.all(); + let stream = watcher(api, Default::default()).map_ok(|ev| { + ev.modify(|tgb| { + tgb.metadata.annotations = None; + tgb.metadata.labels = None; + tgb.status = None; + }) + }); + let signal = service_registry.register("reflector:TargetGroupBinding"); + run_reflector(shutdown, tgb_writer, stream, signal) + })?; + } + + Ok(Stores::new( + pod_reader, + service_reader, + ingress_reader, + tgb_reader, + )) +} + +fn run_reflector( + shutdown: &Shutdown, + writer: Writer, + stream: impl Stream>>, + signal: ServiceSignal, +) -> impl Future +where + K: Resource + k8s_openapi::Resource + Clone, + K::DynamicType: Default + Eq + Hash + Clone, +{ + let shutdown = shutdown.clone(); + async move { + instrumented!( + span!(Level::ERROR, "reflector", "{}", K::KIND), + async move { + let mut results = Box::pin( + kube::runtime::reflector(writer, stream) + .take_until(shutdown.wait_shutdown_triggered()), + ); + + // Log until Event::InitDone + while let Some(result) = results.next().await { + log(&result); + + // TODO : raise appropriate signal when Event::Init restarted + if let Ok(Event::InitDone) = result { + signal.ready(); + break; + } + } + + while let Some(result) = results.next().await { + log(&result) + } + + fn log(result: &watcher::Result>) + where + K: Resource, + K::DynamicType: Default, + { + match result { + Ok(event) => match event { + Event::Apply(resource) => { + let object_ref = ObjectRef::from_obj(resource); + trace!(%object_ref, "resource applied"); + } + Event::Delete(resource) => { + let object_ref = ObjectRef::from_obj(resource); + trace!(%object_ref, "resource deleted"); + } + Event::Init => { + trace!("stream restart"); + } + Event::InitApply(resource) => { + let object_ref = ObjectRef::from_obj(resource); + trace!(%object_ref, "stream restarting"); + } + Event::InitDone => { + trace!("stream restart done"); + } + }, + Err(err) => { + error!(?err, "reflector error"); + } + } + } + } + ) + } +} + +impl Stores { + pub fn get_pod(&self, key: &ObjectRef) -> Option> { + self.inner.pods.get(key) + } + + pub fn get_service(&self, key: &ObjectRef) -> Option> { + self.inner.services.get(key) + } + + pub fn ingresses(&self) -> Vec> { + self.inner.ingresses.state() + } + + pub fn target_group_bindings(&self) -> Vec> { + self.inner.tgbs.state() + } +} diff --git a/src/service_registry.rs b/src/service_registry.rs new file mode 100644 index 0000000..64c386e --- /dev/null +++ b/src/service_registry.rs @@ -0,0 +1,74 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use tracing::debug; + +#[derive(Clone)] +pub struct ServiceRegistry { + services: Arc>>>, +} + +impl Default for ServiceRegistry { + fn default() -> Self { + Self { + services: Arc::new(Mutex::new(Vec::new())), + } + } +} + +impl ServiceRegistry { + pub fn register(&self, name: &str) -> ServiceSignal { + let state = Arc::new(ServiceState { + name: name.to_string(), + ready: AtomicBool::new(false), + }); + + let mut services = self.services.lock().unwrap(); + services.push(Arc::clone(&state)); + debug!(%name, "Service registered"); + ServiceSignal { state } + } + + pub fn get_not_ready_services(&self) -> Vec { + let services = self.services.lock().unwrap(); + let mut result = Vec::new(); + for service in services.iter() { + if !service.ready.load(Ordering::SeqCst) { + result.push(service.name.clone()); + } + } + + result + } +} + +pub struct ServiceSignal { + state: Arc, +} + +impl ServiceSignal { + pub fn ready(&self) { + self.state.ready.store(true, Ordering::SeqCst); + debug!(%self.state.name, "Service ready"); + } +} + +struct ServiceState { + name: String, + ready: AtomicBool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_be_ready_after_all_registered_service_ready() { + let registry = ServiceRegistry::default(); + let signal = registry.register("test"); + + assert!(!registry.get_not_ready_services().is_empty()); + signal.ready(); + assert!(registry.get_not_ready_services().is_empty()); + } +} diff --git a/src/shutdown.rs b/src/shutdown.rs new file mode 100644 index 0000000..85cf9bc --- /dev/null +++ b/src/shutdown.rs @@ -0,0 +1,116 @@ +use std::future::Future; + +use async_shutdown::{ + DelayShutdownToken, ShutdownAlreadyCompleted, ShutdownComplete, ShutdownManager, + ShutdownSignal, WrapDelayShutdown, +}; +use eyre::Result; +use tokio::signal; +use tracing::info; + +#[derive(Clone)] +pub struct Shutdown { + drain: ShutdownManager<()>, + shutdown: ShutdownManager<()>, +} + +impl Shutdown { + #[allow(clippy::new_without_default)] + pub fn new() -> Shutdown { + Self::new_with_drain_signal(shutdown_signal()) + } + + pub fn new_with_drain_signal(signal: F) -> Shutdown + where + F: Future + Send + Sync + 'static, + { + let drain = ShutdownManager::new(); + let shutdown = ShutdownManager::new(); + + tokio::spawn({ + let drain = drain.clone(); + let shutdown = shutdown.clone(); + + async move { + signal.await; + + info!("Drain start"); + _ = drain.trigger_shutdown(()); + drain.wait_shutdown_complete().await; + + info!("Shutdown start"); + _ = shutdown.trigger_shutdown(()); + } + }); + + Shutdown { drain, shutdown } + } + + pub fn is_drain_triggered(&self) -> bool { + self.drain.is_shutdown_triggered() + } + + pub fn wait_drain_triggered(&self) -> ShutdownSignal<()> { + self.drain.wait_shutdown_triggered() + } + + pub fn wait_drain_complete(&self) -> ShutdownComplete<()> { + self.drain.wait_shutdown_complete() + } + + pub fn delay_drain_token( + &self, + ) -> Result, ShutdownAlreadyCompleted<()>> { + self.drain.delay_shutdown_token() + } + + pub fn wrap_delay_shutdown( + &self, + future: F, + ) -> Result, ShutdownAlreadyCompleted<()>> { + self.shutdown.wrap_delay_shutdown(future) + } + + pub fn trigger_shutdown(&self) { + _ = self.drain.trigger_shutdown(()); + _ = self.shutdown.trigger_shutdown(()); + } + + pub fn is_shutdown_triggered(&self) -> bool { + self.shutdown.is_shutdown_triggered() + } + + pub fn wait_shutdown_triggered(&self) -> ShutdownSignal<()> { + self.shutdown.wait_shutdown_triggered() + } + + pub fn wait_shutdown_complete(&self) -> ShutdownComplete<()> { + self.shutdown.wait_shutdown_complete() + } +} + +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler") + }; + + #[cfg(not(unix))] + ctrl_c.await; + + #[cfg(unix)] + { + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + }; + } +} diff --git a/src/spawn_service.rs b/src/spawn_service.rs new file mode 100644 index 0000000..36acf86 --- /dev/null +++ b/src/spawn_service.rs @@ -0,0 +1,173 @@ +use std::future::Future; +use std::time::Duration; + +use eyre::{Context, Result}; +use tokio::task::{JoinError, JoinHandle}; +use tokio::{select, spawn}; +use tracing::{debug, error, span, warn, Instrument, Level}; + +use crate::shutdown::Shutdown; + +#[derive(Debug)] +pub enum ServiceExit { + GracefulShutdown, + EarlyStop, + Panic(JoinError), +} + +pub fn spawn_service( + shutdown: &Shutdown, + name: impl Into, + future: impl Future + Send + 'static, +) -> Result> { + let shutdown = shutdown.clone(); + let service_name = name.into(); + + let wrapped = { + let shutdown = shutdown.clone(); + async move { + match spawn(future).await { + Ok(_) if shutdown.is_shutdown_triggered() => ServiceExit::GracefulShutdown, + Ok(_) => { + shutdown.trigger_shutdown(); + ServiceExit::EarlyStop + } + Err(err) => { + shutdown.trigger_shutdown(); + ServiceExit::Panic(err) + } + } + } + }; + + let logged = { + let shutdown = shutdown.clone(); + async move { + let mut wrapped = Box::pin(wrapped); + let shutdown_log = async move { + shutdown.wait_shutdown_triggered().await; + tokio::time::sleep(Duration::from_secs(3)).await; + }; + + debug!("Service starting"); + select! { + exit = &mut wrapped => { + match &exit { + ServiceExit::GracefulShutdown => { + debug!("Service gracefully shutdown") + } + ServiceExit::EarlyStop => error!("Service stopped early"), + ServiceExit::Panic(err) => error!(%err, "Service panicked"), + } + exit + }, + _ = shutdown_log => { + warn!("Service shutdown is taking some time"); + wrapped.await + }, + } + } + }; + + let instrumented = logged.instrument(span!(Level::ERROR, "service", "{}", service_name)); + + let waited = shutdown + .wrap_delay_shutdown(instrumented) + .context(service_name)?; + + Ok(spawn(waited)) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::assert_matches; + + #[tokio::test] + async fn graceful_shutdown_on_shutdown_request() { + let shutdown = Shutdown::new(); + let handle = spawn_service(&shutdown, "test", { + let shutdown = shutdown.clone(); + async move { + shutdown.wait_shutdown_triggered().await; + tokio::time::sleep(Duration::from_micros(500)).await; + } + }) + .unwrap(); + + shutdown.trigger_shutdown(); + + assert_matches!(handle.await, Ok(ServiceExit::GracefulShutdown)); + } + + #[tokio::test] + async fn should_capture_early_shutdown() { + let shutdown = Shutdown::new(); + let handle = spawn_service(&shutdown, "test", async move { + tokio::time::sleep(Duration::from_micros(500)).await; + }) + .unwrap(); + + assert_matches!(handle.await, Ok(ServiceExit::EarlyStop)); + } + + #[tokio::test] + async fn should_capture_panic() { + let shutdown = Shutdown::new(); + let handle = spawn_service(&shutdown, "test", async move { + tokio::time::sleep(Duration::from_micros(500)).await; + panic!(); + }) + .unwrap(); + + assert_matches!(handle.await, Ok(ServiceExit::Panic(_))); + } + + #[tokio::test] + async fn should_early_shutdown_trigger_others_graceful_shutdown() { + let shutdown = Shutdown::new(); + let handle = spawn_service(&shutdown, "test", async move { + tokio::time::sleep(Duration::from_micros(500)).await; + }) + .unwrap(); + + let other_handle = spawn_service(&shutdown, "other", { + let shutdown = shutdown.clone(); + async move { + shutdown.wait_shutdown_triggered().await; + tokio::time::sleep(Duration::from_micros(500)).await; + } + }) + .unwrap(); + + assert_matches!(handle.await, Ok(ServiceExit::EarlyStop)); + assert!(shutdown.is_shutdown_triggered()); + assert_matches!(other_handle.await, Ok(ServiceExit::GracefulShutdown)); + } + + #[tokio::test] + async fn should_panic_trigger_others_graceful_shutdown() { + let shutdown = Shutdown::new(); + let handle = spawn_service(&shutdown, "test", async move { + tokio::time::sleep(Duration::from_micros(500)).await; + panic!(); + }) + .unwrap(); + + let other_handle = spawn_service(&shutdown, "other", { + let shutdown = shutdown.clone(); + async move { + shutdown.wait_shutdown_triggered().await; + tokio::time::sleep(Duration::from_micros(500)).await; + } + }) + .unwrap(); + + assert_matches!(handle.await, Ok(ServiceExit::Panic(_))); + assert!(shutdown.is_shutdown_triggered()); + shutdown.wait_shutdown_complete().await; + assert_matches!(other_handle.await, Ok(ServiceExit::GracefulShutdown)); + } +} diff --git a/src/status.rs b/src/status.rs new file mode 100644 index 0000000..5896a3e --- /dev/null +++ b/src/status.rs @@ -0,0 +1,72 @@ +use kube::error::ErrorResponse; +use kube::Error; + +const STATUS_CODE_404_NOT_FOUND: u16 = 404; +const STATUS_CODE_408_TIMEOUT: u16 = 408; +const STATUS_CODE_409_CONFLICT: u16 = 409; +const STATUS_CODE_410_GONE: u16 = 410; +const STATUS_CODE_422_UNPROCESSABLE_ENTITY: u16 = 422; +const STATUS_CODE_429_TOO_MANY_REQUESTS: u16 = 429; +const STATUS_CODE_500_INTERNAL_SERVER_ERROR: u16 = 500; +const STATUS_CODE_502_BAD_GATEWAY: u16 = 502; +const STATUS_CODE_503_SERVICE_UNAVAILABLE: u16 = 503; +const STATUS_CODE_504_GATEWAY_TIMEOUT: u16 = 504; + +pub fn is_404_not_found_error(err: &Error) -> bool { + matches!( + err, + Error::Api(ErrorResponse { + code: STATUS_CODE_404_NOT_FOUND, + .. + }) + ) +} + +pub fn is_409_conflict_error(err: &Error) -> bool { + matches!( + err, + Error::Api(ErrorResponse { + code: STATUS_CODE_409_CONFLICT, + .. + }) + ) +} + +pub fn is_410_gone_error(err: &Error) -> bool { + matches!( + err, + Error::Api(ErrorResponse { + code: STATUS_CODE_410_GONE, + .. + }) + ) +} + +pub fn is_generic_server_response_422_invalid_for_json_patch_error(err: &Error) -> bool { + matches!( + err, + Error::Api(ErrorResponse { + code, + reason, + .. + }) if *code == STATUS_CODE_422_UNPROCESSABLE_ENTITY && reason == "Invalid" + ) +} + +pub fn is_transient_error(err: &Error) -> bool { + match err { + Error::Api(ErrorResponse { + code: + STATUS_CODE_408_TIMEOUT + | STATUS_CODE_429_TOO_MANY_REQUESTS + | STATUS_CODE_500_INTERNAL_SERVER_ERROR + | STATUS_CODE_502_BAD_GATEWAY + | STATUS_CODE_503_SERVICE_UNAVAILABLE + | STATUS_CODE_504_GATEWAY_TIMEOUT.., + .. + }) => true, + + // TODO: Handle more transient err + _ => false, + } +} diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..9cf0f6c --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,147 @@ +use eyre::eyre; +use eyre::Result; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::DeleteOptions; +use kube::api::{DeleteParams, Preconditions, PropagationPolicy}; +use kube::runtime::reflector::ObjectRef; +use kube::Resource; + +pub fn get_object_ref_from_name( + name: impl AsRef, + ns: Option>, +) -> ObjectRef +where + K::DynamicType: Default, +{ + let object_ref = ObjectRef::new(name.as_ref()); + match ns { + Some(ns) => object_ref.within(ns.as_ref()), + None => object_ref, + } +} + +pub(crate) fn to_delete_params( + delete_options: DeleteOptions, + dry_run: bool, +) -> Result { + let preconditions = delete_options + .preconditions + .map(|preconditions| Preconditions { + uid: preconditions.uid.clone(), + resource_version: preconditions.resource_version.clone(), + }); + + let propagation_policy = + if let Some(propagation_policy) = delete_options.propagation_policy.as_ref() { + match propagation_policy.as_str() { + "Orphan" => Some(PropagationPolicy::Orphan), + "Background" => Some(PropagationPolicy::Background), + "Foreground" => Some(PropagationPolicy::Foreground), + other => return Err(eyre!("Unknown propagation policy: '{other}'")), + } + } else { + None + }; + + Ok(DeleteParams { + dry_run, + grace_period_seconds: delete_options.grace_period_seconds.map(|x| x as _), + preconditions, + propagation_policy, + }) +} + +#[macro_export] +macro_rules! instrumented { + ($span:expr, $($tt:tt)+) => {{ + use ::tracing::Instrument; + + let span = $span; + { + $($tt)* + } + .instrument(span) + .await + }} +} + +#[cfg(test)] +#[macro_export] +macro_rules! assert_matches { + ($expr:expr, $($tt:tt)+) => {{ + let value = $expr; + match value { + $($tt)* => {} + _ => ::std::panic!( + "Expression = `{}`, value = `{:?}` does not match with pattern = `{}`.", + stringify!($expr), + value, + stringify!($($tt)*), + ), + } + }}; +} + +#[macro_export] +macro_rules! try_some { + (@coalesce ($($h:tt)*)) => { + $($h)* + }; + (@coalesce ($($h:tt)*) ? $($t:tt)*) => { + $crate::try_some!(@coalesce ($($h)*.as_ref()?) $($t)*) + }; + (@coalesce ($($h:tt)*) $m:tt $($t:tt)*) => { + $crate::try_some!(@coalesce ($($h)* $m) $($t)*) + }; + + (@coalesce_mut ($($h:tt)*)) => { + $($h)* + }; + (@coalesce_mut ($($h:tt)*) ? $($t:tt)*) => { + $crate::try_some!(@coalesce_mut ($($h)*.as_mut()?) $($t)*) + }; + (@coalesce_mut ($($h:tt)*) $m:tt $($t:tt)*) => { + $crate::try_some!(@coalesce_mut ($($h)* $m) $($t)*) + }; + + (mut $($tt:tt)*) => { + { + fn call(f: impl FnOnce() ->::std::option::Option) -> ::std::option::Option { + f() + } + call(|| { + ::std::option::Option::Some($crate::try_some!(@coalesce_mut () $($tt)*)) + }) + } + }; + (&mut $($tt:tt)*) => { + { + fn call(f: impl FnOnce() ->::std::option::Option) -> ::std::option::Option + { + f() + } + call(|| { + ::std::option::Option::Some(&mut $crate::try_some!(@coalesce_mut () $($tt)*)) + }) + } + }; + (& $($tt:tt)*) => { + { + fn call(f: impl FnOnce() ->::std::option::Option) -> ::std::option::Option { + f() + } + call(|| { + ::std::option::Option::Some(&$crate::try_some!(@coalesce () $($tt)*)) + }) + } + }; + ($($tt:tt)*) => { + { + fn call(f: impl FnOnce() ->::std::option::Option) -> ::std::option::Option { + f() + } + call(|| { + ::std::option::Option::Some($crate::try_some!(@coalesce () $($tt)*)) + }) + } + }; +} diff --git a/src/webhooks/config.rs b/src/webhooks/config.rs new file mode 100644 index 0000000..ac737ad --- /dev/null +++ b/src/webhooks/config.rs @@ -0,0 +1,44 @@ +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +pub struct WebhookConfig { + pub(crate) bind: BindConfig, + pub(crate) cert: CertConfig, +} + +pub enum BindConfig { + SocketAddr(SocketAddr), + RandomForTest, +} + +pub enum CertConfig { + // Find certs `{CertDir}/{tls.crt,tls.key}` + CertDir(PathBuf), + // Override cert for test + Override(CertificateDer<'static>, PrivateKeyDer<'static>), +} + +impl WebhookConfig { + pub fn controller_runtime_default() -> Self { + // `sigs.k8s.io/controller-runtime` look for `{TempDir}/k8s-webhook-server/serving-certs/{tls.key,tls.crt}` files by default. + let temp_dir = std::env::temp_dir(); + let default_path = temp_dir.join(Path::new("k8s-webhook-server/serving-certs")); + Self { + bind: BindConfig::SocketAddr(SocketAddr::from(([0, 0, 0, 0], 9443))), + cert: CertConfig::CertDir(default_path), + } + } +} + +impl WebhookConfig { + pub fn random_port_for_test( + cert: CertificateDer<'static>, + key_pair_der: PrivateKeyDer<'static>, + ) -> Self { + Self { + bind: BindConfig::RandomForTest, + cert: CertConfig::Override(cert, key_pair_der), + } + } +} diff --git a/src/webhooks/handle_delete.rs b/src/webhooks/handle_delete.rs new file mode 100644 index 0000000..29e0743 --- /dev/null +++ b/src/webhooks/handle_delete.rs @@ -0,0 +1,178 @@ +use chrono::{Duration, SecondsFormat, Utc}; +use eyre::{eyre, Context, Result}; +use k8s_openapi::api::authentication::v1::UserInfo; +use k8s_openapi::api::core::v1::Pod; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::DeleteOptions; +use k8s_openapi::apimachinery::pkg::runtime::RawExtension; +use kube::core::admission::AdmissionRequest; +use kube::ResourceExt; +use serde::Deserialize; + +use crate::pod_draining_info::{get_pod_draining_info, PodDrainingInfo}; +use crate::pod_state::{is_pod_exposed, is_pod_ready}; +use crate::utils::to_delete_params; +use crate::webhooks::report::{debug_report_for, report_for}; +use crate::webhooks::{patch_pod_isolate, AppState, InterceptResult}; +use crate::ApiResolver; + +/// This handler delays the admission of DELETE Pod request. +/// +/// We can't patch out the DELETE Pod request, so we delay it. +/// +/// # Compatibility +/// +/// The handler cannot deny the request due to the following compatibility reasons. +/// +/// * `kubectl drain --disable-eviction`: fail and stop if it meets the first pod that cannot be deleted. +/// * `kubectl delete`: returns non-zero exit code and prints the reason of denial. +/// Human operators might be able to read the reason, but machines don't. +/// We might break some existing tools that wraps `kubectl delete` if we deny the request. +/// +/// These are known to be fine with the admission request denial. +/// +/// * ReplicaSet controller: it can retry and progress. +/// * `kubectl rollout restart`: It patches the deployment's annotation `kubectl.kubernetes.io/restartedAt`, +/// so it is controlled by ReplicaSet controller. +pub async fn delete_handler( + state: &AppState, + request: &AdmissionRequest, + user_info: &UserInfo, +) -> Result { + let pod = request + .old_object + .as_ref() + .ok_or(eyre!("old_object for validation is missing"))?; + + match get_pod_draining_info(pod) { + PodDrainingInfo::None => { + if !is_pod_exposed(&state.config, &state.stores, pod) { + debug_report_for( + state, + pod, + "AllowDeletion", + "NotExposed", + "Deletion is allowed because the pod is not exposed".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + if !is_pod_ready(pod) { + debug_report_for( + state, + pod, + "AllowDeletion", + "NotReady", + "Deletion is allowed because the pod is not ready".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + let drain_until = Utc::now() + Duration::from_std(state.config.delete_after)?; + check_delete_permission(&state.api_resolver, pod, &request.options, user_info) + .await + .context("checking permission")?; + let patched_result = patch_pod_isolate( + &state.api_resolver, + pod, + drain_until, + None, + &state.loadbalancing, + ) + .await + .context("apply patch")?; + + if patched_result.is_none() { + debug_report_for( + state, + pod, + "AllowDeletion", + "Gone", + "Pod is already gone".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + report_for( + state, + pod, + "DelayDeletion", + "Drain", + format!( + "Deletion is delayed, and the pod is isolated. It'll be deleted after '{}'", + drain_until.to_rfc3339_opts(SecondsFormat::Secs, true), + ), + ) + .await; + + let duration = (drain_until - Utc::now()).to_std().unwrap_or_default(); + Ok(InterceptResult::Delay(duration)) + } + PodDrainingInfo::DrainUntil(drain_until) => { + if let Ok(duration) = (drain_until - Utc::now()).to_std() { + report_for( + state, + pod, + "DelayDeletion", + "Draining", + format!( + "Deletion is delayed. It'll be deleted after '{}'", + drain_until.to_rfc3339_opts(SecondsFormat::Secs, true), + ), + ) + .await; + + Ok(InterceptResult::Delay(duration)) + } else { + debug_report_for( + state, + pod, + "AllowDeletion", + "Expired", + "Deletion is allowed because the pod is drained enough".to_string(), + ) + .await; + + Ok(InterceptResult::Allow) + } + } + PodDrainingInfo::Deleted => Ok(InterceptResult::Allow), + PodDrainingInfo::DrainDisabled => { + debug_report_for( + state, + pod, + "AllowDeletion", + "Disabled", + "Pod graceful drain is disabled".to_string(), + ) + .await; + + Ok(InterceptResult::Allow) + } + PodDrainingInfo::AnnotationParseError { message } => Err(eyre!(message)), + } +} + +async fn check_delete_permission( + api_resolver: &ApiResolver, + pod: &Pod, + raw_options: &Option, + user_info: &UserInfo, +) -> Result<()> { + let api = api_resolver + .impersonate_as(user_info.username.clone(), user_info.groups.clone())? + .api_for(pod); + + let delete_options = if let Some(delete_options) = raw_options { + DeleteOptions::deserialize(&delete_options.0)? + } else { + DeleteOptions::default() + }; + + let name = pod.name_any(); + let delete_params = to_delete_params(delete_options, true)?; + api.delete(&name, &delete_params).await?; + Ok(()) +} diff --git a/src/webhooks/handle_eviction.rs b/src/webhooks/handle_eviction.rs new file mode 100644 index 0000000..a511592 --- /dev/null +++ b/src/webhooks/handle_eviction.rs @@ -0,0 +1,196 @@ +use chrono::{Duration, SecondsFormat, Utc}; +use eyre::{eyre, Context, Result}; +use k8s_openapi::api::authentication::v1::UserInfo; +use k8s_openapi::api::core::v1::{ObjectReference, Pod}; +use k8s_openapi::api::policy::v1::Eviction; +use kube::api::{EvictParams, PostParams}; +use kube::core::admission::{AdmissionRequest, AdmissionResponse}; +use kube::{Api, ResourceExt}; + +use crate::pod_draining_info::{get_pod_draining_info, PodDrainingInfo}; +use crate::pod_state::{is_pod_exposed, is_pod_ready}; +use crate::utils::{get_object_ref_from_name, to_delete_params}; +use crate::webhooks::patch::make_patch_eviction_to_dry_run; +use crate::webhooks::report::{debug_report_for, report_for}; +use crate::webhooks::{debug_report_for_ref, patch_pod_isolate, AppState, InterceptResult}; +use crate::{try_some, ApiResolver}; + +/// The handler patches CREATE Eviction request as dry-run. +/// The controller will delete them later anyhow. +/// +/// # Compatibility +/// +/// The handler cannot deny the admission request due to the following compatibility reasons. +/// +/// * `kubectl drain`: fail and stop if it meets the first pod that cannot be deleted. +pub async fn eviction_handler( + state: &AppState, + request: &AdmissionRequest, + user_info: &UserInfo, +) -> Result { + let eviction = request + .object + .as_ref() + .ok_or(eyre!("object for mutation is missing"))?; + + let object_ref = get_object_ref_from_name(&request.name, request.namespace.as_ref()); + if let Some(dry_run) = try_some!(eviction.delete_options?.dry_run?) { + if !dry_run.is_empty() { + debug_report_for_ref( + state, + ObjectReference::from(object_ref), + "AllowEviction", + "DryRun", + format!("Eviction request is allowed because `eviction.deleteOptions.dryRun = {dry_run:?}`"), + ) + .await; + return Ok(InterceptResult::Allow); + } + } + + let pod = state + .stores + .get_pod(&object_ref) + .ok_or(eyre!("pod is not found"))?; + + let draining = get_pod_draining_info(&pod); + match draining { + PodDrainingInfo::None => { + if !is_pod_exposed(&state.config, &state.stores, &pod) { + debug_report_for( + state, + &pod, + "AllowEviction", + "NotExposed", + "Eviction is allowed because the pod is not exposed".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + if !is_pod_ready(&pod) { + debug_report_for( + state, + &pod, + "AllowEviction", + "NotReady", + "Eviction is allowed because the pod is not ready".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + let drain_until = Utc::now() + Duration::from_std(state.config.delete_after)?; + check_eviction_permission(&state.api_resolver, eviction, user_info) + .await + .context("checking permission")?; + let patched_result = patch_pod_isolate( + &state.api_resolver, + &pod, + drain_until, + eviction.delete_options.as_ref(), + &state.loadbalancing, + ) + .await + .context("apply patch")?; + + if patched_result.is_none() { + debug_report_for( + state, + &pod, + "AllowDeletion", + "Gone", + "Pod is already gone".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + + report_for( + state, + &pod, + "InterceptEviction", + "Drain", + format!( + "Eviction is intercepted, and the pod is isolated. It'll be deleted after '{}'", + drain_until.to_rfc3339_opts(SecondsFormat::Secs, true), + ), + ) + .await; + } + PodDrainingInfo::DrainUntil(drain_until) => { + if Utc::now() > drain_until { + debug_report_for( + state, + &pod, + "AllowEviction", + "Expired", + "Eviction is allowed because the pod is drained enough".to_string(), + ) + .await; + + return Ok(InterceptResult::Allow); + } + + report_for( + state, + &pod, + "InterceptEviction", + "Draining", + format!( + "Eviction is intercepted. It'll be deleted after '{}'", + drain_until.to_rfc3339_opts(SecondsFormat::Secs, true), + ), + ) + .await; + } + PodDrainingInfo::Deleted => { + return Ok(InterceptResult::Allow); + } + PodDrainingInfo::DrainDisabled => { + debug_report_for( + state, + &pod, + "InterceptEviction", + "Disabled", + "Pod graceful drain is disabled".to_string(), + ) + .await; + return Ok(InterceptResult::Allow); + } + PodDrainingInfo::AnnotationParseError { message } => { + return Err(eyre!(message)); + } + }; + + let eviction_patch = make_patch_eviction_to_dry_run(eviction).context("patch")?; + let response = AdmissionResponse::from(request) + .with_patch(eviction_patch) + .context("attaching patch")?; + + Ok(InterceptResult::Patch(Box::new(response))) +} + +async fn check_eviction_permission( + api_resolver: &ApiResolver, + eviction: &Eviction, + user_info: &UserInfo, +) -> Result<()> { + let api: Api = api_resolver + .impersonate_as(user_info.username.clone(), user_info.groups.clone())? + .all(); + + let name = eviction.name_any(); + let delete_params = + to_delete_params(eviction.delete_options.clone().unwrap_or_default(), true)?; + let evict_params = EvictParams { + delete_options: Some(delete_params), + post_options: PostParams { + dry_run: true, + ..PostParams::default() + }, + }; + + api.evict(&name, &evict_params).await?; + Ok(()) +} diff --git a/src/webhooks/mod.rs b/src/webhooks/mod.rs new file mode 100644 index 0000000..d44dfdd --- /dev/null +++ b/src/webhooks/mod.rs @@ -0,0 +1,247 @@ +mod config; +mod handle_delete; +mod handle_eviction; +mod patch; +mod reactive_rustls_config; +mod report; +mod try_bind; + +use std::fmt::Debug; +use std::future::Future; +use std::net::SocketAddr; +use std::time::Duration; + +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{extract::State, routing::post, Json, Router}; +use eyre::Result; +use k8s_openapi::api::authentication::v1::UserInfo; +use k8s_openapi::api::core::v1::ObjectReference; +use k8s_openapi::api::{core::v1::Pod, policy::v1::Eviction}; +use k8s_openapi::serde::Serialize; +use kube::core::admission::{AdmissionRequest, AdmissionResponse, AdmissionReview}; +use kube::core::DynamicObject; +use kube::runtime::events::{EventType, Reporter}; +use kube::runtime::reflector::ObjectRef; +use kube::Resource; +use serde_json::{json, Value}; +use tracing::{info, span, trace, Level}; + +use crate::api_resolver::ApiResolver; +use crate::config::Config; +use crate::consts::CONTROLLER_NAME; +use crate::reflector::Stores; +use crate::shutdown::Shutdown; +use crate::spawn_service::spawn_service; +use crate::utils::get_object_ref_from_name; +pub use crate::webhooks::config::WebhookConfig; +use crate::webhooks::handle_delete::delete_handler; +use crate::webhooks::handle_eviction::eviction_handler; +pub use crate::webhooks::patch::patch_pod_isolate; +use crate::webhooks::reactive_rustls_config::build_reactive_rustls_config; +use crate::webhooks::report::{debug_report_for_ref, report}; +use crate::webhooks::try_bind::try_bind; +use crate::{instrumented, LoadBalancingConfig, ServiceRegistry}; + +/// Start an admission webhook that intercepts pod deletion, pod eviction requests. +pub async fn start_webhook( + api_resolver: &ApiResolver, + config: Config, + webhook_config: WebhookConfig, + stores: Stores, + service_registry: &ServiceRegistry, + loadbalancing: &LoadBalancingConfig, + shutdown: &Shutdown, +) -> Result { + let app = Router::new() + .route("/healthz", get(healthz_handler)) + .route("/merics", get(metrics_handler)) + .route("/webhook/mutate", post(mutate_handler)) + .route("/webhook/validate", post(validate_handler)) + .with_state(AppState { + api_resolver: api_resolver.clone(), + config: config.clone(), + stores, + service_registry: service_registry.clone(), + loadbalancing: loadbalancing.clone(), + event_reporter: Reporter { + controller: String::from(CONTROLLER_NAME), + instance: hostname::get() + .ok() + .and_then(|n| n.to_str().map(String::from)), + }, + }); + + let rustls_config = build_reactive_rustls_config(&webhook_config.cert, shutdown).await?; + + let addr_incoming = try_bind(&webhook_config.bind).await?; + let local_addr = addr_incoming.local_addr()?; + info!("listening {}", local_addr); + + let handle = axum_server::Handle::new(); + let server = { + axum_server::bind_rustls(local_addr, rustls_config) + .handle(handle.clone()) + .serve(app.into_make_service()) + }; + + tokio::spawn({ + let shutdown = shutdown.clone(); + let handle = handle.clone(); + let draining_graceful_period = config.delete_after; + + async move { + shutdown.wait_drain_triggered().await; + handle.graceful_shutdown(Some(draining_graceful_period)); + shutdown.wait_drain_complete().await; + } + }); + + let signal = service_registry.register("webhook"); + spawn_service(shutdown, "webhook", { + let shutdown = shutdown.clone(); + + async move { + let _drain_token = shutdown.delay_drain_token(); + signal.ready(); + server.await.unwrap(); + } + })?; + + Ok(local_addr) +} + +#[derive(Clone)] +struct AppState { + api_resolver: ApiResolver, + config: Config, + stores: Stores, + service_registry: ServiceRegistry, + event_reporter: Reporter, + loadbalancing: LoadBalancingConfig, +} + +async fn healthz_handler(State(state): State) -> (StatusCode, Json) { + let not_ready = state.service_registry.get_not_ready_services(); + let status_code = if not_ready.is_empty() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + (status_code, Json(json!({ "not_ready": not_ready }))) +} + +async fn metrics_handler(State(_state): State) -> StatusCode { + // TODO + StatusCode::OK +} + +async fn mutate_handler( + State(state): State, + Json(review): Json>, +) -> ValueOrStatusCode> { + handle_common(eviction_handler, &state, &review).await +} + +async fn validate_handler( + State(state): State, + Json(review): Json>, +) -> ValueOrStatusCode> { + handle_common(delete_handler, &state, &review).await +} + +#[derive(Debug, Clone, Copy)] +enum ValueOrStatusCode { + Value(T), + StatusCode(StatusCode), +} + +impl IntoResponse for ValueOrStatusCode +where + Json: IntoResponse, +{ + fn into_response(self) -> Response { + match self { + ValueOrStatusCode::Value(value) => Json(value).into_response(), + ValueOrStatusCode::StatusCode(status_code) => status_code.into_response(), + } + } +} + +enum InterceptResult { + Allow, + Delay(Duration), + Patch(Box), +} + +async fn handle_common<'a, K, Fut>( + handle: impl FnOnce(&'a AppState, &'a AdmissionRequest, &'a UserInfo) -> Fut, + state: &'a AppState, + review: &'a AdmissionReview, +) -> ValueOrStatusCode> +where + K: Resource + Debug + Serialize, + K::DynamicType: Default, + Fut: Future>, +{ + let request = match &review.request { + Some(request) => request, + None => return ValueOrStatusCode::StatusCode(StatusCode::BAD_REQUEST), + }; + + let object_ref: ObjectRef = + get_object_ref_from_name(&request.name, request.namespace.as_ref()); + let request_id: u32 = rand::random(); + instrumented!( + span!(Level::ERROR, "admission", %object_ref, operation = ?request.operation, request_id), + async move { + trace!(user_info=?request.user_info); + + if request.dry_run { + debug_report_for_ref( + state, + ObjectReference::from(object_ref), + "Allow", + "DryRun", + format!( + "operation={:?}, kind={}", + request.operation, + ::kind(&Default::default()) + ), + ) + .await; + + return ValueOrStatusCode::Value(AdmissionResponse::from(request).into_review()); + } + + let result = handle(state, request, &request.user_info).await; + + match result { + Ok(InterceptResult::Allow) => { + ValueOrStatusCode::Value(AdmissionResponse::from(request).into_review()) + } + Ok(InterceptResult::Delay(duration)) => { + tokio::time::sleep(duration).await; + ValueOrStatusCode::Value(AdmissionResponse::from(request).into_review()) + } + Ok(InterceptResult::Patch(response)) => { + ValueOrStatusCode::Value(response.into_review()) + } + Err(err) => { + report( + state, + ObjectReference::from(object_ref), + EventType::Warning, + "Error", + "Error", + format!("{err:#}"), + ) + .await; + ValueOrStatusCode::StatusCode(StatusCode::INTERNAL_SERVER_ERROR) + } + } + } + ) +} diff --git a/src/webhooks/patch.rs b/src/webhooks/patch.rs new file mode 100644 index 0000000..1346860 --- /dev/null +++ b/src/webhooks/patch.rs @@ -0,0 +1,426 @@ +use std::fmt::Debug; + +use backoff::backoff::Backoff; +use backoff::ExponentialBackoff; +use chrono::{DateTime, SecondsFormat, Utc}; +use eyre::{eyre, Context, Result}; +use json_patch::{Patch, PatchOperation, TestOperation}; +use jsonptr::Pointer; +use k8s_openapi::api::{core::v1::Pod, policy::v1::Eviction}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::DeleteOptions; +use k8s_openapi::serde::de::DeserializeOwned; +use k8s_openapi::serde::Serialize; +use kube::api::PatchParams; +use kube::core::NamespaceResourceScope; +use kube::{Resource, ResourceExt}; +use serde_json::Value; +use tracing::trace; + +use crate::api_resolver::ApiResolver; +use crate::consts::{ + DELETE_OPTIONS_ANNOTATION_KEY, DRAINING_LABEL_KEY, DRAIN_CONTROLLER_ANNOTATION_KEY, + DRAIN_UNTIL_ANNOTATION_KEY, ORIGINAL_LABELS_ANNOTATION_KEY, +}; +use crate::pod_draining_info::{get_pod_draining_info, PodDrainingInfo}; +use crate::status::{ + is_404_not_found_error, is_409_conflict_error, is_410_gone_error, + is_generic_server_response_422_invalid_for_json_patch_error, is_transient_error, +}; +use crate::LoadBalancingConfig; + +async fn apply_patch( + api_resolver: &ApiResolver, + res: &K, + patch: impl Fn(&K) -> Result + Clone, + check: impl Fn(&K) -> bool + Clone, +) -> Result> +where + K: Resource + Clone + Serialize + DeserializeOwned + Debug, + K::DynamicType: Default, +{ + let api = api_resolver.api_for(res); + let name = res.name_any(); + + let mut res = res.clone(); + let mut backoff = ExponentialBackoff::default(); + 'patch: while !check(&res) { + let patch = patch(&res).context("patch")?; + trace!(?patch, "patching"); + let result = api + .patch( + &name, + &PatchParams::default(), + &kube::api::Patch::::Json(patch), + ) + .await; + + let err = match result { + Ok(new_res) => { + return Ok(Some(new_res)); + } + Err(err) if is_404_not_found_error(&err) || is_410_gone_error(&err) => { + return Ok(None); // this is what we desire. + } + Err(err) => err, + }; + + if !(is_transient_error(&err) + // kubernetes api server returns 422 when JsonPatch fails to test, not 409. + // SEE: https://github.com/kubernetes/kubernetes/blob/2a1d4172e22abb6759b3d2ad21bb09a04eef596d/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go#L394 + || is_generic_server_response_422_invalid_for_json_patch_error(&err) + // Conflict is to reduce future confusion. + || is_409_conflict_error(&err)) + { + return Err(err.into()); + } + + // transient errors, conflict errors + 'refresh: loop { + if let Some(backoff) = backoff.next_backoff() { + tokio::time::sleep(backoff).await; + } else { + return Err(eyre!("no more backoff")); + } + + let refreshed = api.get(&name).await; + match refreshed { + Err(err) if is_404_not_found_error(&err) || is_410_gone_error(&err) => { + // Resource is gone + return Ok(None); + } + Err(err) if is_transient_error(&err) => { + continue 'refresh; + } + Err(err) => { + return Err(err.into()); + } + Ok(refreshed) => { + if res.meta().resource_version != refreshed.meta().resource_version { + res = refreshed; + continue 'patch; + } + + return Err(eyre!("resource isn't changed after the refresh")); + } + } + } + } + + Ok(Some(res)) +} + +fn make_patch( + res: &K, + modify: impl Fn(&mut K) -> Result<()>, +) -> Result { + let before = serde_json::to_value(res).context("serialize")?; + let after = { + let mut modified = res.clone(); + modify(&mut modified).context("modify")?; + serde_json::to_value(modified).context("serialize modified")? + }; + + let patch = json_patch::diff(&before, &after); + Ok(patch) +} + +fn prepend_uid_and_resource_version_test(mut patch: Patch, pod: &Pod) -> Result { + let uid = pod.uid().ok_or(eyre!("no uid"))?; + let version = pod.resource_version().ok_or(eyre!("no resource version"))?; + patch.0.insert( + 0, + PatchOperation::Test(TestOperation { + path: Pointer::new(["metadata", "uid"]), + value: Value::String(uid), + }), + ); + patch.0.insert( + 1, + PatchOperation::Test(TestOperation { + path: Pointer::new(["metadata", "resourceVersion"]), + value: Value::String(version), + }), + ); + + Ok(patch) +} + +pub async fn patch_pod_isolate( + api_resolver: &ApiResolver, + pod: &Pod, + drain_until: DateTime, + eviction_delete_options: Option<&DeleteOptions>, + loadbalancing: &LoadBalancingConfig, +) -> Result> { + let res = apply_patch( + api_resolver, + pod, + |pod| make_patch_pod_isolate(pod, drain_until, eviction_delete_options, loadbalancing), + |pod| !matches!(get_pod_draining_info(pod), PodDrainingInfo::None), + ) + .await?; + Ok(res) +} + +fn make_patch_pod_isolate( + pod: &Pod, + drain_until: DateTime, + eviction_delete_options: Option<&DeleteOptions>, + loadbalancing: &LoadBalancingConfig, +) -> Result { + let patch = make_patch(pod, |pod| { + backup_original_labels(pod).context("backup")?; + set_draining_label(pod); + set_drain_until_annotation(pod, drain_until); + if let Some(eviction_delete_options) = eviction_delete_options { + set_eviction_delete_options(pod, eviction_delete_options)?; + } + set_controller_annotation(pod, loadbalancing); + remove_owner_reference(pod); + Ok(()) + })?; + return prepend_uid_and_resource_version_test(patch, pod); + + fn backup_original_labels(pod: &mut Pod) -> Result<()> { + let labels = pod.labels_mut(); + let original_labels = serde_json::to_string(labels).context("serialize old labels")?; + labels.clear(); + pod.annotations_mut().insert( + String::from(ORIGINAL_LABELS_ANNOTATION_KEY), + original_labels, + ); + Ok(()) + } + + fn set_draining_label(pod: &mut Pod) { + pod.labels_mut() + .insert(String::from(DRAINING_LABEL_KEY), String::from("true")); + } + + fn set_drain_until_annotation(pod: &mut Pod, drain_until: DateTime) { + let string = drain_until.to_rfc3339_opts(SecondsFormat::Secs, true); + pod.annotations_mut() + .insert(String::from(DRAIN_UNTIL_ANNOTATION_KEY), string); + } + + fn set_eviction_delete_options(pod: &mut Pod, delete_options: &DeleteOptions) -> Result<()> { + let annotation = serde_json::to_string(&DeleteOptions { + // this is not dry-run + dry_run: None, + // preconditions.uid is this pod, so it is duplicate. + // preconditions.resourceVersion will be voided by this patch. + preconditions: None, + kind: None, + api_version: None, + ..delete_options.clone() + }) + .context("serialize old labels")?; + + pod.annotations_mut() + .insert(String::from(DELETE_OPTIONS_ANNOTATION_KEY), annotation); + Ok(()) + } + + fn set_controller_annotation(pod: &mut Pod, loadbalancing: &LoadBalancingConfig) { + pod.annotations_mut().insert( + String::from(DRAIN_CONTROLLER_ANNOTATION_KEY), + loadbalancing.get_id(), + ); + } + + /// To stop the pod controller's GC kicking in, we remove the OwnerReferences. + fn remove_owner_reference(pod: &mut Pod) { + for owner_ref in pod.owner_references_mut() { + if owner_ref.api_version == "v1" && owner_ref.kind == "ReplicaSet" { + owner_ref.controller = None; + } + } + } +} + +pub fn make_patch_eviction_to_dry_run(eviction: &Eviction) -> Result { + return make_patch(eviction, set_dry_run); + + fn set_dry_run(eviction: &mut Eviction) -> Result<()> { + let delete_options = eviction.delete_options.clone().unwrap_or_default(); + eviction.delete_options = Some(DeleteOptions { + dry_run: Some(vec![String::from("All")]), + ..delete_options + }); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use chrono::DateTime; + use serde_json::{json, Value}; + use uuid::Uuid; + + macro_rules! from_json { + ($($json:tt)+) => { + ::serde_json::from_value(::serde_json::json!($($json)+)).expect("Invalid json") + }; + } + + fn apply(res: &K, patch: &Patch) -> Result + where + K: Serialize, + { + let mut modified = serde_json::to_value(res)?; + json_patch::patch(&mut modified, &patch.0)?; + Ok(modified) + } + + #[test] + fn pod_patch_isolate() { + let pod: Pod = from_json! ({ + "metadata": { + "uid": "uid1234", + "resourceVersion": "version1234", + "labels": { + "app": "test" + }, + "ownerReferences": [{ + "apiVersion": "v1", + "kind": "ReplicaSet", + "name": "owner", + "uid": "12345", + "controller": true, + }] + } + }); + + let drain_until = DateTime::parse_from_rfc3339("2023-02-08T15:30:00Z") + .unwrap() + .with_timezone(&Utc); + let loadbalancing = LoadBalancingConfig::new(Uuid::nil()); + let patch = make_patch_pod_isolate(&pod, drain_until, None, &loadbalancing).unwrap(); + + let applied = apply(&pod, &patch).unwrap(); + assert_eq!( + applied, + json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "uid": "uid1234", + "resourceVersion": "version1234", + "labels": { + "pod-graceful-drain/draining": "true", + }, + "annotations": { + "pod-graceful-drain/drain-until": "2023-02-08T15:30:00Z", + "pod-graceful-drain/controller": "00000000-0000-0000-0000-000000000000", + "pod-graceful-drain/original-labels": "{\"app\":\"test\"}", + }, + "ownerReferences": [{ + "apiVersion": "v1", + "kind": "ReplicaSet", + "name": "owner", + "uid": "12345", + }] + }, + }) + ); + } + + #[test] + fn pod_patch_isolate_should_contain_test_resource_version() { + let pod: Pod = from_json! ({ + "metadata": { + "uid": "uid1234", + "resourceVersion": "version1234", + } + }); + + let drain_until = DateTime::parse_from_rfc3339("2023-02-08T15:30:00Z") + .unwrap() + .with_timezone(&Utc); + let loadbalancing = LoadBalancingConfig::new(Uuid::nil()); + let patch = make_patch_pod_isolate(&pod, drain_until, None, &loadbalancing).unwrap(); + + assert_eq!( + &patch[..2], + &[ + PatchOperation::Test(TestOperation { + path: "/metadata/uid".try_into().unwrap(), + value: json!("uid1234"), + }), + PatchOperation::Test(TestOperation { + path: "/metadata/resourceVersion".try_into().unwrap(), + value: json!("version1234"), + }) + ], + ); + } + + #[test] + fn eviction_patch_none_delete_options() { + let eviction: Eviction = from_json!({}); + + let patch = make_patch_eviction_to_dry_run(&eviction).unwrap(); + + let applied = apply(&eviction, &patch).unwrap(); + assert_eq!( + applied, + json!({ + "apiVersion": "policy/v1", + "kind": "Eviction", + "metadata": {}, + "deleteOptions": { + "dryRun": ["All"], + }, + }) + ); + } + + #[test] + fn eviction_patch_none_dry_run() { + let eviction: Eviction = from_json!({ + "deleteOptions": {}, + }); + + let patch = make_patch_eviction_to_dry_run(&eviction).unwrap(); + + let applied = apply(&eviction, &patch).unwrap(); + assert_eq!( + applied, + json!({ + "apiVersion": "policy/v1", + "kind": "Eviction", + "metadata": {}, + "deleteOptions": { + "dryRun": ["All"], + }, + }) + ); + } + + #[test] + fn eviction_patch_empty_dry_run() { + let eviction: Eviction = from_json!({ + "deleteOptions": { + "dryRun": [], + }, + }); + + let patch = make_patch_eviction_to_dry_run(&eviction).unwrap(); + + let applied = apply(&eviction, &patch).unwrap(); + assert_eq!( + applied, + json!({ + "apiVersion": "policy/v1", + "kind": "Eviction", + "metadata": {}, + "deleteOptions": { + "dryRun": ["All"], + }, + }) + ); + } +} diff --git a/src/webhooks/reactive_rustls_config.rs b/src/webhooks/reactive_rustls_config.rs new file mode 100644 index 0000000..3618a35 --- /dev/null +++ b/src/webhooks/reactive_rustls_config.rs @@ -0,0 +1,129 @@ +use axum_server::tls_rustls::RustlsConfig; +use std::io::Cursor; +use std::path::Path; +use std::time::Duration; + +use debounced::debounced; +use eyre::{Context, ContextCompat, Result}; +use futures::StreamExt; +use genawaiter::sync::Gen; +use notify::{RecursiveMode, Watcher}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use tokio::fs::File; +use tokio::io::copy; +use tokio::sync::mpsc; +use tracing::error; + +use crate::shutdown::Shutdown; +use crate::spawn_service::spawn_service; +use crate::webhooks::config::CertConfig; + +const TLS_CRT: &str = "tls.crt"; +const TLS_KEY: &str = "tls.key"; + +pub async fn build_reactive_rustls_config( + config: &CertConfig, + shutdown: &Shutdown, +) -> Result { + match config { + CertConfig::CertDir(cert_dir) => { + let config = build(cert_dir, shutdown).await?; + Ok(config) + } + CertConfig::Override(cert, key) => { + let serialized = SerializedCertifiedKey::new_with(&[cert.clone()], key); + let config = RustlsConfig::from_der(serialized.certs, serialized.key).await?; + Ok(config) + } + } +} + +async fn build(cert_dir: &Path, shutdown: &Shutdown) -> Result { + let config = { + let cert = load_cert_from(cert_dir).await?; + RustlsConfig::from_der(cert.certs, cert.key).await? + }; + + let (watcher_tx, mut watcher_rx) = mpsc::channel(1); + let mut watcher_stream = { + let mut watcher = notify::recommended_watcher(move |_| { + let _ = watcher_tx.try_send(()); + })?; + watcher.watch(&cert_dir.join(TLS_CRT), RecursiveMode::NonRecursive)?; + watcher.watch(&cert_dir.join(TLS_KEY), RecursiveMode::NonRecursive)?; + + let stream = Gen::new(move |mut co| async move { + let _watcher = watcher; // move watcher into generator + while let Some(event) = watcher_rx.recv().await { + co.yield_(event).await; + } + }); + + let debounced = debounced(stream, Duration::from_secs(1)); + debounced.take_until(shutdown.wait_shutdown_triggered()) + }; + + spawn_service(shutdown, "certwatcher", { + let config = config.clone(); + let cert_dir = cert_dir.to_path_buf(); + async move { + while watcher_stream.next().await.is_some() { + let cert = match load_cert_from(&cert_dir).await { + Ok(cert) => cert, + Err(err) => { + error!(?err, "Reloading cert fail"); + continue; + } + }; + + if let Err(err) = config.reload_from_der(cert.certs, cert.key).await { + error!(?err, "Reloading cert fail"); + continue; + }; + } + } + })?; + + Ok(config) +} + +struct SerializedCertifiedKey { + certs: Vec>, + key: Vec, +} + +impl SerializedCertifiedKey { + fn new_with(cert_der: &[CertificateDer], key_der: &PrivateKeyDer) -> Self { + let certs = cert_der + .iter() + .map(|cert| Vec::from(cert.as_ref())) + .collect(); + let key = Vec::from(key_der.secret_der()); + + SerializedCertifiedKey { certs, key } + } +} + +async fn load_cert_from(cert_dir: &Path) -> Result { + let certs = { + let path = cert_dir.join(TLS_CRT); + let mut file = File::open(&path).await.context(format!("File({path:?})"))?; + let mut crt = Vec::new(); + copy(&mut file, &mut crt).await?; + rustls_pemfile::certs(&mut Cursor::new(crt)) + .collect::>>() + .context(format!("Cert({path:?})"))? + }; + + let key = { + let path = cert_dir.join(TLS_KEY); + let mut file = File::open(&path).await.context(format!("File({path:?}"))?; + let mut key = Vec::new(); + copy(&mut file, &mut key).await?; + rustls_pemfile::private_key(&mut Cursor::new(key)) + .context(format!("Key({path:?})"))? + .context("empty key")? + }; + + Ok(SerializedCertifiedKey::new_with(&certs, &key)) +} diff --git a/src/webhooks/report.rs b/src/webhooks/report.rs new file mode 100644 index 0000000..3e153d9 --- /dev/null +++ b/src/webhooks/report.rs @@ -0,0 +1,88 @@ +use k8s_openapi::api::core::v1::{ObjectReference, Pod}; +use kube::runtime::events::{Event, EventType, Recorder}; +use kube::Resource; +use tracing::{debug, enabled, info, Level}; + +use crate::webhooks::AppState; + +pub async fn report( + state: &AppState, + reference: ObjectReference, + type_: EventType, + action: &str, + reason: &str, + note: String, +) { + let recorder = Recorder::new( + state.api_resolver.client.clone(), + state.event_reporter.clone(), + reference, + ); + + // max limit of the note is 1KB + let note = if note.len() > 1024 { + let mut boundary = 1024 - "...".len(); + loop { + if note.is_char_boundary(boundary) { + break format!("{}...", ¬e[..boundary]); + } + + boundary -= 1; + } + } else { + note + }; + + // ignore the error of diagnostic events + let _ = recorder + .publish(Event { + type_, + action: action.to_string(), + reason: reason.to_string(), + note: Some(note), + secondary: None, + }) + .await; +} + +pub async fn debug_report_for_ref( + state: &AppState, + object_ref: ObjectReference, + action: &str, + reason: &str, + note: String, +) { + if !enabled!(Level::DEBUG) { + return; + } + + debug!(action, reason, note); + report(state, object_ref, EventType::Normal, action, reason, note).await; +} + +pub async fn debug_report_for( + state: &AppState, + pod: &Pod, + action: &str, + reason: &str, + note: String, +) { + debug_report_for_ref(state, pod.object_ref(&()), action, reason, note).await; +} + +pub async fn report_for(state: &AppState, pod: &Pod, action: &str, reason: &str, note: String) { + if !enabled!(Level::INFO) { + return; + } + + info!(action, reason, note); + report( + state, + pod.object_ref(&()), + EventType::Normal, + action, + reason, + note, + ) + .await; +} diff --git a/src/webhooks/try_bind.rs b/src/webhooks/try_bind.rs new file mode 100644 index 0000000..09140b0 --- /dev/null +++ b/src/webhooks/try_bind.rs @@ -0,0 +1,35 @@ +use std::net::SocketAddr; + +use eyre::Result; +use rand::Rng; +use tokio::net::TcpListener; + +use crate::webhooks::config::BindConfig; + +pub async fn try_bind(bind_config: &BindConfig) -> Result { + match bind_config { + BindConfig::SocketAddr(bind_addr) => { + let incoming = TcpListener::bind(bind_addr).await?; + Ok(incoming) + } + BindConfig::RandomForTest => { + let mut retry = 0; + loop { + let random_port = rand::thread_rng().gen_range(49152..=65535); + let bind_addr = SocketAddr::from(([0, 0, 0, 0], random_port)); + + match TcpListener::bind(bind_addr).await { + Ok(incoming) => return Ok(incoming), + Err(err) => { + retry += 1; + if retry < 10 { + continue; + } + + return Err(err.into()); + } + } + } + } + } +} diff --git a/tests/controller.rs b/tests/controller.rs new file mode 100644 index 0000000..6351051 --- /dev/null +++ b/tests/controller.rs @@ -0,0 +1,191 @@ +use std::ops::Add; + +use chrono::TimeDelta; +use k8s_openapi::api::core::v1::Pod; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::DeleteOptions; +use tokio::time::Duration; + +use pod_graceful_drain::webhooks::patch_pod_isolate; +use pod_graceful_drain::ServiceRegistry; + +use crate::testutils::context::{within_test_namespace, TestContext}; +use crate::testutils::operations::install_test_host_service; + +mod testutils; + +async fn setup(context: &TestContext) { + install_test_host_service(context).await; + let service_registry = ServiceRegistry::default(); + + pod_graceful_drain::start_controller( + &context.api_resolver, + &service_registry, + &context.loadbalancing, + &context.shutdown, + ) + .unwrap(); +} + +#[tokio::test] +async fn controller_shouldnt_delete_too_early() { + within_test_namespace(|context| async move { + setup(&context).await; + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + patch_drain_until(&context, "some-pod", TimeDelta::seconds(10), None).await; + + tokio::time::sleep(Duration::from_secs(5)).await; + assert!( + !pod_has_been_deleted(&context, "some-pod").await, + "pod shouldn't be deleted yet" + ); + }) + .await; +} + +#[tokio::test] +async fn controller_shouldnt_evict_too_early() { + within_test_namespace(|context| async move { + setup(&context).await; + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + patch_drain_until( + &context, + "some-pod", + TimeDelta::seconds(10), + Some(&DeleteOptions::default()), + ) + .await; + + tokio::time::sleep(Duration::from_secs(5)).await; + assert!( + !pod_has_been_deleted(&context, "some-pod").await, + "pod shouldn't be deleted yet" + ); + }) + .await; +} + +#[tokio::test] +async fn controller_should_delete_expired_pod() { + within_test_namespace(|context| async move { + setup(&context).await; + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + patch_drain_until(&context, "some-pod", TimeDelta::seconds(5), None).await; + + tokio::time::sleep(Duration::from_secs(10)).await; + assert!( + pod_has_been_deleted(&context, "some-pod").await, + "pod should've been deleted" + ); + }) + .await; +} + +#[tokio::test] +async fn controller_should_evict_expired_pod() { + within_test_namespace(|context| async move { + setup(&context).await; + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + patch_drain_until( + &context, + "some-pod", + TimeDelta::seconds(5), + Some(&DeleteOptions::default()), + ) + .await; + + tokio::time::sleep(Duration::from_secs(10)).await; + assert!( + pod_has_been_deleted(&context, "some-pod").await, + "pod should've been deleted" + ); + }) + .await; +} + +async fn patch_drain_until( + context: &TestContext, + name: &str, + delta: TimeDelta, + delete_options: Option<&DeleteOptions>, +) { + let now = chrono::Utc::now(); + let drain_until = now.add(delta); + let pod: Pod = context.api_resolver.all().get(name).await.unwrap(); + patch_pod_isolate( + &context.api_resolver, + &pod, + drain_until, + delete_options, + &context.loadbalancing, + ) + .await + .unwrap(); +} + +async fn pod_has_been_deleted(context: &TestContext, name: &str) -> bool { + let result = context.api_resolver.all::().get(name).await; + match result { + Err(_) => true, + Ok(pod) => pod.metadata.deletion_timestamp.is_some(), + } +} diff --git a/tests/reflectors.rs b/tests/reflectors.rs new file mode 100644 index 0000000..5c94e40 --- /dev/null +++ b/tests/reflectors.rs @@ -0,0 +1,412 @@ +mod testutils; + +use std::collections::BTreeMap; +use std::time::Duration; + +use k8s_openapi::api::core::v1::Service; +use k8s_openapi::api::networking::v1::Ingress; +use kube::runtime::reflector::ObjectRef; +use kube::ResourceExt; + +use crate::testutils::context::{within_test_namespace, TestContext}; + +use pod_graceful_drain::{start_reflectors, try_some, Config, Stores}; + +fn start_test_reflector(context: &TestContext) -> Stores { + let config = Config { + delete_after: Duration::from_secs(10), + experimental_general_ingress: true, + }; + + start_reflectors( + &context.api_resolver, + &config, + &Default::default(), + &context.shutdown, + ) + .unwrap() +} + +#[tokio::test] +async fn should_reflect_pod() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + kubectl!( + &context, + [ + "run", + "some-pod", + "--image=public.ecr.aws/docker/library/busybox", + "--labels=some-label=some-value", + "--", + "sleep", + "9999" + ] + ); + + let pod = eventually_some!( + stores.get_pod(&ObjectRef::new("some-pod").within(&context.namespace)) + ); + + assert_eq!( + pod.metadata.labels.as_ref(), + Some(&{ + let mut selector = BTreeMap::new(); + selector.insert(String::from("some-label"), String::from("some-value")); + selector + }), + "should reflect labels" + ); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_pod_delete() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + kubectl!( + &context, + [ + "run", + "some-pod", + "--image=public.ecr.aws/docker/library/busybox", + "--", + "sleep", + "9999" + ] + ); + let pod = eventually_some!( + stores.get_pod(&ObjectRef::new("some-pod").within(&context.namespace)) + ); + + kubectl!(&context, ["delete", "pod", "some-pod"]); + + assert!(eventually!(stores + .get_pod(&ObjectRef::from_obj(&pod)) + .is_none())); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_pod_label_edit() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + kubectl!( + &context, + [ + "run", + "some-pod", + "--image=public.ecr.aws/docker/library/busybox", + "--", + "sleep", + "9999" + ] + ); + + let pod = eventually_some!( + stores.get_pod(&ObjectRef::new("some-pod").within(&context.namespace)) + ); + + kubectl!( + &context, + ["label", "pod", "some-pod", "some-label=edited-value"] + ); + + assert_eq!( + eventually_some!(try_some!((stores + .get_pod(&ObjectRef::from_obj(&pod))? + .labels() + .get("some-label")?) + .clone())), + String::from("edited-value") + ); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_service() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Service, + "\ +metadata: + name: some-service +spec: + ports: + - port: 80 + protocol: TCP + selector: + some-label: some-value" + ); + + let service = eventually_some!( + stores.get_service(&ObjectRef::new("some-service").within(&context.namespace)) + ); + + assert_eq!( + try_some!(service.spec?.selector?), + Some(&{ + let mut selector = BTreeMap::new(); + selector.insert(String::from("some-label"), String::from("some-value")); + selector + }), + "should reflect selectors" + ); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_service_spec_edit() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Service, + "\ +metadata: + name: some-service +spec: + ports: + - port: 80 + protocol: TCP + selector: + some-label: some-value" + ); + + let service = eventually_some!( + stores.get_service(&ObjectRef::new("some-service").within(&context.namespace)) + ); + + apply_yaml!( + &context, + Service, + "\ +metadata: + name: some-service +spec: + ports: + - port: 80 + protocol: TCP + selector: + some-label: edited-value" + ); + + assert_eq!( + eventually_some!({ + let label = try_some!(stores + .get_service(&ObjectRef::from_obj(&service))? + .spec? + .selector? + .get("some-label") + .cloned()) + .flatten(); + match label { + Some(str) if str == "some-value" => None, + _ => label, + } + }), + String::from("edited-value") + ); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_service_delete() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Service, + "\ +metadata: + name: some-service +spec: + ports: + - port: 80 + protocol: TCP + selector: + some-label: some-value" + ); + + let service = eventually_some!( + stores.get_service(&ObjectRef::new("some-service").within(&context.namespace)) + ); + kubectl!(&context, ["delete", "service", "some-service"]); + + assert!(eventually!(stores + .get_service(&ObjectRef::from_obj(&service)) + .is_none())); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_ingress() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Ingress, + "\ +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: test + port: + number: 80" + ); + + let ingress = eventually_some!({ + stores + .ingresses() + .into_iter() + .find(|ingress| ingress.name_any() == "some-ingress") + }); + + assert_eq!( + try_some!(ingress.spec?.rules?[0].http?.paths[0] + .backend + .service? + .name + .clone()), + Some(String::from("test")) + ); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_ingress_delete() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Ingress, + "\ +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: test + port: + number: 80" + ); + + eventually_some!({ + stores + .ingresses() + .into_iter() + .find(|ingress| ingress.name_any() == "some-ingress") + }); + + kubectl!(&context, ["delete", "ingress", "some-ingress"]); + + assert!(eventually!(stores.ingresses().is_empty())); + }) + .await; +} + +#[tokio::test] +async fn should_reflect_ingress_spec_edit() { + within_test_namespace(|context| async move { + let stores = start_test_reflector(&context); + + apply_yaml!( + &context, + Ingress, + "\ +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: test + port: + number: 80" + ); + + let ingress = eventually_some!(stores + .ingresses() + .into_iter() + .find(|ingress| ingress.name_any() == "some-ingress")); + + apply_yaml!( + &context, + Ingress, + "\ +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: another-service + port: + number: 80" + ); + + let old_label = try_some!(ingress.spec?.rules?[0].http?.paths[0] + .backend + .service? + .name + .clone()); + assert_eq!( + eventually_some!({ + let new_ingress = try_some!(stores + .ingresses() + .into_iter() + .find(|ing| ing.name_any() == ingress.name_any())) + .unwrap(); + let new_label = try_some!(new_ingress?.spec?.rules?[0].http?.paths[0] + .backend + .service? + .name + .clone()); + if new_label == old_label { + None + } else { + new_label + } + }), + String::from("another-service") + ); + }) + .await; +} diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..41fe627 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,119 @@ +import subprocess +import sys +from datetime import datetime, timedelta +import random +import time + +namespace = "" + + +def setup_module(): + global namespace + namespace = f"pgd-test-{random.randrange(10000, 99999)}" + kubectl("create", "namespace", namespace) + kubectl("label", "namespace", namespace, "test=true") + print("testing on namespace: ", namespace) + + +def teardown_module(): + global namespace + kubectl("delete", "namespace", namespace) + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def kubectl(*args): + global namespace + + result = subprocess.run( + ["kubectl", "--namespace", namespace, *args], + capture_output=True) + + if result.returncode != 0: + eprint("stdout:") + eprint(result.stdout) + eprint("stderr:") + eprint(result.stderr) + raise Exception(f"'kubectl {" ".join(args)}' failed with exit code '{result.returncode}'") + + +def kubectl_stdin(args, /, input): + global namespace + + result = subprocess.run( + ["kubectl", "--namespace", namespace, *args], + capture_output=True, + input=input, encoding="utf-8") + + if result.returncode != 0: + eprint("stdout:") + eprint(result.stdout) + eprint("stderr:") + eprint(result.stderr) + raise Exception(f"'kubectl {" ".join(args)}' failed with exit code '{result.returncode}'") + + +def kubectl_nowait(args): + global namespace + + child = subprocess.Popen( + ["kubectl", "--namespace", namespace, *args]) + + return child + + +def pod_is_alive(name): + global namespace + + result = subprocess.run( + ["kubectl", "--namespace", namespace, "get", name, "-o", "jsonpath={.metadata.deletionTimestamp}"], + capture_output=True, encoding="utf-8") + + if result.returncode != 0: + return False + + stdout = result.stdout.strip() + return not stdout + + +def test_can_delete_pod_without_delay_if_no_ingress(): + kubectl("run", "busybox-sleep", "--image=public.ecr.aws/docker/library/busybox", "--", "sleep", "1000") + kubectl("wait", "pod/busybox-sleep", "--for=condition=Ready") + start = datetime.now() + kubectl("delete", "pod/busybox-sleep", "--wait=false") + diff = datetime.now() - start + assert diff < timedelta(seconds=10), "it should be quick" + assert not pod_is_alive("pod/busybox-sleep") + + +def test_delete_is_delayed_with_ingress(): + kubectl("run", "nginx", "--image=nginx") + kubectl("expose", "pod", "nginx", "--port=80", "--target-port=8000") + kubectl_stdin(["apply", "-f", "-"], input=""" +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: nginx + port: + number: 80 + """) + kubectl("wait", "pod/nginx", "--for=condition=Ready") + + time.sleep(1) # give some time to settle down + + kubectl_nowait(["delete", "pod/nginx", "--wait=false"]) + + for _ in range(0, 20 - 5): + assert pod_is_alive("pod/nginx"), "pod should be alive for approx. 20s" + time.sleep(1) diff --git a/tests/testutils/context.rs b/tests/testutils/context.rs new file mode 100644 index 0000000..d6030ad --- /dev/null +++ b/tests/testutils/context.rs @@ -0,0 +1,270 @@ +use std::collections::{BTreeMap, HashSet}; +use std::future::Future; +use std::io::Write; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use eyre::{Context, Result}; +use k8s_openapi::api::core::v1::Namespace; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::api::{ApiResource, DeleteParams, DynamicObject}; +use kube::config::{KubeConfigOptions, Kubeconfig}; +use kube::{Api, Client, Config}; +use rand::Rng; +use tempfile::NamedTempFile; +use tokio::task::JoinError; +use tracing::dispatcher::DefaultGuard; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::EnvFilter; +use uuid::Uuid; + +use pod_graceful_drain::{ApiResolver, LoadBalancingConfig, Shutdown}; + +use crate::testutils::run_command::{get_command_output, run_command, CommandParams}; + +const DEFAULT_KIND_IMAGE: &str = "kindest/node:v1.30.2"; +const DEFAULT_TEST_CLUSTER_NAME: &str = "test-pgd"; +const TEST_NAMESPACE_PREFIX: &str = "test"; +const TEST_NAMESPACE_LABEL_KEY: &str = "test-pgd-ns"; +const TEST_NAMESPACE_NAME_LABEL: &str = "name"; + +#[derive(Clone)] +pub struct TestContext { + pub(crate) kubeconfig: Arc, + pub api_resolver: ApiResolver, + pub cluster_name: String, + pub namespace: String, + pub loadbalancing: LoadBalancingConfig, + pub shutdown: Shutdown, + teardown: Arc>>, + pub(crate) cluster_resources: Arc>>, +} + +type Teardown = Box Pin>> + Send + 'static>; +impl TestContext { + pub fn register_teardown(&self, func: impl FnOnce(TestContext) -> Fut + Send + 'static) + where + Fut: Future, + { + self.teardown.lock().unwrap().push(Box::new( + move |context| -> Pin>> { + Box::pin(async move { + func(context).await; + }) + }, + )) + } +} + +pub async fn within_test_namespace(f: F) -> Fut::Output +where + F: for<'a> FnOnce(TestContext) -> Fut + Send + 'static, + Fut: Future + Send, + Fut::Output: Send + 'static, +{ + let _logger = set_default_test_logger(); + + let kind_cluster = + std::env::var("KIND_CLUSTER").unwrap_or(DEFAULT_TEST_CLUSTER_NAME.to_owned()); + let result = within_random_namespace_with_cluster(&kind_cluster, f).await; + match result { + Ok(result) => result, + Err(err) => std::panic::resume_unwind(err.into_panic()), + } +} + +pub async fn within_test_cluster(f: F) -> Fut::Output +where + F: for<'a> FnOnce(TestContext) -> Fut + Send + 'static, + Fut: Future + Send, + Fut::Output: Send + 'static, +{ + let _logger = set_default_test_logger(); + + let random_cluster_name = format!( + "{DEFAULT_TEST_CLUSTER_NAME}-{}", + rand::thread_rng().gen_range(0..100000) + ); + let kind_image = std::env::var("KIND_IMAGE").unwrap_or(DEFAULT_KIND_IMAGE.to_owned()); + let dummy_kubeconfig = NamedTempFile::new().unwrap(); + let mut cluster_config = NamedTempFile::new().unwrap(); + write!( + &mut cluster_config, + r#" +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker +- role: worker +"# + ) + .unwrap(); + + run_command(&CommandParams { + command: "kind", + config_args: &[], + args: &[ + "create", + "cluster", + "--image", + &kind_image, + "--name", + &random_cluster_name, + "--config", + cluster_config.path().to_str().unwrap(), + // not to messing with global kubeconfig. + "--kubeconfig", + dummy_kubeconfig.path().to_str().unwrap(), + ], + stdin: None, + }) + .await + .unwrap(); + + let result = within_random_namespace_with_cluster(&random_cluster_name, f).await; + + run_command(&CommandParams { + command: "kind", + config_args: &[], + args: &["delete", "cluster", "--name", &random_cluster_name], + stdin: None, + }) + .await + .unwrap(); + + match result { + Ok(result) => result, + Err(err) => std::panic::resume_unwind(err.into_panic()), + } +} + +async fn within_random_namespace_with_cluster( + cluster_name: &str, + f: F, +) -> Result +where + F: for<'a> FnOnce(TestContext) -> Fut + Send + 'static, + Fut: Future + Send, + Fut::Output: Send + 'static, +{ + let context = match new_test_context(cluster_name, Uuid::nil()).await { + Ok(context) => context, + Err(err) => { + eprintln!("{err:?}"); + panic!( + "Tests require kind cluster named '{cluster_name}'. \ + Run `kind create cluster --image={DEFAULT_KIND_IMAGE} --name '{cluster_name}'` first." + ); + } + }; + + let shutdown = context.shutdown.clone(); + + let result = tokio::spawn({ + let context = context.clone(); + async move { f(context).await } + }) + .await; + + shutdown.trigger_shutdown(); + shutdown.wait_shutdown_complete().await; + + let teardowns: Vec<_> = context.teardown.lock().unwrap().drain(..).collect(); + for teardown in teardowns.into_iter().rev() { + let context = context.clone(); + teardown(context).await; + } + + result +} + +async fn new_test_context(cluster_name: &str, instance_id: Uuid) -> Result { + let file = get_temp_kubeconfig_file_from_kind(cluster_name).await?; + let kubeconfig = Kubeconfig::read_from(file.path()).context("valid kubeconfig yaml")?; + let config = Config::from_custom_kubeconfig(kubeconfig, &KubeConfigOptions::default()).await?; + let shutdown = Shutdown::new(); + let namespace = create_random_namespace(&config).await?; + let context = TestContext { + kubeconfig: Arc::new(file), + api_resolver: ApiResolver::try_new_within(config, &namespace)?, + cluster_name: cluster_name.to_string(), + namespace: namespace.clone(), + loadbalancing: LoadBalancingConfig::new(instance_id), + shutdown, + teardown: Arc::new(Mutex::new(Vec::new())), + cluster_resources: Arc::new(Mutex::new(HashSet::new())), + }; + context.register_teardown(|context| async move { + let client = &context.api_resolver.client; + let namespace = &context.namespace; + let _ = delete_namespace(client, namespace).await; + }); + context.register_teardown({ + |context| async move { + let client = &context.api_resolver.client; + let cluster_resources = context.cluster_resources.lock().unwrap().clone(); + for (dyntype, name) in cluster_resources.iter() { + let _ = Api::::all_with(client.clone(), dyntype) + .delete(name, &DeleteParams::default()) + .await; + } + } + }); + + Ok(context) +} + +async fn get_temp_kubeconfig_file_from_kind(context: &str) -> Result { + let mut file = NamedTempFile::new()?; + + let params = CommandParams { + command: "kind", + config_args: &[], + args: &["get", "kubeconfig", "--name", context], + stdin: None, + }; + let output = get_command_output(¶ms).await?; + file.as_file_mut().write_all(&output)?; + + Ok(file) +} + +async fn create_random_namespace(config: &Config) -> Result { + let client = Client::try_from(config.clone())?; + let api: Api = Api::all(client); + let random_id = rand::thread_rng().gen_range(1..1000000); + let random_name = format!("{TEST_NAMESPACE_PREFIX}-{random_id}"); + let namespace = Namespace { + metadata: ObjectMeta { + name: Some(random_name.clone()), + labels: Some({ + let mut labels = BTreeMap::new(); + labels.insert(String::from(TEST_NAMESPACE_LABEL_KEY), String::new()); + labels.insert(String::from(TEST_NAMESPACE_NAME_LABEL), random_name.clone()); + labels + }), + ..ObjectMeta::default() + }, + ..Namespace::default() + }; + + api.create(&Default::default(), &namespace).await?; + Ok(random_name) +} + +async fn delete_namespace(client: &Client, ns: &str) -> Result<()> { + let api: Api = Api::all(client.clone()); + let _ns = api.delete(ns, &DeleteParams::default()).await?; + Ok(()) +} + +fn set_default_test_logger() -> DefaultGuard { + tracing::subscriber::set_default({ + let filter = EnvFilter::new("pod_graceful_drain=trace,test=trace"); + + tracing_subscriber::registry() + .with(filter) + .with(tracing_subscriber::fmt::layer().with_test_writer()) + }) +} diff --git a/tests/testutils/event_tracker.rs b/tests/testutils/event_tracker.rs new file mode 100644 index 0000000..df4a1ab --- /dev/null +++ b/tests/testutils/event_tracker.rs @@ -0,0 +1,44 @@ +use crate::testutils::context::TestContext; +use futures::stream::BoxStream; +use futures::StreamExt; +use k8s_openapi::api::events::v1::Event; +use kube::api::{WatchEvent, WatchParams}; +use kube::Api; +use std::time::{Duration, Instant}; + +pub struct EventTracker { + stream: BoxStream<'static, kube::Result>>, + timeout: Duration, +} + +impl EventTracker { + pub async fn new(context: &TestContext, timeout: Duration) -> Self { + let api: Api = + Api::namespaced(context.api_resolver.client.clone(), &context.namespace); + let params = WatchParams { + field_selector: Some("reportingController=pod-graceful-drain".to_string()), + ..WatchParams::default() + }; + let stream = api.watch(¶ms, "0").await.unwrap().boxed(); + Self { stream, timeout } + } + + pub async fn issued_soon(&mut self, action: &str, reason: &str) -> bool { + let start = Instant::now(); + while Instant::now() - start < self.timeout { + let Some(Ok(watch_event)) = self.stream.next().await else { + break; + }; + + let WatchEvent::Added(event) = watch_event else { + continue; + }; + + if event.action.as_deref() == Some(action) && event.reason.as_deref() == Some(reason) { + return true; + } + } + + false + } +} diff --git a/tests/testutils/macros.rs b/tests/testutils/macros.rs new file mode 100644 index 0000000..f2139c3 --- /dev/null +++ b/tests/testutils/macros.rs @@ -0,0 +1,55 @@ +#[macro_export] +macro_rules! eventually { + ($cond:expr $(,)?) => {{ + eventually!(timeout = 10, $cond) + }}; + + (timeout = $time:literal, $cond:expr $(,)?) => {{ + use ::std::time::Duration; + use ::tokio::time::{sleep, timeout}; + + let result = timeout(Duration::from_secs($time), async { + loop { + if async { $cond }.await { + return; + } + + sleep(Duration::from_millis(100)).await; + } + }) + .await; + + match result { + Ok(()) => true, + Err(_) => false, + } + }}; +} + +#[macro_export] +macro_rules! eventually_some { + ($cond:expr $(,)?) => {{ + eventually_some!(timeout = 10, $cond) + }}; + + (timeout = $time:expr, $cond:expr $(,)?) => {{ + use ::std::time::Duration; + use ::tokio::time::{sleep, timeout}; + + let result = timeout(Duration::from_secs($time), async { + loop { + if let Some(res) = async { $cond }.await { + return res; + } + + sleep(Duration::from_millis(100)).await; + } + }) + .await; + + match result { + Ok(res) => res, + Err(_) => panic!("Timeout error: {}", stringify!($cond)), + } + }}; +} diff --git a/tests/testutils/mod.rs b/tests/testutils/mod.rs new file mode 100644 index 0000000..15e6481 --- /dev/null +++ b/tests/testutils/mod.rs @@ -0,0 +1,7 @@ +#![allow(dead_code)] + +pub mod context; +pub mod event_tracker; +pub mod macros; +pub mod operations; +mod run_command; diff --git a/tests/testutils/operations.rs b/tests/testutils/operations.rs new file mode 100644 index 0000000..5346ed9 --- /dev/null +++ b/tests/testutils/operations.rs @@ -0,0 +1,137 @@ +use std::any::TypeId; +use std::fmt::Debug; + +use eyre::Result; +use k8s_openapi::api::core::v1::Service; +use k8s_openapi::serde::de::DeserializeOwned; +use k8s_openapi::serde::Serialize; +use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope}; +use kube::api::{ApiResource, DynamicObject, Patch, PatchParams, PostParams}; +use kube::{Api, Resource, ResourceExt}; + +use super::context::TestContext; +use super::run_command::run_command; +use crate::testutils::run_command::CommandParams; + +pub async fn kubectl<'a>(context: &'a TestContext, args: &[&str], stdin: Option<&[u8]>) { + let kubectl = std::env::var("KUBECTL").unwrap_or("kubectl".to_owned()); + run_command(&CommandParams { + command: &kubectl, + config_args: &[ + "--kubeconfig", + context.kubeconfig.path().to_str().unwrap(), + "--namespace", + &context.namespace, + ], + args, + stdin: stdin.map(|x| x.to_vec()), + }) + .await + .unwrap() +} + +#[macro_export] +macro_rules! kubectl { + ($ctx:expr, [$($arg:tt)*]) => { + $crate::testutils::operations::kubectl($ctx, &[$($arg)*], None).await + }; + + ($ctx:expr, [$($arg:tt)*] <<< $yaml:expr $(,)*) => {{ + let yaml = format!($yaml); + $crate::testutils::operations::kubectl($ctx, &[$($arg)*], Some(yaml.as_bytes())).await + }}; + + ($ctx:expr, [$($arg:tt)*] <<< $yaml:expr, $($tt:tt)*) => {{ + let yaml = format!($yaml, $($tt)*); + $crate::testutils::operations::kubectl($ctx, &[$($arg)*], Some(yaml.as_bytes())).await + }}; +} + +pub async fn apply(context: &TestContext, res: &K) -> Result +where + K: Resource + Serialize + Debug + Clone + DeserializeOwned, + K::DynamicType: Default, + K::Scope: 'static, +{ + let dyntype = ApiResource::erase::(&Default::default()); + let type_id = TypeId::of::(); + let api: Api = { + if type_id == TypeId::of::() { + Api::namespaced_with( + context.api_resolver.client.clone(), + &context.namespace, + &dyntype, + ) + } else if type_id == TypeId::of::() { + Api::all_with(context.api_resolver.client.clone(), &dyntype) + } else { + unimplemented!(); + } + }; + + let name = res.name_any(); + let created = if api.get(&name).await.is_ok() { + api.patch( + &res.name_any(), + &PatchParams::default(), + &Patch::Strategic(res.clone()), + ) + .await? + } else { + let json = serde_json::to_string(&res)?; + let object = serde_json::from_str(&json)?; + api.create(&PostParams::default(), &object).await? + }; + + if type_id == TypeId::of::() { + context + .cluster_resources + .lock() + .unwrap() + .insert((dyntype, name)); + } + + Ok(created.try_parse()?) +} + +#[macro_export] +macro_rules! apply_yaml { + ($ctx:expr, $kind:ty, $yaml:expr $(,)?) => { + { + let yaml = format!($yaml); + let res = ::serde_yaml::from_str::<$kind>(&yaml).unwrap(); + $crate::testutils::operations::apply($ctx, &res).await.unwrap() + } + }; + ($ctx:expr, $kind:ty, $yaml:expr, $($arg:tt)+) => { + { + let yaml = format!($yaml, $($arg)+); + let res = ::serde_yaml::from_str::<$kind>(&yaml).unwrap(); + $crate::testutils::operations::apply($ctx, &res).await.unwrap() + } + }; +} + +/// If you run a kind cluster in the docker, you can connect to the host with `host.docker.internal` +/// If it were in the podman, then it would've been `host.containers.internal` +/// Moreover, rootless podman requires `--network=slirp4netns:allow_host_loopback=true` argument, +/// but the kind podman provider doesn't pass the argument to create a cluster. +/// Even if it does, windows podman doesn't handle that argument well for now. +/// So, this is a reliable solution. +pub async fn install_test_host_service(context: &TestContext) -> String { + let local_ip = local_ip_address::local_ip().unwrap(); + // Headless service + apply_yaml!( + context, + Service, + r#" +metadata: + name: test-host +spec: + type: ExternalName + externalName: {local_ip} +"# + ); + + format!("test-host.{}.svc", context.namespace) +} diff --git a/tests/testutils/run_command.rs b/tests/testutils/run_command.rs new file mode 100644 index 0000000..718af50 --- /dev/null +++ b/tests/testutils/run_command.rs @@ -0,0 +1,167 @@ +use std::fmt::{Display, Formatter, Write as _}; +use std::io::{stderr, stdout, BufRead, BufReader, Write}; +use std::process::{Command, Output, Stdio}; + +use eyre::{eyre, Context, Result}; +use tokio::task::spawn_blocking; +use tracing::{enabled, error, info, span, trace, Level}; + +#[derive(Clone)] +pub struct CommandParams<'a> { + pub command: &'a str, + pub config_args: &'a [&'a str], + pub args: &'a [&'a str], + pub stdin: Option>, +} + +impl Display for CommandParams<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + append(self.command, f)?; + for arg in self.args { + f.write_char(' ')?; + append(arg, f)?; + } + if let Some(stdin) = self.stdin.as_ref() { + if let Ok(str) = std::str::from_utf8(stdin) { + f.write_str(" <) -> std::fmt::Result { + if str.contains([' ', '\'']) { + f.write_char('\'')?; + for c in str.chars() { + if c == '\'' { + f.write_str("'\''")?; + } + } + f.write_char('\'')?; + } else { + f.write_str(str)?; + } + + Ok(()) + } + } +} + +pub async fn run_command(params: &CommandParams<'_>) -> Result<()> { + assert!( + enabled!(target: "test", Level::INFO), + "logger should be set" + ); + + let mut command = Command::new(params.command); + command.args(params.config_args); + command.args(params.args); + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + if params.stdin.is_some() { + command.stdin(Stdio::piped()); + } + + let mut child = spawn_blocking(move || command.spawn()).await??; + + let span = span!(target: "test", Level::ERROR, "command", pid=child.id()); + + info!(target: "test", parent: &span, "{}", params); + + if let Some(stdin) = params.stdin.as_ref() { + let mut pipe = child.stdin.as_ref().expect("piped"); + pipe.write_all(stdin)?; + } + + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + spawn_blocking({ + let stdout = child.stdout.take().expect("piped"); + let tx = tx.clone(); + move || { + for line in BufReader::new(stdout).lines() { + if tx.send(("stdout", line.unwrap())).is_err() { + break; + } + } + } + }); + spawn_blocking({ + let stderr = child.stderr.take().expect("piped"); + let tx = tx.clone(); + move || { + for line in BufReader::new(stderr).lines() { + if tx.send(("stderr", line.unwrap())).is_err() { + break; + } + } + } + }); + + // logger is installed on this thread, so we send logs to this thread. + tokio::spawn({ + let span = span.clone(); + async move { + while let Some((stream, line)) = rx.recv().await { + trace!(target: "test", parent: &span, stream=stream, "{}", line); + } + } + }); + + let output = spawn_blocking(move || child.wait_with_output()).await??; + + if output.status.success() { + trace!(target: "test", parent: &span, exit_code=output.status.code(), "exited"); + } else { + error!(target: "test", parent: &span, exit_code=output.status.code(), "exited"); + }; + + if !output.status.success() { + return Err(eyre!("Command({params}) ({:?}):", output.status)); + } + + Ok(()) +} + +pub async fn get_command_output(params: &CommandParams<'_>) -> Result> { + let span = span!(target: "test", Level::ERROR, "command", %params); + assert!(!span.is_disabled(), "logger should be set"); + + info!(target: "test", parent: &span, "start"); + + let mut command = Command::new(params.command); + command.args(params.config_args); + command.args(params.args); + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); + if params.stdin.is_some() { + command.stdin(Stdio::piped()); + } + + let output = spawn_blocking({ + let params = params.clone(); + move || -> Result { + let child = command.spawn()?; + + if let Some(stdin) = params.stdin.as_ref() { + let mut pipe = child.stdin.as_ref().expect("piped"); + pipe.write_all(stdin)?; + } + + Ok(child.wait_with_output()?) + } + }) + .await + .context(format!("command: {params}"))??; + + info!(target: "test", parent: &span, "exited ({status})", status = output.status); + + if !output.status.success() { + stdout().write_all(&output.stdout).unwrap(); + stderr().write_all(&output.stderr).unwrap(); + return Err(eyre!("Command({params}) ({:?}):", output.status)); + } + + Ok(output.stdout) +} diff --git a/tests/webhooks.rs b/tests/webhooks.rs new file mode 100644 index 0000000..be629f9 --- /dev/null +++ b/tests/webhooks.rs @@ -0,0 +1,545 @@ +use std::io::Cursor; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use base64::Engine; +use eyre::{ContextCompat, Result}; +use k8s_openapi::api::admissionregistration::v1::{ + MutatingWebhookConfiguration, ValidatingWebhookConfiguration, +}; +use k8s_openapi::api::apps::v1::Deployment; +use k8s_openapi::api::core::v1::{Pod, Service}; +use k8s_openapi::api::discovery::v1::EndpointSlice; +use k8s_openapi::api::networking::v1::Ingress; +use kube::api::{ListParams, ObjectList}; +use rcgen::generate_simple_self_signed; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use uuid::Uuid; + +use pod_graceful_drain::{Config, LoadBalancingConfig, ServiceRegistry, WebhookConfig}; + +use crate::testutils::context::{within_test_namespace, TestContext}; +use crate::testutils::event_tracker::EventTracker; +use crate::testutils::operations::install_test_host_service; + +mod testutils; + +async fn generate_self_signed_cert( + subject: String, +) -> Result<(String, CertificateDer<'static>, PrivateKeyDer<'static>)> { + let cert_key = generate_simple_self_signed(vec![subject])?; + + let ca_bundle = base64::engine::general_purpose::STANDARD.encode(cert_key.cert.pem()); + let cert = cert_key.cert.der().clone(); + let private_key = { + let pem = cert_key.key_pair.serialize_pem(); + let mut cursor = Cursor::new(pem.as_bytes()); + rustls_pemfile::private_key(&mut cursor)?.context("private key")? + }; + + Ok((ca_bundle, cert, private_key)) +} + +async fn setup(context: &TestContext, config: Config) { + let namespace = &context.namespace; + let service_domain = install_test_host_service(context).await; + let (ca_bundle, cert, key_pair) = generate_self_signed_cert(service_domain).await.unwrap(); + let service_registry = ServiceRegistry::default(); + let loadbalancing = LoadBalancingConfig::new(Uuid::nil()); + + pod_graceful_drain::start_controller( + &context.api_resolver, + &service_registry, + &loadbalancing, + &context.shutdown, + ) + .unwrap(); + + let stores = pod_graceful_drain::start_reflectors( + &context.api_resolver, + &config, + &service_registry, + &context.shutdown, + ) + .unwrap(); + + let port = pod_graceful_drain::start_webhook( + &context.api_resolver, + config, + WebhookConfig::random_port_for_test(cert, key_pair), + stores, + &service_registry, + &loadbalancing, + &context.shutdown, + ) + .await + .unwrap() + .port(); + + apply_yaml!( + context, + ValidatingWebhookConfiguration, + r#" +metadata: + name: {namespace}-webhook +webhooks: + - name: validate.pod-graceful-drain.io + admissionReviewVersions: [v1beta1, v1] + clientConfig: + caBundle: {ca_bundle} + service: + namespace: {namespace} + name: test-host + path: /webhook/validate + port: {port} + rules: + - apiGroups: [""] + apiVersions: [v1] + operations: [DELETE] + resources: [pods] + failurePolicy: Fail + sideEffects: None + timeoutSeconds: 15 + namespaceSelector: + matchLabels: + name: {namespace}"#, + ); + + apply_yaml!( + context, + MutatingWebhookConfiguration, + r#" +metadata: + name: {namespace}-webhook +webhooks: + - name: mutate.pod-graceful-drain.io + admissionReviewVersions: [v1beta1, v1] + clientConfig: + caBundle: {ca_bundle} + service: + namespace: {namespace} + name: test-host + path: /webhook/mutate + port: {port} + rules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE] + resources: [pods/eviction] + failurePolicy: Fail + sideEffects: NoneOnDryRun + namespaceSelector: + matchLabels: + name: {namespace}"#, + ); +} + +async fn pod_is_alive(context: &TestContext, name: &str) -> bool { + let pod = context.api_resolver.all::().get_metadata(name).await; + match pod { + Ok(pod) => pod.metadata.deletion_timestamp.is_none(), + Err(kube::Error::Api(err)) if err.code == 404 || err.code == 409 => false, + Err(err) => panic!("error: {err:?}"), + } +} + +async fn pod_is_alive_for(context: &TestContext, name: &str, timeout: Duration) -> bool { + let start = Instant::now(); + while Instant::now() - start < timeout { + if !pod_is_alive(context, name).await { + return false; + } + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + true +} + +async fn pod_is_deleted_within(context: &TestContext, name: &str, timeout: Duration) -> bool { + let start = Instant::now(); + while Instant::now() - start < timeout { + if !pod_is_alive(context, name).await { + return true; + } + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + false +} + +const DELETE_AFTER_SECS: u64 = 10; +const DELETE_AFTER: Duration = Duration::from_secs(DELETE_AFTER_SECS); +const DELETE_DELAY_APPROX_SECS: u64 = DELETE_AFTER_SECS * 60 / 100; +const DELETE_DELAY_APPROX: Duration = Duration::from_secs(DELETE_DELAY_APPROX_SECS); + +#[tokio::test] +async fn should_delay_deletion_by_kubectl_delete() { + within_test_namespace(|context| async move { + let config = Config { + delete_after: DELETE_AFTER, + experimental_general_ingress: true, + }; + setup(&context, config).await; + + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + + apply_yaml!( + &context, + Service, + r#" +metadata: + name: some-service +spec: + ports: + - name: http + port: 80 + selector: + app: test"# + ); + + apply_yaml!( + &context, + Ingress, + r#" +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - backend: + service: + name: some-service + port: + name: http + pathType: Exact + path: /"# + ); + + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + let context = Arc::new(context); + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(5)).await; + + let first = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!(&context, ["delete", "pod", "some-pod"]); + let duration = Instant::now() - start; + + assert!( + duration > DELETE_DELAY_APPROX, + "should be delayed approx. 10s" + ); + } + }); + + assert!(event_tracker.issued_soon("DelayDeletion", "Drain").await); + + assert_eq!( + Some(true), + { + let pod: Pod = context.api_resolver.all().get("some-pod").await.unwrap(); + let labels = pod.metadata.labels.as_ref(); + let annotations = pod.metadata.annotations.as_ref(); + labels + .map(|l| l.contains_key("pod-graceful-drain/draining")) + .and(annotations.map(|a| a.contains_key("pod-graceful-drain/drain-until"))) + }, + "pod should've been patched" + ); + assert!( + { + let es_list: ObjectList = context + .api_resolver + .all() + .list(&ListParams::default().labels("kubernetes.io/service-name=some-service")) + .await + .unwrap(); + es_list.items.iter().all(|es| es.endpoints.is_empty()) + }, + "pod should've been removed from the endpointslices" + ); + + let second = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!(&context, ["delete", "pod", "some-pod"]); + let duration = Instant::now() - start; + assert!( + duration > DELETE_DELAY_APPROX, + "should still wait approx. 10s" + ); + } + }); + assert!(event_tracker.issued_soon("DelayDeletion", "Draining").await); + + assert!( + pod_is_alive_for(&context, "some-pod", DELETE_DELAY_APPROX).await, + "pod is alive for approx. 10s" + ); + + first.await.unwrap(); + second.await.unwrap(); + + assert!( + pod_is_deleted_within(&context, "some-pod", Duration::from_secs(20)).await, + "pod is eventually deleted" + ); + }) + .await; +} + +#[tokio::test] +async fn should_allow_deletion_when_pod_is_not_ready() { + within_test_namespace(|context| async move { + let config = Config { + delete_after: DELETE_AFTER, + experimental_general_ingress: true, + }; + setup(&context, config).await; + + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"] + readinessProbe: + httpGet: + path: /no-existing + port: 8080"# + ); + + apply_yaml!( + &context, + Service, + r#" +metadata: + name: some-service +spec: + ports: + - name: http + port: 80 + selector: + app: test"# + ); + + apply_yaml!( + &context, + Ingress, + r#" +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - backend: + service: + name: some-service + port: + name: http + pathType: Exact + path: /"# + ); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(1)).await; + kubectl!(&context, ["delete", "pod", "some-pod", "--wait=false"]); + assert!(event_tracker.issued_soon("AllowDeletion", "NotReady").await); + }) + .await; +} + +#[tokio::test] +async fn should_allow_deletion_when_pod_is_not_exposed() { + within_test_namespace(|context| async move { + let config = Config { + delete_after: DELETE_AFTER, + experimental_general_ingress: true, + }; + setup(&context, config).await; + + kubectl!( + &context, + [ + "run", + "some-pod", + "--image=public.ecr.aws/docker/library/busybox", + "--", + "sleep", + "9999" + ] + ); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(1)).await; + kubectl!(&context, ["delete", "pod", "some-pod"]); + assert!( + event_tracker + .issued_soon("AllowDeletion", "NotExposed") + .await + ); + }) + .await; +} + +#[tokio::test] +async fn should_allow_deletion_when_dry_run() { + within_test_namespace(|context| async move { + let config = Config { + delete_after: DELETE_AFTER, + experimental_general_ingress: true, + }; + setup(&context, config).await; + + kubectl!( + &context, + [ + "run", + "some-pod", + "--image=public.ecr.aws/docker/library/busybox", + "--", + "sleep", + "9999" + ] + ); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(1)).await; + kubectl!(&context, ["delete", "pod", "some-pod", "--dry-run=server"]); + assert!(event_tracker.issued_soon("Allow", "DryRun").await); + }) + .await; +} + +#[tokio::test] +async fn should_delay_deletion_by_deployment_rollout() { + within_test_namespace(|context| async move { + let config = Config { + delete_after: DELETE_AFTER, + experimental_general_ingress: true, + }; + setup(&context, config).await; + + apply_yaml!( + &context, + Deployment, + r#" +metadata: + name: some-deploy +spec: + selector: + matchLabels: + app: some-deploy + template: + metadata: + labels: + app: some-deploy + spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + + apply_yaml!( + &context, + Service, + r#" +metadata: + name: some-service +spec: + ports: + - name: http + port: 80 + selector: + app: some-deploy"# + ); + + apply_yaml!( + &context, + Ingress, + r#" +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - backend: + service: + name: some-service + port: + name: http + pathType: Exact + path: /"# + ); + + kubectl!( + &context, + [ + "wait", + "deployment/some-deploy", + "--for=condition=Available" + ] + ); + + let pod_name = context + .api_resolver + .all::() + .list_metadata(&ListParams::default()) + .await + .expect("list success") + .iter() + .next() + .expect("there's a pod") + .metadata + .name + .clone() + .expect("there's a pod name"); + + kubectl!(&context, ["rollout", "restart", "deployment/some-deploy"]); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(5)).await; + assert!(event_tracker.issued_soon("DelayDeletion", "Drain").await); + + assert!( + pod_is_alive_for(&context, &pod_name, DELETE_DELAY_APPROX).await, + "pod is alive for approx. 10s" + ); + + kubectl!( + &context, + ["rollout", "status", "deployment/some-deploy", "--watch"] + ); + + assert!( + pod_is_deleted_within(&context, &pod_name, Duration::from_secs(20)).await, + "pod is eventually deleted" + ); + }) + .await; +} diff --git a/tests/webhooks_drain.rs b/tests/webhooks_drain.rs new file mode 100644 index 0000000..00f7419 --- /dev/null +++ b/tests/webhooks_drain.rs @@ -0,0 +1,492 @@ +use std::io::Cursor; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use base64::Engine; +use eyre::{ContextCompat, Result}; +use k8s_openapi::api::admissionregistration::v1::{ + MutatingWebhookConfiguration, ValidatingWebhookConfiguration, +}; +use k8s_openapi::api::core::v1::{Pod, Service}; +use k8s_openapi::api::discovery::v1::EndpointSlice; +use k8s_openapi::api::networking::v1::Ingress; +use kube::api::{ListParams, ObjectList}; +use rcgen::generate_simple_self_signed; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use uuid::Uuid; + +use pod_graceful_drain::{Config, LoadBalancingConfig, ServiceRegistry, WebhookConfig}; + +use crate::testutils::context::{within_test_cluster, TestContext}; +use crate::testutils::event_tracker::EventTracker; +use crate::testutils::operations::install_test_host_service; + +mod testutils; + +async fn generate_self_signed_cert( + subject: String, +) -> Result<(String, CertificateDer<'static>, PrivateKeyDer<'static>)> { + let cert_key = generate_simple_self_signed(vec![subject])?; + + let ca_bundle = base64::engine::general_purpose::STANDARD.encode(cert_key.cert.pem()); + let cert = cert_key.cert.der().clone(); + let private_key = { + let pem = cert_key.key_pair.serialize_pem(); + let mut cursor = Cursor::new(pem.as_bytes()); + rustls_pemfile::private_key(&mut cursor)?.context("private key")? + }; + + Ok((ca_bundle, cert, private_key)) +} + +async fn setup(context: &TestContext, config: Config) { + let namespace = &context.namespace; + let service_domain = install_test_host_service(context).await; + let (ca_bundle, cert, key_pair) = generate_self_signed_cert(service_domain).await.unwrap(); + let service_registry = ServiceRegistry::default(); + let loadbalancing = LoadBalancingConfig::new(Uuid::nil()); + + pod_graceful_drain::start_controller( + &context.api_resolver, + &service_registry, + &loadbalancing, + &context.shutdown, + ) + .unwrap(); + let stores = pod_graceful_drain::start_reflectors( + &context.api_resolver, + &config, + &service_registry, + &context.shutdown, + ) + .unwrap(); + let port = pod_graceful_drain::start_webhook( + &context.api_resolver, + config, + WebhookConfig::random_port_for_test(cert, key_pair), + stores, + &service_registry, + &loadbalancing, + &context.shutdown, + ) + .await + .unwrap() + .port(); + + apply_yaml!( + context, + ValidatingWebhookConfiguration, + r#" +metadata: + name: {namespace}-webhook +webhooks: + - name: validate.pod-graceful-drain.io + admissionReviewVersions: [v1beta1, v1] + clientConfig: + caBundle: {ca_bundle} + service: + namespace: {namespace} + name: test-host + path: /webhook/validate + port: {port} + rules: + - apiGroups: [""] + apiVersions: [v1] + operations: [DELETE] + resources: [pods] + failurePolicy: Ignore + sideEffects: None + timeoutSeconds: 15 + namespaceSelector: + matchLabels: + name: {namespace}"#, + ); + + apply_yaml!( + context, + MutatingWebhookConfiguration, + r#" +metadata: + name: {namespace}-webhook +webhooks: + - name: mutate.pod-graceful-drain.io + admissionReviewVersions: [v1beta1, v1] + clientConfig: + caBundle: {ca_bundle} + service: + namespace: {namespace} + name: test-host + path: /webhook/mutate + port: {port} + rules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE] + resources: [pods/eviction] + failurePolicy: Ignore + sideEffects: NoneOnDryRun + namespaceSelector: + matchLabels: + name: {namespace}"#, + ); +} + +async fn pod_is_alive(context: &TestContext, name: &str) -> bool { + let pod = context.api_resolver.all::().get_metadata(name).await; + match pod { + Ok(pod) => pod.metadata.deletion_timestamp.is_none(), + Err(kube::Error::Api(err)) if err.code == 404 || err.code == 409 => false, + Err(err) => panic!("error: {err:?}"), + } +} + +async fn pod_is_alive_for(context: &TestContext, name: &str, timeout: Duration) -> bool { + let start = Instant::now(); + while Instant::now() - start < timeout { + if !pod_is_alive(context, name).await { + return false; + } + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + true +} + +async fn pod_is_deleted_within(context: &TestContext, name: &str, timeout: Duration) -> bool { + let start = Instant::now(); + while Instant::now() - start < timeout { + if !pod_is_alive(context, name).await { + return true; + } + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + false +} + +#[tokio::test] +async fn should_intercept_eviction_by_kubectl_drain() { + within_test_cluster(|context| async move { + let config = Config { + delete_after: Duration::from_secs(10), + experimental_general_ingress: true, + }; + setup(&context, config).await; + + // forcefully place Pod/some-pod to worker1 + kubectl!( + &context, + ["cordon", &format!("{}-worker2", &context.cluster_name)] + ); + + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + + apply_yaml!( + &context, + Service, + r#" +metadata: + name: some-service +spec: + ports: + - name: http + port: 80 + selector: + app: test"# + ); + + apply_yaml!( + &context, + Ingress, + r#" +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - backend: + service: + name: some-service + port: + name: http + pathType: Exact + path: /"# + ); + + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + kubectl!( + &context, + ["uncordon", &format!("{}-worker2", &context.cluster_name)] + ); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(5)).await; + let context = Arc::new(context); + + let first = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!( + &context, + [ + "drain", + "--force", + "--ignore-daemonsets", + &format!("{}-worker", &context.cluster_name) + ] + ); + let duration = Instant::now() - start; + assert!( + duration > Duration::from_secs(10 - 2), + "should be delayed approx 10s" + ); + } + }); + assert!( + event_tracker + .issued_soon("InterceptEviction", "Drain") + .await + ); + + assert_eq!( + Some(true), + { + let pod: Pod = context.api_resolver.all().get("some-pod").await.unwrap(); + let labels = pod.metadata.labels.as_ref(); + let annotations = pod.metadata.annotations.as_ref(); + labels + .map(|l| l.contains_key("pod-graceful-drain/draining")) + .and(annotations.map(|a| a.contains_key("pod-graceful-drain/drain-until"))) + }, + "pod should've been patched" + ); + assert!( + { + let es_list: ObjectList = context + .api_resolver + .all() + .list(&ListParams::default().labels("kubernetes.io/service-name=some-service")) + .await + .unwrap(); + es_list.items.iter().all(|es| es.endpoints.is_empty()) + }, + "pod should've been removed from the endpointslices" + ); + + let second = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!( + &context, + [ + "drain", + "--force", + "--ignore-daemonsets", + &format!("{}-worker", &context.cluster_name) + ] + ); + let duration = Instant::now() - start; + assert!( + duration > Duration::from_secs(10 - 2), + "should wait approx 10s" + ); + } + }); + assert!( + event_tracker + .issued_soon("InterceptEviction", "Draining") + .await + ); + + assert!( + pod_is_alive_for(&context, "some-pod", Duration::from_secs(10 - 2)).await, + "pod is alive for approx 10s" + ); + + first.await.unwrap(); + second.await.unwrap(); + + assert!(pod_is_deleted_within(&context, "some-pod", Duration::from_secs(5)).await); + }) + .await; +} + +#[tokio::test] +async fn should_intercept_deletion_by_kubectl_drain_disable_eviction() { + within_test_cluster(|context| async move { + let config = Config { + delete_after: Duration::from_secs(30), + experimental_general_ingress: true, + }; + setup(&context, config).await; + + // forcefully place Pod/some-pod to worker1 + kubectl!( + &context, + ["cordon", &format!("{}-worker2", &context.cluster_name)] + ); + + apply_yaml!( + &context, + Pod, + r#" +metadata: + name: some-pod + labels: + app: test +spec: + containers: + - name: app + image: public.ecr.aws/docker/library/busybox + command: ["sleep", "9999"]"# + ); + + apply_yaml!( + &context, + Service, + r#" +metadata: + name: some-service +spec: + ports: + - name: http + port: 80 + selector: + app: test"# + ); + + apply_yaml!( + &context, + Ingress, + r#" +metadata: + name: some-ingress +spec: + rules: + - http: + paths: + - backend: + service: + name: some-service + port: + name: http + pathType: Exact + path: /"# + ); + + kubectl!(&context, ["wait", "pod/some-pod", "--for=condition=Ready"]); + + kubectl!( + &context, + ["uncordon", &format!("{}-worker2", &context.cluster_name)] + ); + + let mut event_tracker = EventTracker::new(&context, Duration::from_secs(5)).await; + let context = Arc::new(context); + + let first = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!( + &context, + [ + "drain", + "--force", + "--ignore-daemonsets", + "--disable-eviction=true", + &format!("{}-worker", &context.cluster_name) + ] + ); + let duration = Instant::now() - start; + assert!( + duration > Duration::from_secs(30 - 5), + "should be delayed approx 30s again" + ); + } + }); + assert!(event_tracker.issued_soon("DelayDeletion", "Drain").await); + + assert_eq!( + Some(true), + { + let pod = context + .api_resolver + .all::() + .get_metadata("some-pod") + .await + .unwrap(); + let labels = pod.metadata.labels.as_ref(); + let annotations = pod.metadata.annotations.as_ref(); + labels + .map(|l| l.contains_key("pod-graceful-drain/draining")) + .and(annotations.map(|a| a.contains_key("pod-graceful-drain/drain-until"))) + }, + "pod should've been patched" + ); + assert!( + { + let es_list: ObjectList = context + .api_resolver + .all() + .list(&ListParams::default().labels("kubernetes.io/service-name=some-service")) + .await + .unwrap(); + es_list.items.iter().all(|es| es.endpoints.is_empty()) + }, + "pod should've been removed from the endpointslices" + ); + + let second = tokio::spawn({ + let context = Arc::clone(&context); + async move { + let start = Instant::now(); + kubectl!( + &context, + [ + "drain", + "--force", + "--ignore-daemonsets", + "--disable-eviction=true", + &format!("{}-worker", &context.cluster_name) + ] + ); + let duration = Instant::now() - start; + assert!( + duration > Duration::from_secs(10 - 2), + "should wait approx 10s" + ); + } + }); + assert!(event_tracker.issued_soon("DelayDeletion", "Draining").await); + + assert!( + pod_is_alive_for(&context, "some-pod", Duration::from_secs(10 - 2)).await, + "pod is alive for approx 10s" + ); + + first.await.unwrap(); + second.await.unwrap(); + + assert!(!pod_is_alive(&context, "some-pod").await); + }) + .await; +}