From ca75cc7d9222b21d991dafeafe9f07efe66c5b3e Mon Sep 17 00:00:00 2001 From: upodroid Date: Thu, 1 Feb 2024 13:58:39 +0000 Subject: [PATCH] add gce scale scenario --- tests/e2e/scenarios/scalability/run-test.sh | 149 +++++++++----------- 1 file changed, 66 insertions(+), 83 deletions(-) diff --git a/tests/e2e/scenarios/scalability/run-test.sh b/tests/e2e/scenarios/scalability/run-test.sh index 31fc31779f5ca..ef137657552f5 100755 --- a/tests/e2e/scenarios/scalability/run-test.sh +++ b/tests/e2e/scenarios/scalability/run-test.sh @@ -17,30 +17,7 @@ set -e set -x -REPO_ROOT=$(git rev-parse --show-toplevel) -cd "${REPO_ROOT}" -cd .. -WORKSPACE=$(pwd) -cd "${WORKSPACE}/kops" - -# Create bindir -BINDIR="${WORKSPACE}/bin" -export PATH="${BINDIR}:${PATH}" -mkdir -p "${BINDIR}" - -# Build kubetest-2 kOps support -pushd "${WORKSPACE}/kops" -GOBIN=${BINDIR} make test-e2e-install -popd - -# Setup our cleanup function; as we allocate resources we set a variable to indicate they should be cleaned up -function cleanup { - # shellcheck disable=SC2153 - if [[ "${DELETE_CLUSTER:-}" == "true" ]]; then - kubetest2 kops "${KUBETEST2_ARGS[@]}" --down || echo "kubetest2 down failed" - fi -} -trap cleanup EXIT +make test-e2e-install # Default cluster name SCRIPT_NAME=$(basename "$(dirname "$0")") @@ -53,30 +30,26 @@ if [[ -z "${K8S_VERSION:-}" ]]; then K8S_VERSION="$(curl -s -L https://dl.k8s.io/release/stable.txt)" fi -# Download latest prebuilt kOps -if [[ -z "${KOPS_BASE_URL:-}" ]]; then - KOPS_BASE_URL="$(curl -s https://storage.googleapis.com/kops-ci/bin/latest-ci.txt)" -fi -export KOPS_BASE_URL - -KOPS_BIN=${BINDIR}/kops -wget -qO "${KOPS_BIN}" "$KOPS_BASE_URL/$(go env GOOS)/$(go env GOARCH)/kops" -chmod +x "${KOPS_BIN}" +# A temp patch for kubetest2 https://github.com/kubernetes-sigs/kubetest2/pull/256 +git clone https://github.com/kubernetes-sigs/kubetest2.git /tmp/kubetest2 +pushd /tmp/kubetest2 +make install-all +popd # Default cloud provider to aws if [[ -z "${CLOUD_PROVIDER:-}" ]]; then CLOUD_PROVIDER="aws" fi echo "CLOUD_PROVIDER=${CLOUD_PROVIDER}" - -# KOPS_STATE_STORE holds metadata about the clusters we create -if [[ -z "${KOPS_STATE_STORE:-}" ]]; then - echo "Must specify KOPS_STATE_STORE" - exit 1 +if [[ "${CLOUD_PROVIDER}" != "gce" ]]; then + # KOPS_STATE_STORE holds metadata about the clusters we create + if [[ -z "${KOPS_STATE_STORE:-}" ]]; then + echo "Must specify KOPS_STATE_STORE" + exit 1 + fi + echo "KOPS_STATE_STORE=${KOPS_STATE_STORE}" + export KOPS_STATE_STORE fi -echo "KOPS_STATE_STORE=${KOPS_STATE_STORE}" -export KOPS_STATE_STORE - if [[ -z "${ADMIN_ACCESS:-}" ]]; then ADMIN_ACCESS="0.0.0.0/0" # Or use your IPv4 with /32 @@ -86,12 +59,25 @@ echo "ADMIN_ACCESS=${ADMIN_ACCESS}" # cilium does not yet pass conformance tests (shared hostport test) #create_args="--networking cilium" create_args=() -create_args=("--network-cidr=10.0.0.0/16,10.1.0.0/16,10.2.0.0/16,10.3.0.0/16,10.4.0.0/16,10.5.0.0/16,10.6.0.0/16,10.7.0.0/16,10.8.0.0/16,10.9.0.0/16,10.10.0.0/16,10.11.0.0/16,10.12.0.0/16") +if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then + create_args+=("--network-cidr=10.0.0.0/16,10.1.0.0/16,10.2.0.0/16,10.3.0.0/16,10.4.0.0/16,10.5.0.0/16,10.6.0.0/16,10.7.0.0/16,10.8.0.0/16,10.9.0.0/16,10.10.0.0/16,10.11.0.0/16,10.12.0.0/16") + create_args+=("--node-size=t3a.medium,t3.medium,t2.medium,t3a.large,c5a.large,t3.large,c5.large,m5a.large,m6a.large,m5.large,c4.large,c7a.large,r5a.large,r6a.large,m7a.large") + create_args+=("--node-volume-size=20") + create_args+=("--zones=us-east-2a,us-east-2b,us-east-2c") + create_args+=("--image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id}") +fi +if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then + create_args+=("--zones=us-east1-b,us-east1-c,us-east1-d") + create_args+=("--node-size=e2-standard-2") + create_args+=("--node-volume-size=30") + create_args+=("--master-volume-size=1000") + create_args+=("--gce-service-account=default") + create_args+=("--image=${INSTANCE_IMAGE:-ubuntu-os-cloud/ubuntu-2204-jammy-v20231213a}") +fi create_args+=("--networking=${CNI_PLUGIN:-calico}") if [[ "${CNI_PLUGIN}" == "amazonvpc" ]]; then -create_args+=("--set spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true") + create_args+=("--set spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true") fi -create_args+=("--image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/20.04/stable/current/amd64/hvm/ebs-gp2/ami-id}") create_args+=("--set spec.etcdClusters[0].manager.listenMetricsURLs=http://localhost:2382") create_args+=("--set spec.etcdClusters[0].manager.env=ETCD_QUOTA_BACKEND_BYTES=8589934592") create_args+=("--set spec.etcdClusters[1].manager.env=ETCD_QUOTA_BACKEND_BYTES=8589934592") @@ -114,18 +100,16 @@ create_args+=("--set spec.kubeAPIServer.maxRequestsInflight=800") create_args+=("--set spec.kubeAPIServer.maxMutatingRequestsInflight=400") create_args+=("--set spec.kubeAPIServer.enableProfiling=true") create_args+=("--set spec.kubeAPIServer.enableContentionProfiling=true") +create_args+=("--set spec.kubeAPIServer.logLevel=2") # this is required for Prometheus server to scrape metrics endpoint on APIServer create_args+=("--set spec.kubeAPIServer.anonymousAuth=true") # this is required for prometheus to scrape kube-proxy metrics endpoint create_args+=("--set spec.kubeProxy.metricsBindAddress=0.0.0.0:10249") create_args+=("--node-count=${KUBE_NODE_COUNT:-101}") # TODO: track failures of tests (HostPort & OIDC) when using `--dns=none` -create_args+=("--dns none") -create_args+=("--node-size=t3a.medium,t3.medium,t2.medium,t3a.large,c5a.large,t3.large,c5.large,m5a.large,m6a.large,m5.large,c4.large,c7a.large,r5a.large,r6a.large,m7a.large") -create_args+=("--node-volume-size=20") +create_args+=("--dns=none") create_args+=("--control-plane-count=${CONTROL_PLANE_COUNT:-1}") create_args+=("--master-size=${CONTROL_PLANE_SIZE:-c5.2xlarge}") -create_args+=("--zones=us-east-2a,us-east-2b,us-east-2c") # AWS ONLY feature flags @@ -141,10 +125,16 @@ KUBETEST2_ARGS=() KUBETEST2_ARGS+=("-v=2") KUBETEST2_ARGS+=("--cloud-provider=${CLOUD_PROVIDER}") KUBETEST2_ARGS+=("--cluster-name=${CLUSTER_NAME:-}") -KUBETEST2_ARGS+=("--kops-binary-path=${KOPS_BIN}") +KUBETEST2_ARGS+=("--kops-version-marker=https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt") KUBETEST2_ARGS+=("--admin-access=${ADMIN_ACCESS:-}") KUBETEST2_ARGS+=("--env=KOPS_FEATURE_FLAGS=${KOPS_FEATURE_FLAGS}") +if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then + KUBETEST2_ARGS+=("--boskos-resource-type=scalability-scale-project") + KUBETEST2_ARGS+=("--control-plane-instance-group-overrides=spec.rootVolume.type=pd-ssd") + KUBETEST2_ARGS+=("--max-nodes-to-dump=10") +fi + # More time for bigger clusters KUBETEST2_ARGS+=("--validation-wait=55m") KUBETEST2_ARGS+=("--validation-count=3") @@ -155,24 +145,18 @@ if [[ -z "${DELETE_CLUSTER:-}" ]]; then DELETE_CLUSTER="true" fi -kubetest2 kops "${KUBETEST2_ARGS[@]}" \ - --up \ - --kubernetes-version="${K8S_VERSION}" \ - --create-args="${create_args[*]}" - -KUBECONFIG=$(mktemp -t kubeconfig.XXXXXXXXX) -kops export kubecfg --admin --kubeconfig="${KUBECONFIG}" - -kops get instances +if [[ "${DELETE_CLUSTER:-}" == "true" ]]; then + KUBETEST2_ARGS+=("--down") +fi -# CL2 uses KUBE_SSH_KEY_PATH path to ssh to instances for scraping metrics -export KUBE_SSH_KEY_PATH="/tmp/kops/${CLUSTER_NAME}/id_ed25519" # this is used as a label to select kube-proxy pods on kops for kube-proxy service # used by CL2 Prometheus here https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/pkg/prometheus/manifests/default/kube-proxy-service.yaml#L2 export PROMETHEUS_KUBE_PROXY_SELECTOR_KEY="k8s-app" export PROMETHEUS_SCRAPE_APISERVER_ONLY="true" export CL2_PROMETHEUS_TOLERATE_MASTER="true" if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then + # CL2 uses KUBE_SSH_KEY_PATH path to ssh to instances for scraping metrics + export KUBE_SSH_KEY_PATH="/tmp/kops/${CLUSTER_NAME}/id_ed25519" cat > "${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/load/overrides.yaml < "${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/load/overrides.yaml <