diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 19c6dd13516..82bca84df63 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -376,7 +376,7 @@ blocks: - name: "Felix: Windows FV capz" run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" + when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" dependencies: ["Felix: Build Windows binaries"] task: secrets: @@ -384,6 +384,7 @@ blocks: - name: private-repo prologue: commands: + - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - export REPORT_DIR=/home/semaphore/report - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - export AZURE_TENANT_ID=$AZ_TENANT_ID diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d7846592f0c..3209e3033db 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -376,7 +376,7 @@ blocks: - name: "Felix: Windows FV capz" run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" + when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" dependencies: ["Felix: Build Windows binaries"] task: secrets: @@ -384,6 +384,7 @@ blocks: - name: private-repo prologue: commands: + - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - export REPORT_DIR=/home/semaphore/report - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - export AZURE_TENANT_ID=$AZ_TENANT_ID diff --git a/.semaphore/semaphore.yml.d/blocks/20-felix.yml b/.semaphore/semaphore.yml.d/blocks/20-felix.yml index 92341841690..51c47bcc8e5 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-felix.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-felix.yml @@ -93,7 +93,7 @@ - name: "Felix: Windows FV capz" run: - when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" + when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})" dependencies: ["Felix: Build Windows binaries"] task: secrets: @@ -101,6 +101,7 @@ - name: private-repo prologue: commands: + - az login --service-principal -u "${AZ_SP_ID}" -p "${AZ_SP_PASSWORD}" --tenant "${AZ_TENANT_ID}" --output none - export REPORT_DIR=/home/semaphore/report - export AZURE_SUBSCRIPTION_ID=$AZ_SUBSCRIPTION_ID - export AZURE_TENANT_ID=$AZ_TENANT_ID diff --git a/cni-plugin/.semaphore/cleanup.yml b/cni-plugin/.semaphore/cleanup.yml index 9174613da58..381b6f83c2c 100644 --- a/cni-plugin/.semaphore/cleanup.yml +++ b/cni-plugin/.semaphore/cleanup.yml @@ -35,7 +35,7 @@ blocks: - name: Clean up winfv aws resources commands: - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true - - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}-containerd" ./setup-fv.sh -q -u || true + - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}-containerd" ./setup-fv.sh -q -u || true - NAME_PREFIX="${CLUSTER_NAME}-docker" ./setup-fv.sh -q -u | true env_vars: - name: AWS_DEFAULT_REGION diff --git a/cni-plugin/.semaphore/semaphore.yml b/cni-plugin/.semaphore/semaphore.yml index 72c98d87e18..a371ebb8ee7 100644 --- a/cni-plugin/.semaphore/semaphore.yml +++ b/cni-plugin/.semaphore/semaphore.yml @@ -107,7 +107,7 @@ blocks: - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true - - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u + - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u env_vars: - name: SEMAPHORE_ARTIFACT_EXPIRY value: 2w @@ -152,7 +152,7 @@ blocks: - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true - - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u + - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u env_vars: - name: SEMAPHORE_ARTIFACT_EXPIRY value: 2w diff --git a/felix/.semaphore/run-win-fv b/felix/.semaphore/run-win-fv index ccbad883bc8..46ae6a138d8 100755 --- a/felix/.semaphore/run-win-fv +++ b/felix/.semaphore/run-win-fv @@ -14,10 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -ex -FV_DIR="/home/semaphore/calico/process/testing/winfv" -ERROR_CODE=0 +FV_DIR="/home/semaphore/calico/process/testing/winfv-felix" +EXIT_CODE=0 pushd ${FV_DIR} # Prepare local files @@ -51,18 +51,23 @@ if [[ $FV_PROVISIONER == "aws" ]]; then ${SCP_CMD} -r ubuntu@${MASTER_IP}:/home/ubuntu/report /home/semaphore elif [[ $FV_PROVISIONER == "capz" ]]; then export KUBE_VERSION="$K8S_VERSION" - /bin/bash -x ./setup-fv-capz.sh -q | tee setup-fv.log - if [[ ${PIPESTATUS[0]} != 0 ]]; then - ERROR_CODE=${PIPESTATUS[0]} + /bin/bash -x ./setup-fv-capz.sh -q | tee setup-fv.log; pstat=${PIPESTATUS[0]} + if [[ $pstat != 0 ]]; then + EXIT_CODE=$pstat fi - mv ./report /home/semaphore/report - mv ./setup-fv.log /home/semaphore/report/setup-fv.log + mv ./report /home/semaphore + mv ./setup-fv.log /home/semaphore/report popd fi -ls -ltr ./report -mkdir /home/semaphore/fv.log -# check if *.log glob contains any files so that mv doesn't fail -compgen -G /home/semaphore/report/*.log > /dev/null && mv /home/semaphore/report/*.log /home/semaphore/fv.log +ls -ltr /home/semaphore/report + +# Print relevant snippets from logs +log_regexps='(? /dev/null && \ +for log_file in /home/semaphore/report/*.log; do + prefix="[$(basename ${log_file})]" + cat ${log_file} | iconv -f UTF-16 -t UTF-8 | sed 's/\r$//g' | grep --line-buffered --perl ${log_regexps} -B 2 -A 15 | sed 's/.*/'"${prefix}"' &/g' +done; # Stop for debug echo "Check for pause file..." @@ -72,16 +77,8 @@ do sleep 30 done -# Print relevant snippets from logs -log_regexps='(? /dev/null && \ -for log_file in /home/semaphore/fv.log/*.log; do - prefix="[$(basename ${log_file})]" - cat ${log_file} | iconv -f UTF-16 -t UTF-8 | sed 's/\r$//g' | grep --line-buffered --perl ${log_regexps} -B 2 -A 15 | sed 's/.*/'"${prefix}"' &/g' -done; - # Search for error code file -if [[ -f /home/semaphore/report/error-codes || $ERROR_CODE != 0 ]]; +if [[ -f /home/semaphore/report/error-codes || $EXIT_CODE != 0 ]]; then echo "Windows FV returned error(s)." exit 1 diff --git a/felix/.semaphore/semaphore.yml b/felix/.semaphore/semaphore.yml index 69806ec8c9b..32c5b80e58e 100644 --- a/felix/.semaphore/semaphore.yml +++ b/felix/.semaphore/semaphore.yml @@ -104,7 +104,7 @@ blocks: - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true - - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" /bin/bash -x ./setup-fv.sh -q -u + - cd ~/calico/process/testing/winfv-felix && NAME_PREFIX="${CLUSTER_NAME}" /bin/bash -x ./setup-fv.sh -q -u env_vars: - name: SEMAPHORE_ARTIFACT_EXPIRY value: 2w diff --git a/metadata.mk b/metadata.mk index 27d74f7f277..9f7e6e616d7 100644 --- a/metadata.mk +++ b/metadata.mk @@ -14,8 +14,8 @@ K8S_VERSION=v1.29.7 COREDNS_VERSION=1.5.2 ETCD_VERSION=v3.5.6 HELM_VERSION=v3.11.3 -KINDEST_NODE_VERSION=v1.29.7 -KIND_VERSION=v0.22.0 +KINDEST_NODE_VERSION=v1.29.2 +KIND_VERSION=v0.24.0 PROTOC_VER=v0.1 UBI_VERSION=8.10 diff --git a/process/testing/winfv-cni-plugin/aso/utils.sh b/process/testing/winfv-cni-plugin/aso/utils.sh index 1cf81ae57cb..e496f969399 100755 --- a/process/testing/winfv-cni-plugin/aso/utils.sh +++ b/process/testing/winfv-cni-plugin/aso/utils.sh @@ -19,8 +19,8 @@ function retry_command() { local CMD=$2 echo - for i in `seq 1 $RETRY`; do - echo Trying $CMD, attempt ${i} + for i in $(seq 1 $RETRY); do + echo "Trying '$CMD', attempt ${i}" $CMD && return 0 || sleep 10 done echo "Command '${CMD}' failed after $RETRY attempts" diff --git a/process/testing/winfv/.gitignore b/process/testing/winfv-felix/.gitignore similarity index 100% rename from process/testing/winfv/.gitignore rename to process/testing/winfv-felix/.gitignore diff --git a/process/testing/winfv/README.md b/process/testing/winfv-felix/README.md similarity index 100% rename from process/testing/winfv/README.md rename to process/testing/winfv-felix/README.md diff --git a/process/testing/winfv/calico-node-windows.yaml b/process/testing/winfv-felix/calico-node-windows.yaml similarity index 100% rename from process/testing/winfv/calico-node-windows.yaml rename to process/testing/winfv-felix/calico-node-windows.yaml diff --git a/process/testing/winfv/capz/.gitignore b/process/testing/winfv-felix/capz/.gitignore similarity index 76% rename from process/testing/winfv/capz/.gitignore rename to process/testing/winfv-felix/capz/.gitignore index 72f10c6194c..fb2f0c7890d 100644 --- a/process/testing/winfv/capz/.gitignore +++ b/process/testing/winfv-felix/capz/.gitignore @@ -1,10 +1,13 @@ -.calico_installed +bin/* +*.log .cluster_created +.calico_installed .sshkey .sshkey.pub kubeconfig scp-from-node.sh scp-to-node.sh ssh-node.sh -tigera-operator.yaml win-capz.yaml +tigera-operator.yaml +tigera-prometheus-operator.yaml diff --git a/process/testing/winfv/capz/Makefile b/process/testing/winfv-felix/capz/Makefile similarity index 89% rename from process/testing/winfv/capz/Makefile rename to process/testing/winfv-felix/capz/Makefile index c4f8e36197b..a893fc7b729 100644 --- a/process/testing/winfv/capz/Makefile +++ b/process/testing/winfv-felix/capz/Makefile @@ -9,7 +9,7 @@ CLUSTER_CREATED_MARKER:=.cluster_created .PHONY: create-cluster create-cluster: $(CLUSTER_CREATED_MARKER) -$(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl +$(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl $(BINDIR)/yq @echo "Creating cluster $(CLUSTER_NAME_CAPZ) ..." ./create-cluster.sh $(MAKE) generate-helpers @@ -18,19 +18,16 @@ $(CLUSTER_CREATED_MARKER): $(BINDIR)/kind $(BINDIR)/kubectl $(BINDIR)/clusterctl .PHONY: delete-cluster delete-cluster: $(BINDIR)/kind $(BINDIR)/kubectl -ifeq (,$(wildcard $(CLUSTER_CREATED_MARKER))) - @echo "Cluster marker '$(CLUSTER_CREATED_MARKER)' does not exist, doing nothing" -else @echo "Azure resources for cluster $(CLUSTER_NAME_CAPZ) will now be deleted, this can take up to 20 minutes" -$(BINDIR)/kubectl delete cluster $(CLUSTER_NAME_CAPZ) -$(BINDIR)/kind delete cluster --name kind${SUFFIX} + -az group delete --name $(CI_RG) -y -rm -f kubeconfig -rm -f win-capz.yaml -rm -f tigera-operator.yaml -rm -f tigera-prometheus-operator.yaml -rm -f $(HELPERS) -rm -f $(CLUSTER_CREATED_MARKER) $(CALICO_INSTALLED_MARKER) -endif CALICO_INSTALLED_MARKER:=.calico_installed @@ -70,6 +67,12 @@ $(BINDIR)/clusterctl: touch $@ $(BINDIR)/clusterctl version +$(BINDIR)/yq: + mkdir -p $(@D) + curl -sSf -L --retry 5 https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_linux_$(ARCH) -o $(BINDIR)/yq + chmod +x $@ + touch $@ + .PHONY: clean clean: -rm -f kubeconfig @@ -77,8 +80,10 @@ clean: -rm -f tigera-operator.yaml -rm -f .sshkey .sshkey.pub -rm -f $(HELPERS) + -rm -f az-output.log .PHONY: dist-clean dist-clean: clean -rm -rf $(BINDIR) -rm -f $(CLUSTER_CREATED_MARKER) $(CALICO_INSTALLED_MARKER) + -rm -f *.log diff --git a/process/testing/winfv/capz/OSS/custom-resources.yaml b/process/testing/winfv-felix/capz/OSS/custom-resources.yaml similarity index 100% rename from process/testing/winfv/capz/OSS/custom-resources.yaml rename to process/testing/winfv-felix/capz/OSS/custom-resources.yaml diff --git a/process/testing/winfv/capz/README.md b/process/testing/winfv-felix/capz/README.md similarity index 94% rename from process/testing/winfv/capz/README.md rename to process/testing/winfv-felix/capz/README.md index 9c731430ee8..ce83cdc3651 100644 --- a/process/testing/winfv/capz/README.md +++ b/process/testing/winfv-felix/capz/README.md @@ -45,6 +45,8 @@ Optionally, define `PRODUCT`, `RELEASE_STREAM` and/or `HASH_RELEASE`: make install-calico PRODUCT=calient RELEASE_STREAM=master HASH_RELEASE=true ``` +(Use `RELEASE_STREAM=local` to use local manifests from the monorepo instead of pulling them) + To access your cluster, run `kubectl --kubeconfig=./kubeconfig ...` ### Access Linux or Windows nodes diff --git a/process/testing/winfv/capz/create-cluster.sh b/process/testing/winfv-felix/capz/create-cluster.sh similarity index 73% rename from process/testing/winfv/capz/create-cluster.sh rename to process/testing/winfv-felix/capz/create-cluster.sh index b0315b89e06..97798445796 100755 --- a/process/testing/winfv/capz/create-cluster.sh +++ b/process/testing/winfv-felix/capz/create-cluster.sh @@ -41,7 +41,24 @@ set -o pipefail export AZURE_CONTROL_PLANE_MACHINE_TYPE export AZURE_NODE_MACHINE_TYPE -# Number of Linux node is same as number of Windows nodes +export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$AZURE_CLIENT_ID # for compatibility with CAPZ v1.16 templates + +# Create the resource group and managed identity for the cluster CI +rm az-output.log || true +{ +echo "az group create --name ${CI_RG} --location ${AZURE_LOCATION}" +az group create --name ${CI_RG} --location ${AZURE_LOCATION} +echo +echo "az identity create --name ${USER_IDENTITY} --resource-group ${CI_RG} --location ${AZURE_LOCATION}" +az identity create --name ${USER_IDENTITY} --resource-group ${CI_RG} --location ${AZURE_LOCATION} +sleep 10s +export USER_IDENTITY_ID=$(az identity show --resource-group "${CI_RG}" --name "${USER_IDENTITY}" | jq -r .principalId) +echo +echo az role assignment create --assignee-object-id "${USER_IDENTITY_ID}" --assignee-principal-type "ServicePrincipal" --role "Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}" +az role assignment create --assignee-object-id "${USER_IDENTITY_ID}" --assignee-principal-type "ServicePrincipal" --role "Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}" +} >> az-output.log 2>&1 + +# Number of Linux worker nodes is the same as number of Windows worker nodes : ${WIN_NODE_COUNT:=2} TOTAL_NODES=$((WIN_NODE_COUNT*2+1)) SEMAPHORE="${SEMAPHORE:="false"}" @@ -57,6 +74,7 @@ echo ' WIN_NODE_COUNT='${WIN_NODE_COUNT} : ${KIND:=./bin/kind} : ${KUBECTL:=./bin/kubectl} : ${CLUSTERCTL:=./bin/clusterctl} +: ${YQ:=./bin/yq} : ${KCAPZ:="${KUBECTL} --kubeconfig=./kubeconfig"} # Base64 encode the variables @@ -69,8 +87,6 @@ else export SUFFIX="-${RAND}" fi - - # Settings needed for AzureClusterIdentity used by the AzureCluster export AZURE_CLUSTER_IDENTITY_SECRET_NAME="cluster-identity-secret" export CLUSTER_IDENTITY_NAME="cluster-identity" @@ -118,10 +134,14 @@ ${CLUSTERCTL} generate cluster ${CLUSTER_NAME_CAPZ} \ --flavor machinepool-windows \ > win-capz.yaml +# Cluster templates authenticate with Workload Identity by default. Modify the AzureClusterIdentity for ServicePrincipal authentication. +# See https://capz.sigs.k8s.io/topics/identities for more details. +${YQ} -i "with(. | select(.kind == \"AzureClusterIdentity\"); .spec.type |= \"ServicePrincipal\" | .spec.clientSecret.name |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}\" | .spec.clientSecret.namespace |= \"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}\")" win-capz.yaml + retry_command 600 "${KUBECTL} apply -f win-capz.yaml" # Wait for CAPZ deployments -${KUBECTL} wait --for=condition=Available --timeout=5m -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure +timeout --foreground 600 bash -c "while ! ${KUBECTL} wait --for=condition=Available --timeout=30s -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure; do sleep 5; done" # Wait for the kubeconfig to become available. timeout --foreground 600 bash -c "while ! ${KUBECTL} get secrets | grep ${CLUSTER_NAME_CAPZ}-kubeconfig; do sleep 5; done" @@ -140,9 +160,9 @@ retry_command 300 "${KCAPZ} taint nodes --selector=!node-role.kubernetes.io/cont echo "Done creating cluster" -ID0=$(${KCAPZ} get node -o wide | grep win-p-win000000 | awk '{print $6}' | awk -F '.' '{print $4}') -echo "ID0: $ID0" -if [[ ${WIN_NODE_COUNT} -gt 1 ]]; then - ID1=$(${KCAPZ} get node -o wide | grep win-p-win000001 | awk '{print $6}' | awk -F '.' '{print $4}') - echo "ID1:$ID1" -fi +WIN_NODES=$(${KCAPZ} get nodes -o wide -l kubernetes.io/os=windows --no-headers | awk '{print $6}' | awk -F '.' '{print $4}' | sort) +i=0 +for n in ${WIN_NODES} +do + echo "ID$i: $n"; i=$(expr $i + 1) +done diff --git a/process/testing/winfv/capz/export-env.sh b/process/testing/winfv-felix/capz/export-env.sh similarity index 57% rename from process/testing/winfv/capz/export-env.sh rename to process/testing/winfv-felix/capz/export-env.sh index 69fdeec0a35..5cde0b1ba60 100755 --- a/process/testing/winfv/capz/export-env.sh +++ b/process/testing/winfv-felix/capz/export-env.sh @@ -1,8 +1,11 @@ export CLUSTER_NAME_CAPZ="${CLUSTER_NAME_CAPZ:=${USER}-capz-win}" -export AZURE_LOCATION="${AZURE_LOCATION:="westcentralus"}" +export AZURE_LOCATION="${AZURE_LOCATION:="westus2"}" # [Optional] Select resource group. The default value is ${CLUSTER_NAME_CAPZ}-rg. export AZURE_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME_CAPZ}-rg}" +# These are required by the machinepool-windows template +export CI_RG="${AZURE_RESOURCE_GROUP}-ci" +export USER_IDENTITY="cloud-provider-user-identity" # Optional, can be windows-2019 or windows-2022 (default) # https://capz.sigs.k8s.io/developers/development.html @@ -13,17 +16,10 @@ export WINDOWS_SERVER_VERSION="${WINDOWS_SERVER_VERSION:="windows-2022"}" export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:="Standard_D2s_v3"}" export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:="Standard_D2s_v3"}" -# Get KINDEST_NODE_VERSION variable from metadata.mk, default to a value if it cannot be found -SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" -METADATAMK=${SCRIPT_CURRENT_DIR}/../../../../metadata.mk -if [ -f "${METADATAMK}" ]; then - export KUBE_VERSION=$(grep KINDEST_NODE_VERSION= ${METADATAMK} | cut -d "=" -f 2) - export KIND_VERSION=$(grep KIND_VERSION= ${METADATAMK} | cut -d "=" -f 2) -else - export KUBE_VERSION=v1.27.11 - export KIND_VERSION=v0.22.0 -fi -export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.6.3"}" +export KUBE_VERSION=v1.28.9 +export KIND_VERSION=v0.24.0 +export CLUSTER_API_VERSION="${CLUSTER_API_VERSION:="v1.8.1"}" export AZURE_PROVIDER_VERSION="${AZURE_PROVIDER_VERSION:="v1.13.2"}" -export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.13"}" +export CONTAINERD_VERSION="${CONTAINERD_VERSION:="v1.7.20"}" export CALICO_VERSION="${CALICO_VERSION:="v3.28.1"}" +export YQ_VERSION="${YQ_VERSION:="v4.44.3"}" diff --git a/process/testing/winfv/capz/generate-helpers.sh b/process/testing/winfv-felix/capz/generate-helpers.sh similarity index 80% rename from process/testing/winfv/capz/generate-helpers.sh rename to process/testing/winfv-felix/capz/generate-helpers.sh index 7a7bc3312e8..2bbb7067752 100755 --- a/process/testing/winfv/capz/generate-helpers.sh +++ b/process/testing/winfv-felix/capz/generate-helpers.sh @@ -1,4 +1,21 @@ #!/bin/bash +# Copyright (c) 2024 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail set -e LOCAL_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" @@ -7,11 +24,12 @@ LOCAL_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # with windows ssh servers, but older versions don't know about that flag. Only use # it when necessary (i.e. supported). OFLAG="-O " -if scp -O 2>&1 | grep -q "unknown option -- O"; then +if [ "$(scp -O 2>&1 | grep -c 'unknown option -- O')" -gt 0 ]; then OFLAG="" fi : ${KUBECTL:=${LOCAL_PATH}/bin/kubectl} +: ${WIN_NODE_COUNT:=2} KCAPZ="${KUBECTL} --kubeconfig=./kubeconfig" diff --git a/process/testing/winfv/capz/install-calico.sh b/process/testing/winfv-felix/capz/install-calico.sh similarity index 86% rename from process/testing/winfv/capz/install-calico.sh rename to process/testing/winfv-felix/capz/install-calico.sh index 600b428fcdd..05e96aed48b 100755 --- a/process/testing/winfv/capz/install-calico.sh +++ b/process/testing/winfv-felix/capz/install-calico.sh @@ -39,6 +39,9 @@ if [ ${PRODUCT} == 'calient' ]; then : "${TSEE_TEST_LICENSE:?Environment variable empty or not defined.}" fi +SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" +LOCAL_MANIFESTS_DIR="${SCRIPT_CURRENT_DIR}/../../../../manifests" + if [ ${PRODUCT} == 'calient' ]; then RELEASE_BASE_URL="https://downloads.tigera.io/ee/${RELEASE_STREAM}" else @@ -61,18 +64,33 @@ fi # Check release url and installation scripts echo "Set release base url ${RELEASE_BASE_URL}" sed -i "s,export RELEASE_BASE_URL.*,export RELEASE_BASE_URL=\"${RELEASE_BASE_URL}\"," ./export-env.sh + # Create a storage class and persistent volume for Calico Enterprise. if [ ${PRODUCT} == 'calient' ]; then ${KCAPZ} create -f ./EE/storage-class-azure-file.yaml ${KCAPZ} create -f ./EE/persistent-volume.yaml fi -# Install Calico on Linux nodes using local manifests -SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" -${KCAPZ} create -f ${SCRIPT_CURRENT_DIR}../../../../manifests/tigera-operator.yaml + +# Install Calico on Linux nodes +if [[ ${RELEASE_STREAM} == 'local' ]]; then + # Use local manifests + ${KCAPZ} create -f ${LOCAL_MANIFESTS_DIR}/tigera-operator.yaml +else + # Use release url + echo "Set release base url ${RELEASE_BASE_URL}" + sed -i "s,export RELEASE_BASE_URL.*,export RELEASE_BASE_URL=\"${RELEASE_BASE_URL}\"," ./export-env.sh + curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-operator.yaml -o tigera-operator.yaml + ${KCAPZ} create -f ./tigera-operator.yaml +fi + if [[ ${PRODUCT} == 'calient' ]]; then # Install prometheus operator - curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-prometheus-operator.yaml -o tigera-prometheus-operator.yaml - ${KCAPZ} create -f ./tigera-prometheus-operator.yaml + if [[ ${RELEASE_STREAM} == 'local' ]]; then + ${KCAPZ} create -f ${LOCAL_MANIFESTS_DIR}/tigera-prometheus-operator.yaml + else + curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-prometheus-operator.yaml -o tigera-prometheus-operator.yaml + ${KCAPZ} create -f ./tigera-prometheus-operator.yaml + fi # Install pull secret. ${KCAPZ} create secret generic tigera-pull-secret \ @@ -84,12 +102,6 @@ if [[ ${PRODUCT} == 'calient' ]]; then ${KCAPZ} patch deployment tigera-operator -n tigera-operator --patch '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"tigera-pull-secret"}]}}}}' fi - # Install prometheus operator pull secret. - ${KCAPZ} create secret generic tigera-pull-secret \ - --type=kubernetes.io/dockerconfigjson -n tigera-prometheus \ - --from-file=.dockerconfigjson=${GCR_IO_PULL_SECRET} - ${KCAPZ} patch deployment calico-prometheus-operator -n tigera-prometheus --patch '{"spec":{"template":{"spec":{"imagePullSecrets":[{"name":"tigera-pull-secret"}]}}}}' - # Create custom resources ${KCAPZ} create -f ./EE/custom-resources.yaml @@ -137,7 +149,7 @@ echo "Calico is ready on Windows nodes" # Create the kube-proxy-windows daemonset for iter in {1..5};do - curl -sSf -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/calico/kube-proxy/kube-proxy.yml | sed "s/KUBE_PROXY_VERSION/${KUBE_VERSION}/g" | ${KCAPZ} apply -f - && exit_code=0 && break || echo "download error: retry $iter in 5s" && sleep 5; + curl -sSf -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/calico/kube-proxy/kube-proxy.yml | sed "s/KUBE_PROXY_VERSION/${KUBE_VERSION}/g" | ${KCAPZ} apply -f - && break || echo "download error: retry $iter in 5s" && sleep 5; done; echo "Wait for kube-proxy to be ready on Windows nodes..." diff --git a/process/testing/winfv/capz/replace-win-containerd.ps1 b/process/testing/winfv-felix/capz/replace-win-containerd.ps1 similarity index 100% rename from process/testing/winfv/capz/replace-win-containerd.ps1 rename to process/testing/winfv-felix/capz/replace-win-containerd.ps1 diff --git a/process/testing/winfv/capz/replace-win-containerd.sh b/process/testing/winfv-felix/capz/replace-win-containerd.sh similarity index 100% rename from process/testing/winfv/capz/replace-win-containerd.sh rename to process/testing/winfv-felix/capz/replace-win-containerd.sh diff --git a/process/testing/winfv/capz/utils.sh b/process/testing/winfv-felix/capz/utils.sh similarity index 92% rename from process/testing/winfv/capz/utils.sh rename to process/testing/winfv-felix/capz/utils.sh index fc4fb477305..e0908f6e63f 100755 --- a/process/testing/winfv/capz/utils.sh +++ b/process/testing/winfv-felix/capz/utils.sh @@ -20,8 +20,8 @@ function retry_command() { local CMD=$2 echo - for i in `seq 1 $RETRY`; do - echo Trying $CMD, attempt ${i} + for i in $(seq 1 $RETRY); do + echo "Trying '$CMD', attempt ${i}" $CMD && return 0 || sleep 10 done echo "Command '${CMD}' failed after $RETRY attempts" diff --git a/process/testing/winfv/cloudformation-simple-vpc.json b/process/testing/winfv-felix/cloudformation-simple-vpc.json similarity index 100% rename from process/testing/winfv/cloudformation-simple-vpc.json rename to process/testing/winfv-felix/cloudformation-simple-vpc.json diff --git a/process/testing/winfv/cloudformation-windows-server.json b/process/testing/winfv-felix/cloudformation-windows-server.json similarity index 100% rename from process/testing/winfv/cloudformation-windows-server.json rename to process/testing/winfv-felix/cloudformation-windows-server.json diff --git a/process/testing/winfv/create_kubeadm_cluster.sh b/process/testing/winfv-felix/create_kubeadm_cluster.sh similarity index 100% rename from process/testing/winfv/create_kubeadm_cluster.sh rename to process/testing/winfv-felix/create_kubeadm_cluster.sh diff --git a/process/testing/winfv/infra/client.yaml b/process/testing/winfv-felix/infra/client.yaml similarity index 100% rename from process/testing/winfv/infra/client.yaml rename to process/testing/winfv-felix/infra/client.yaml diff --git a/process/testing/winfv/infra/ee/role.yaml b/process/testing/winfv-felix/infra/ee/role.yaml similarity index 100% rename from process/testing/winfv/infra/ee/role.yaml rename to process/testing/winfv-felix/infra/ee/role.yaml diff --git a/process/testing/winfv/infra/egress.yaml b/process/testing/winfv-felix/infra/egress.yaml similarity index 100% rename from process/testing/winfv/infra/egress.yaml rename to process/testing/winfv-felix/infra/egress.yaml diff --git a/process/testing/winfv/infra/ingress.yaml b/process/testing/winfv-felix/infra/ingress.yaml similarity index 100% rename from process/testing/winfv/infra/ingress.yaml rename to process/testing/winfv-felix/infra/ingress.yaml diff --git a/process/testing/winfv/infra/installation-bgp.yaml b/process/testing/winfv-felix/infra/installation-bgp.yaml similarity index 100% rename from process/testing/winfv/infra/installation-bgp.yaml rename to process/testing/winfv-felix/infra/installation-bgp.yaml diff --git a/process/testing/winfv/infra/installation-vxlan.yaml b/process/testing/winfv-felix/infra/installation-vxlan.yaml similarity index 100% rename from process/testing/winfv/infra/installation-vxlan.yaml rename to process/testing/winfv-felix/infra/installation-vxlan.yaml diff --git a/process/testing/winfv/infra/nginx.yaml b/process/testing/winfv-felix/infra/nginx.yaml similarity index 100% rename from process/testing/winfv/infra/nginx.yaml rename to process/testing/winfv-felix/infra/nginx.yaml diff --git a/process/testing/winfv/infra/porter.yaml b/process/testing/winfv-felix/infra/porter.yaml similarity index 100% rename from process/testing/winfv/infra/porter.yaml rename to process/testing/winfv-felix/infra/porter.yaml diff --git a/process/testing/winfv/infra/setup.sh b/process/testing/winfv-felix/infra/setup.sh similarity index 100% rename from process/testing/winfv/infra/setup.sh rename to process/testing/winfv-felix/infra/setup.sh diff --git a/process/testing/winfv/restart-felix.ps1 b/process/testing/winfv-felix/restart-felix.ps1 similarity index 100% rename from process/testing/winfv/restart-felix.ps1 rename to process/testing/winfv-felix/restart-felix.ps1 diff --git a/process/testing/winfv/run-fv-cni-plugin.ps1 b/process/testing/winfv-felix/run-fv-cni-plugin.ps1 similarity index 100% rename from process/testing/winfv/run-fv-cni-plugin.ps1 rename to process/testing/winfv-felix/run-fv-cni-plugin.ps1 diff --git a/process/testing/winfv/run-fv-full.ps1 b/process/testing/winfv-felix/run-fv-full.ps1 similarity index 96% rename from process/testing/winfv/run-fv-full.ps1 rename to process/testing/winfv-felix/run-fv-full.ps1 index 84d2236213a..8cec7985721 100644 --- a/process/testing/winfv/run-fv-full.ps1 +++ b/process/testing/winfv-felix/run-fv-full.ps1 @@ -10,6 +10,10 @@ Param( $Root="c:\\CalicoWindows" +if ($Provisioner -eq "capz") { + Set-Item -Path env:HPC -Value "true" +} + # Force powershell to run in 64-bit mode . if ([Environment]::Is64BitProcess -eq $false) { write-warning "This script requires PowerShell 64-bit, relaunching..." @@ -22,7 +26,7 @@ if ([Environment]::Is64BitProcess -eq $false) { } if ($Provisioner -ne "capz") { - # Install Calico for Windows + # Install Calico for Windows using the manual installation method Invoke-WebRequest https://docs.projectcalico.org/scripts/install-calico-windows.ps1 -OutFile c:\\install-calico-windows.ps1 c:\\install-calico-windows.ps1 -KubeVersion $KubeVersion diff --git a/process/testing/winfv/setup-fv-capz.sh b/process/testing/winfv-felix/setup-fv-capz.sh similarity index 65% rename from process/testing/winfv/setup-fv-capz.sh rename to process/testing/winfv-felix/setup-fv-capz.sh index 6aef156bad3..7f0f092457b 100644 --- a/process/testing/winfv/setup-fv-capz.sh +++ b/process/testing/winfv-felix/setup-fv-capz.sh @@ -28,15 +28,27 @@ set -o pipefail GIT_VERSION=$(git describe --tags --dirty --long --always --abbrev=12) CALICO_HOME=$(cd "$(dirname $0)"/../../../; pwd) -CAPZ_LOCATION=$CALICO_HOME/process/testing/winfv/capz -KUBECONFIG=$CALICO_HOME/process/testing/winfv/capz/kubeconfig +CAPZ_LOCATION=$CALICO_HOME/process/testing/winfv-felix/capz +KUBECONFIG=$CALICO_HOME/process/testing/winfv-felix/capz/kubeconfig +KUBECTL=$CAPZ_LOCATION/bin/kubectl +KCAPZ="${KUBECTL} --kubeconfig=${KUBECONFIG}" +REPORT_DIR=$CALICO_HOME/process/testing/winfv-felix/report +SSH_OUTPUT_FILE=$REPORT_DIR/ssh_output.log SEMAPHORE="${SEMAPHORE:="false"}" export RAND=$(tr -dc a-z0-9 output - $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'ctr --namespace k8s.io images import --base-name calico/cni-windows c:\calico-cni-plugin-windows.tar --all-platforms' > output + $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'ctr --namespace k8s.io images import --base-name calico/node-windows c:\calico-node-windows.tar --all-platforms' >> $SSH_OUTPUT_FILE + $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'ctr --namespace k8s.io images import --base-name calico/cni-windows c:\calico-cni-plugin-windows.tar --all-platforms' >> $SSH_OUTPUT_FILE } function upload_fv(){ if [[ $FV_TYPE == "cni-plugin" ]]; then - $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv/run-cni-fv.ps1 c:\\run-cni-fv.ps1 + $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 c:\\run-cni-fv.ps1 $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/cni-plugin/bin/windows/win-fv.exe c:\\k\\win-cni-fv.exe elif [[ $FV_TYPE == "calico-felix" ]]; then - $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv/run-felix-fv.ps1 c:\\run-felix-fv.ps1 + $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 c:\\run-felix-fv.ps1 $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/felix/fv/win-fv.exe c:\\k\\win-felix-fv.exe fi } @@ -115,8 +131,8 @@ function prepare_windows_images(){ function prepare_fv(){ if [[ $FV_TYPE == "cni-plugin" ]]; then make -C $CALICO_HOME/cni-plugin bin/windows/win-fv.exe - FV_RUN_CNI=$CALICO_HOME/process/testing/winfv/run-cni-fv.ps1 - cp $CALICO_HOME/process/testing/winfv/run-fv-cni-plugin.ps1 $FV_RUN_CNI + FV_RUN_CNI=$CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 + cp $CALICO_HOME/process/testing/winfv-cni-plugin/run-fv-cni-plugin.ps1 $FV_RUN_CNI sed -i "s??${KUBE_VERSION}?g" $FV_RUN_CNI sed -i "s??${LINUX_NODE}?g" $FV_RUN_CNI sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_CNI @@ -125,8 +141,8 @@ function prepare_fv(){ sed -i "s?win-fv.exe?win-cni-fv.exe?g" $FV_RUN_CNI elif [[ $FV_TYPE == "calico-felix" ]]; then make -C $CALICO_HOME/felix fv/win-fv.exe - FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv/run-felix-fv.ps1 - cp $CALICO_HOME/process/testing/winfv/run-fv-full.ps1 $FV_RUN_FELIX + FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 + cp $CALICO_HOME/process/testing/winfv-felix/run-fv-full.ps1 $FV_RUN_FELIX sed -i "s??${KUBE_VERSION}?g" $FV_RUN_FELIX sed -i "s??${LINUX_NODE}?g" $FV_RUN_FELIX sed -i "s??${WINDOWS_SERVER_VERSION}?g" $FV_RUN_FELIX @@ -143,8 +159,8 @@ function prepare_fv(){ function wait_for_nodes(){ #Wait for calico-node-windows daemon set to update sleep 30 - for i in `seq 1 30`; do - if [[ `kubectl get ds calico-node-windows -n calico-system --no-headers --kubeconfig $KUBECONFIG | awk -v OFS='\t\t' '{print $6}'` = "$WIN_NODE_COUNT" ]] ; then + for i in $(seq 1 30); do + if [[ $(${KCAPZ} get ds calico-node-windows -n calico-system --no-headers | awk -v OFS='\t\t' '{print $6}') = "$WIN_NODE_COUNT" ]] ; then echo "Calico Node Windows is ready" return fi @@ -157,19 +173,19 @@ function wait_for_nodes(){ function update_windows_node(){ upload_calico_images - kubectl annotate ds -n calico-system calico-node-windows unsupported.operator.tigera.io/ignore="true" --kubeconfig $KUBECONFIG - kubectl patch ds -n calico-system calico-node-windows --patch-file $CALICO_HOME/process/testing/winfv/calico-node-windows.yaml --kubeconfig $KUBECONFIG + ${KCAPZ} annotate ds -n calico-system calico-node-windows unsupported.operator.tigera.io/ignore="true" + ${KCAPZ} patch ds -n calico-system calico-node-windows --patch-file $CALICO_HOME/process/testing/winfv-felix/calico-node-windows.yaml } function start_test_infra(){ - $CALICO_HOME/process/testing/winfv/infra/setup.sh $KUBECONFIG + $CALICO_HOME/process/testing/winfv-felix/infra/setup.sh $KUBECONFIG #Wait for porter pod to be running on windows node - for i in `seq 1 30`; do - if [[ `kubectl -n demo get pods porter --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready --kubeconfig $KUBECONFIG | awk -v OFS='\t\t' '{print $4}'` = "true" ]] ; then - echo "Porter is ready" - return - fi + for i in $(seq 1 30); do + if [[ $(${KCAPZ} -n demo get pods porter --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | awk -v OFS='\t\t' '{print $4}') = "true" ]] ; then + echo "Porter is ready" + return + fi echo "Waiting for porter to be ready" sleep 30 done @@ -179,23 +195,19 @@ function start_test_infra(){ function run_windows_fv(){ if [[ $FV_TYPE == "cni-plugin" ]]; then - $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'c:\\run-cni-fv.ps1' > output + $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'c:\\run-cni-fv.ps1' >> $SSH_OUTPUT_FILE elif [[ $FV_TYPE == "calico-felix" ]]; then - $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'c:\\run-felix-fv.ps1' > output + $CAPZ_LOCATION/ssh-node.sh $WIN_NODE_IP 'c:\\run-felix-fv.ps1' >> $SSH_OUTPUT_FILE fi } function get_test_results(){ - $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report $CALICO_HOME/process/testing/winfv/ + $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report\\* $REPORT_DIR if [[ $SEMAPHORE == "false" ]]; then - cat $CALICO_HOME/process/testing/winfv/report/fv-test.log + cat $REPORT_DIR/fv-test.log fi } -function shutdown_cluster(){ - make -C $CAPZ_LOCATION delete-cluster -} - prepare_env start_cluster prepare_windows_images diff --git a/process/testing/winfv/setup-fv.sh b/process/testing/winfv-felix/setup-fv.sh similarity index 99% rename from process/testing/winfv/setup-fv.sh rename to process/testing/winfv-felix/setup-fv.sh index 1f57cb0ace7..105ca9d16ec 100755 --- a/process/testing/winfv/setup-fv.sh +++ b/process/testing/winfv-felix/setup-fv.sh @@ -313,7 +313,7 @@ function show_all_instances() { echo "Linux node: ubuntu@${LINUX_EIP}" echo "Windows node: ${WINDOWS_EIP}" - echo "For credentials and other info, attach into the running job and go to ~/calico/process/testing/winfv and `cat connect`" + echo "For credentials and other info, attach into the running job and go to ~/calico/process/testing/winfv-felix and `cat connect`" CONNECT_FILE="connect" echo