CAPZ Win FV fixes

Add timeout and retry to command that waits for CAPZ deployments.

Move process/testing/winfv to process/testing/winfv-felix and adjust scripts accordingly.

Add RELEASE_STREAM=local option to use local manifests and make CI runs use it.

.semaphore/semaphore.yml.d/blocks/20-felix.yml

@@ -93,7 +93,7 @@
 
 - name: "Felix: Windows FV capz"
   run:
-    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
+    when: "false or change_in(['/*', '/api/', '/libcalico-go/', '/typha/', '/felix/', '/node', '/hack/test/certs/', '/process/testing/winfv-felix/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
   dependencies: ["Felix: Build Windows binaries"]
   task:
     secrets:

cni-plugin/.semaphore/cleanup.yml

@@ -35,7 +35,7 @@ blocks:
     - name: Clean up winfv aws resources
       commands:
       - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true
-      - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}-containerd" ./setup-fv.sh -q -u || true
+      - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}-containerd" ./setup-fv.sh -q -u || true
       - NAME_PREFIX="${CLUSTER_NAME}-docker" ./setup-fv.sh -q -u | true
       env_vars:
       - name: AWS_DEFAULT_REGION

cni-plugin/.semaphore/semaphore.yml

@@ -107,7 +107,7 @@ blocks:
             - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
             - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
             - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true
-            - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u
+            - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u
       env_vars:
         - name: SEMAPHORE_ARTIFACT_EXPIRY
           value: 2w
@@ -152,7 +152,7 @@ blocks:
             - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
             - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
             - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true
-            - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u
+            - cd ~/calico/process/testing/winfv-cni-plugin && NAME_PREFIX="${CLUSTER_NAME}" ./setup-fv.sh -q -u
       env_vars:
         - name: SEMAPHORE_ARTIFACT_EXPIRY
           value: 2w

felix/.semaphore/run-win-fv

@@ -16,7 +16,7 @@
 
 set -e
 
-FV_DIR="/home/semaphore/calico/process/testing/winfv"
+FV_DIR="/home/semaphore/calico/process/testing/winfv-felix"
 ERROR_CODE=0
 
 pushd ${FV_DIR}

felix/.semaphore/semaphore.yml

@@ -104,7 +104,7 @@ blocks:
           - artifact push job ${REPORT_DIR} --destination semaphore/test-results --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
           - artifact push job ${LOGS_DIR} --destination semaphore/logs --expire-in ${SEMAPHORE_ARTIFACT_EXPIRY} || true
           - aws ec2 delete-key-pair --key-name ${KEYPAIR_NAME} || true
-          - cd ~/calico/process/testing/winfv && NAME_PREFIX="${CLUSTER_NAME}" /bin/bash -x ./setup-fv.sh -q -u
+          - cd ~/calico/process/testing/winfv-felix && NAME_PREFIX="${CLUSTER_NAME}" /bin/bash -x ./setup-fv.sh -q -u
     env_vars:
       - name: SEMAPHORE_ARTIFACT_EXPIRY
         value: 2w

process/testing/winfv-cni-plugin/aso/utils.sh

@@ -19,8 +19,8 @@ function retry_command() {
   local CMD=$2
   echo
 
-  for i in `seq 1 $RETRY`; do
-    echo Trying $CMD, attempt ${i}
+  for i in $(seq 1 $RETRY); do
+    echo "Trying '$CMD', attempt ${i}"
     $CMD && return 0 || sleep 10
   done
   echo "Command '${CMD}' failed after $RETRY attempts"

process/testing/winfv/capz/README.md → process/testing/winfv-felix/capz/README.md

@@ -45,6 +45,8 @@ Optionally, define `PRODUCT`, `RELEASE_STREAM` and/or `HASH_RELEASE`:
 make install-calico PRODUCT=calient RELEASE_STREAM=master HASH_RELEASE=true
 ```
 
+(Use `RELEASE_STREAM=local` to use local manifests from the monorepo instead of pulling them)
+
 To access your cluster, run `kubectl --kubeconfig=./kubeconfig ...`
 
 ### Access Linux or Windows nodes

process/testing/winfv/capz/create-cluster.sh → process/testing/winfv-felix/capz/create-cluster.sh

@@ -121,7 +121,7 @@ ${CLUSTERCTL} generate cluster ${CLUSTER_NAME_CAPZ} \
 retry_command 600 "${KUBECTL} apply -f win-capz.yaml"
 
 # Wait for CAPZ deployments
-${KUBECTL} wait --for=condition=Available --timeout=5m -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure
+timeout --foreground 600 bash -c "while ! ${KUBECTL} wait --for=condition=Available --timeout=30s -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure; do sleep 5; done"
 
 # Wait for the kubeconfig to become available.
 timeout --foreground 600 bash -c "while ! ${KUBECTL} get secrets | grep ${CLUSTER_NAME_CAPZ}-kubeconfig; do sleep 5; done"

process/testing/winfv/capz/install-calico.sh → process/testing/winfv-felix/capz/install-calico.sh

@@ -39,6 +39,9 @@ if [ ${PRODUCT} == 'calient' ]; then
     : "${TSEE_TEST_LICENSE:?Environment variable empty or not defined.}"
 fi
 
+SCRIPT_CURRENT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )"
+LOCAL_MANIFESTS_DIR="${SCRIPT_CURRENT_DIR}/../../../../manifests"
+
 if [ ${PRODUCT} == 'calient' ]; then
     RELEASE_BASE_URL="https://downloads.tigera.io/ee/${RELEASE_STREAM}"
 else
@@ -58,21 +61,32 @@ if [ ${HASH_RELEASE} == 'true' ]; then
     RELEASE_BASE_URL=$(curl -sS ${URL_HASH})
 fi
 
-# Check release url and installation scripts
-echo "Set release base url ${RELEASE_BASE_URL}"
-sed -i "s,export RELEASE_BASE_URL.*,export RELEASE_BASE_URL=\"${RELEASE_BASE_URL}\"," ./export-env.sh
 # Create a storage class and persistent volume for Calico Enterprise.
 if [ ${PRODUCT} == 'calient' ]; then
     ${KCAPZ} create -f ./EE/storage-class-azure-file.yaml
     ${KCAPZ} create -f ./EE/persistent-volume.yaml
 fi
+
 # Install Calico on Linux nodes
-curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-operator.yaml -o tigera-operator.yaml
-${KCAPZ} create -f ./tigera-operator.yaml
+if [[ ${RELEASE_STREAM} == 'local' ]]; then
+    # Use local manifests
+    ${KCAPZ} create -f ${LOCAL_MANIFESTS_DIR}/tigera-operator.yaml
+else
+    # Use release url
+    echo "Set release base url ${RELEASE_BASE_URL}"
+    sed -i "s,export RELEASE_BASE_URL.*,export RELEASE_BASE_URL=\"${RELEASE_BASE_URL}\"," ./export-env.sh
+    curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-operator.yaml -o tigera-operator.yaml
+    ${KCAPZ} create -f ./tigera-operator.yaml
+fi
+
 if [[ ${PRODUCT} == 'calient' ]]; then
     # Install prometheus operator
-    curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-prometheus-operator.yaml -o tigera-prometheus-operator.yaml
-    ${KCAPZ} create -f ./tigera-prometheus-operator.yaml
+    if [[ ${RELEASE_STREAM} == 'local' ]]; then
+        ${KCAPZ} create -f ${LOCAL_MANIFESTS_DIR}/tigera-prometheus-operator.yaml
+    else
+        curl -sSf -L --retry 5 ${RELEASE_BASE_URL}/manifests/tigera-prometheus-operator.yaml -o tigera-prometheus-operator.yaml
+        ${KCAPZ} create -f ./tigera-prometheus-operator.yaml
+    fi
 
     # Install pull secret.
     ${KCAPZ} create secret generic tigera-pull-secret \
@@ -137,7 +151,7 @@ echo "Calico is ready on Windows nodes"
 
 # Create the kube-proxy-windows daemonset
 for iter in {1..5};do
-    curl -sSf -L  https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/calico/kube-proxy/kube-proxy.yml | sed "s/KUBE_PROXY_VERSION/${KUBE_VERSION}/g" | ${KCAPZ} apply -f - && exit_code=0 && break || echo "download error: retry $iter in 5s" && sleep 5;
+    curl -sSf -L  https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/calico/kube-proxy/kube-proxy.yml | sed "s/KUBE_PROXY_VERSION/${KUBE_VERSION}/g" | ${KCAPZ} apply -f - && break || echo "download error: retry $iter in 5s" && sleep 5;
 done;
 
 echo "Wait for kube-proxy to be ready on Windows nodes..."

process/testing/winfv/setup-fv-capz.sh → process/testing/winfv-felix/setup-fv-capz.sh

@@ -28,8 +28,8 @@ set -o pipefail
 
 GIT_VERSION=$(git describe --tags --dirty --long --always --abbrev=12)
 CALICO_HOME=$(cd "$(dirname $0)"/../../../; pwd)
-CAPZ_LOCATION=$CALICO_HOME/process/testing/winfv/capz
-KUBECONFIG=$CALICO_HOME/process/testing/winfv/capz/kubeconfig
+CAPZ_LOCATION=$CALICO_HOME/process/testing/winfv-felix/capz
+KUBECONFIG=$CALICO_HOME/process/testing/winfv-felix/capz/kubeconfig
 SEMAPHORE="${SEMAPHORE:="false"}"
 export RAND=$(tr -dc a-z0-9 </dev/urandom | head -c 4; echo)
 export WIN_NODE_COUNT=1
@@ -55,7 +55,7 @@ function prepare_env(){
     export CLUSTER_NAME_CAPZ="${USER}-capz-win-felix-${SEMAPHORE_WORKFLOW_ID:0:8}"
   fi
 
-  . $CALICO_HOME/process/testing/winfv/capz/export-env.sh
+  . $CALICO_HOME/process/testing/winfv-felix/capz/export-env.sh
 }
 
 function start_cluster(){
@@ -68,7 +68,7 @@ function start_cluster(){
     fi
     # Use EXIT_CODE to bypass errexit and capture more information about a possible failure here
     EXIT_CODE=0
-    make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=master HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$?
+    make -C $CAPZ_LOCATION install-calico RELEASE_STREAM=local HASH_RELEASE=true PRODUCT=calico || EXIT_CODE=$?
     if [[ $EXIT_CODE -ne 0 ]]; then
         echo "failed to install Calico"
         ${KCAPZ} describe tigerastatus
@@ -91,10 +91,10 @@ function upload_calico_images(){
 
 function upload_fv(){
   if [[ $FV_TYPE == "cni-plugin" ]]; then
-    $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv/run-cni-fv.ps1 c:\\run-cni-fv.ps1
+    $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1 c:\\run-cni-fv.ps1
     $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/cni-plugin/bin/windows/win-fv.exe c:\\k\\win-cni-fv.exe
   elif [[ $FV_TYPE == "calico-felix" ]]; then
-    $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv/run-felix-fv.ps1 c:\\run-felix-fv.ps1
+    $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1 c:\\run-felix-fv.ps1
     $CAPZ_LOCATION/scp-to-node.sh $WIN_NODE_IP $CALICO_HOME/felix/fv/win-fv.exe c:\\k\\win-felix-fv.exe
   fi
 }
@@ -115,8 +115,8 @@ function prepare_windows_images(){
 function prepare_fv(){
   if [[ $FV_TYPE == "cni-plugin" ]]; then
     make -C $CALICO_HOME/cni-plugin bin/windows/win-fv.exe
-    FV_RUN_CNI=$CALICO_HOME/process/testing/winfv/run-cni-fv.ps1
-    cp $CALICO_HOME/process/testing/winfv/run-fv-cni-plugin.ps1 $FV_RUN_CNI
+    FV_RUN_CNI=$CALICO_HOME/process/testing/winfv-cni-plugin/run-cni-fv.ps1
+    cp $CALICO_HOME/process/testing/winfv-cni-plugin/run-fv-cni-plugin.ps1 $FV_RUN_CNI
     sed -i "s?<your kube version>?${KUBE_VERSION}?g" $FV_RUN_CNI
     sed -i "s?<your linux pip>?${LINUX_NODE}?g" $FV_RUN_CNI
     sed -i "s?<your os version>?${WINDOWS_SERVER_VERSION}?g" $FV_RUN_CNI
@@ -125,8 +125,8 @@ function prepare_fv(){
     sed -i "s?win-fv.exe?win-cni-fv.exe?g" $FV_RUN_CNI
   elif [[ $FV_TYPE == "calico-felix" ]]; then
     make -C $CALICO_HOME/felix fv/win-fv.exe
-    FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv/run-felix-fv.ps1
-    cp $CALICO_HOME/process/testing/winfv/run-fv-full.ps1 $FV_RUN_FELIX
+    FV_RUN_FELIX=$CALICO_HOME/process/testing/winfv-felix/run-felix-fv.ps1
+    cp $CALICO_HOME/process/testing/winfv-felix/run-fv-full.ps1 $FV_RUN_FELIX
     sed -i "s?<your kube version>?${KUBE_VERSION}?g" $FV_RUN_FELIX
     sed -i "s?<your linux pip>?${LINUX_NODE}?g" $FV_RUN_FELIX
     sed -i "s?<your os version>?${WINDOWS_SERVER_VERSION}?g" $FV_RUN_FELIX
@@ -158,11 +158,11 @@ function wait_for_nodes(){
 function update_windows_node(){
   upload_calico_images
   kubectl annotate ds -n calico-system calico-node-windows unsupported.operator.tigera.io/ignore="true" --kubeconfig $KUBECONFIG
-  kubectl patch ds -n calico-system calico-node-windows --patch-file $CALICO_HOME/process/testing/winfv/calico-node-windows.yaml  --kubeconfig $KUBECONFIG
+  kubectl patch ds -n calico-system calico-node-windows --patch-file $CALICO_HOME/process/testing/winfv-felix/calico-node-windows.yaml  --kubeconfig $KUBECONFIG
 }
 
 function start_test_infra(){
-  $CALICO_HOME/process/testing/winfv/infra/setup.sh $KUBECONFIG
+  $CALICO_HOME/process/testing/winfv-felix/infra/setup.sh $KUBECONFIG
 
   #Wait for porter pod to be running on windows node
   for i in `seq 1 30`; do
@@ -186,9 +186,9 @@ function run_windows_fv(){
 }
 
 function get_test_results(){
-  $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report $CALICO_HOME/process/testing/winfv/
+  $CAPZ_LOCATION/scp-from-node.sh $WIN_NODE_IP c:\\k\\report $CALICO_HOME/process/testing/winfv-felix/
   if [[ $SEMAPHORE == "false" ]]; then
-    cat $CALICO_HOME/process/testing/winfv/report/fv-test.log
+    cat $CALICO_HOME/process/testing/winfv-felix/report/fv-test.log
   fi
 }
 

process/testing/winfv/setup-fv.sh → process/testing/winfv-felix/setup-fv.sh

@@ -313,7 +313,7 @@ function show_all_instances() {
 
   echo "Linux node: ubuntu@${LINUX_EIP}"
   echo "Windows node: ${WINDOWS_EIP}"
-  echo "For credentials and other info, attach into the running job and go to ~/calico/process/testing/winfv and `cat connect`"
+  echo "For credentials and other info, attach into the running job and go to ~/calico/process/testing/winfv-felix and `cat connect`"
 
   CONNECT_FILE="connect"
   echo