Add a new Github workflow to validate FlexibleIPAM (#6879)

The workflow is using the runners provided by the CNCF through
the Github Enterprise plan (`ubuntu-latest-4-cores`).

Signed-off-by: KMAnju-2021 <[email protected]>

Author: KMAnju-2021

.github/workflows/kind.yml

@@ -277,6 +277,69 @@ jobs:
           path: log.tar.gz
           retention-days: 30
 
+  test-e2e-ipam-feature-enabled:
+    name: E2e tests on a Kind cluster on Linux with FlexibleIPAM feature enabled
+    needs: [build-antrea-coverage-image]
+    runs-on: [ubuntu-latest-4-cores]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - name: Download Antrea image from previous job
+        uses: actions/download-artifact@v4
+        with:
+          name: antrea-ubuntu-cov
+      - name: Load Antrea image
+        run: |
+          docker load -i antrea-ubuntu.tar
+      - name: Install Kind
+        run: |
+          KIND_VERSION=$(head -n1 ./ci/kind/version)
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+      - name: Run ipam e2e tests
+        # We enable multicast as some FlexibleIPAM e2e tests require it
+        run: |
+          mkdir log
+          mkdir test-ipam-e2e-coverage
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-ipam-e2e-coverage ./ci/kind/test-e2e-kind.sh \
+           --encap-mode noEncap \
+           --coverage \
+           --multicast \
+           --flexible-ipam \
+      - name: Tar coverage files
+        run: tar -czf test-ipam-e2e-coverage.tar.gz test-ipam-e2e-coverage
+      - name: Upload coverage for test-ipam-e2e-coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-ipam-e2e-coverage
+          path: test-ipam-e2e-coverage.tar.gz
+          retention-days: 30
+      - name: Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: '*.cov.out*'
+          disable_search: true
+          flags: kind-e2e-tests
+          name: test-ipam-e2e-coverage
+          directory: test-ipam-e2e-coverage
+          fail_ci_if_error: ${{ github.event_name == 'push' }}
+      - name: Tar log files
+        if: ${{ failure() }}
+        run: tar -czf log.tar.gz log
+      - name: Upload test log
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: e2e-kind-ipam-features-enabled.tar.gz
+          path: log.tar.gz
+          retention-days: 30
+
   test-e2e-noencap:
     name: E2e tests on a Kind cluster on Linux (noEncap)
     needs: [build-antrea-coverage-image]

ci/kind/kind-setup.sh

@@ -300,6 +300,22 @@ function configure_vlan_subnets {
       docker_run_with_host_net iptables -t filter -A FORWARD -i ${vlan_interfaces[j]} -o ${vlan_interfaces[i]} -j ACCEPT
     done
   done
+
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+    docker_run_with_host_net ipset create excluded_subnets hash:net
+    docker_run_with_host_net ipset add excluded_subnets 192.168.241.0/24
+    docker_run_with_host_net ipset add excluded_subnets 192.168.242.0/24
+    docker_run_with_host_net ipset add excluded_subnets 192.168.240.0/24
+    docker_run_with_host_net ipset list excluded_subnets
+    
+    # Bypass default Docker SNAT rule for FlexibleIPAM traffic from the untagged subnet (192.168.240.0/24, which is the subnet for the Docker bridge network)
+    # and destined to the VLAN subnets (192.168.241.0/24, 192.168.242.0/24).
+    docker_run_with_host_net iptables -t nat -I POSTROUTING 1 ! -o $bridge_interface -s 192.168.240.0/24 -m set --match-set excluded_subnets dst -j RETURN
+
+    # With FlexibleIPAM, Antrea SNAT is disabled (noSNAT: true) so Pods don't have access to the external network by default (including regular / NodeIPAM Pods).
+    # Our e2e tests require external network access for regular Pods, so we need to add a custom SNAT rule.
+    docker_run_with_host_net iptables -t nat -A POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_subnets dst -j MASQUERADE
+  fi
 }
 
 function delete_vlan_subnets {
@@ -318,17 +334,29 @@ function delete_vlan_subnets {
       docker_run_with_host_net ip link del $interface_name
     fi
   done
+
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+    docker_run_with_host_net iptables -t nat -D POSTROUTING ! -o $bridge_interface -s 192.168.240.0/24 -m set --match-set excluded_subnets dst -j RETURN || true
+    docker_run_with_host_net iptables -t nat -D POSTROUTING ! -o $bridge_interface -s 10.244.0.0/16 -m set ! --match-set excluded_subnets dst -j MASQUERADE || true
+    docker_run_with_host_net ipset destroy excluded_subnets || true  
+  fi
 }
 
-function delete_networks {
-  networks=$(docker network ls -f name=antrea --format '{{.Name}}')
-  networks="$(echo $networks)"
-  if [[ ! -z $networks ]]; then
+function delete_network_by_filter {
+  local networks=$(docker network ls -f name="$1" --format '{{.Name}}')
+  if [[ -n $networks ]]; then
     docker network rm $networks > /dev/null 2>&1
-    echo "deleted networks $networks"
+    echo "Deleted networks: $networks"
   fi
 }
 
+function delete_networks {
+  if [[ $FLEXIBLE_IPAM == true ]]; then
+    delete_network_by_filter "kind"
+  fi
+  delete_network_by_filter "antrea"
+}
+
 function load_images {
   echo "load images"
   set +e
@@ -711,7 +739,6 @@ if [[ $ACTION == "destroy" ]]; then
       exit
 fi
 
-
 kind_version=$(kind version | awk  '{print $2}')
 kind_version=${kind_version:1} # strip leading 'v'
 function version_lt() { test "$(printf '%s\n' "$@" | sort -rV | head -n 1)" != "$1"; }
@@ -728,5 +755,10 @@ if [[ $ACTION == "create" ]]; then
         echoerr "Only one of '--subnets' and '--extra-networks' can be specified"
         exit 1
     fi
+
+   # Reserve IPs after 192.168.240.63 for e2e tests.
+    if [[ $FLEXIBLE_IPAM == true ]]; then
+        docker network create -d bridge --subnet 192.168.240.0/24 --gateway 192.168.240.1 --ip-range 192.168.240.0/26 kind
+    fi
     create
 fi

ci/kind/test-e2e-kind.sh

@@ -89,6 +89,7 @@ setup_only=false
 cleanup_only=false
 test_only=false
 run=""
+flexible_ipam=false
 antrea_controller_image="antrea/antrea-controller-ubuntu"
 antrea_agent_image="antrea/antrea-agent-ubuntu"
 use_non_default_images=false
@@ -110,6 +111,10 @@ case $key in
     proxy_all=true
     shift
     ;;
+    --flexible-ipam)
+    flexible_ipam=true
+    shift
+    ;;
     --no-kube-proxy)
     no_kube_proxy=true
     shift
@@ -248,6 +253,9 @@ fi
 if $flow_visibility; then
     manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
 fi
+if $flexible_ipam; then
+    manifest_args="$manifest_args --flexible-ipam"
+fi
 
 COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \
                     "antrea/nginx:1.21.6-alpine" \
@@ -302,6 +310,10 @@ if $extra_vlan; then
   fi
 fi
 
+if $flexible_ipam; then
+   vlan_args="$vlan_args --vlan-subnets 11=192.168.241.1/24 --vlan-subnets 12=192.168.242.1/24" 
+fi
+
 function setup_cluster {
   args=$1
 
@@ -328,7 +340,9 @@ function setup_cluster {
   if $bgp_policy; then
     args="$args --deploy-external-frr"
   fi
-
+  if $flexible_ipam; then
+    args="$args --flexible-ipam"
+  fi
   echo "creating test bed with args $args"
   eval "timeout 600 $TESTBED_CMD create kind $args"
 }
@@ -401,6 +415,11 @@ function run_test {
     EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips"
   fi
 
+  if $flexible_ipam; then
+     EXTRA_ARGS="$EXTRA_ARGS --antrea-ipam"
+     timeout="100m"
+  fi
+ 
   go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS
 
   if $coverage; then

test/e2e/antreapolicy_test.go

@@ -5468,7 +5468,7 @@ func createPolicyForFQDNCacheMinTTL(t *testing.T, data *TestData, testFQDN strin
 
 	annp, err := data.CreateOrUpdateANNP(builder.Get())
 	require.NoError(t, err, "error while deploying Antrea policy")
-	require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second))
+	require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 30*time.Second))
 }
 
 // createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created.