diff --git a/Tiltfile b/Tiltfile index aa9a4927a37..d5aa30d721a 100644 --- a/Tiltfile +++ b/Tiltfile @@ -23,8 +23,8 @@ settings = { "capi_version": "v1.8.5", "caaph_version": "v0.2.5", "cert_manager_version": "v1.16.1", - "kubernetes_version": "v1.28.3", - "aks_kubernetes_version": "v1.28.3", + "kubernetes_version": "v1.28.15", + "aks_kubernetes_version": "v1.28.15", "flatcar_version": "3374.2.1", "azure_location": "eastus", "control_plane_machine_count": "1", @@ -212,10 +212,10 @@ def capz(): yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default # add extra_args if they are defined - if settings.get("extra_args"): - azure_extra_args = settings.get("extra_args").get("azure") + if settings.get("container_args"): + capz_container_args = settings.get("container_args").get("capz-controller-manager") yaml_dict = decode_yaml_stream(yaml) - append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", azure_extra_args) + append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", capz_container_args) yaml = str(encode_yaml_stream(yaml_dict)) yaml = fixup_yaml_empty_arrays(yaml) @@ -317,9 +317,14 @@ def flavors(): for template in template_list: deploy_worker_templates(template, substitutions) + delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false" + + if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): + delete_all_workload_clusters += clear_aks_vnet_peerings() + local_resource( name = "delete-all-workload-clusters", - cmd = kubectl_cmd + " delete clusters --all --wait=false", + cmd = ["sh", "-ec", delete_all_workload_clusters], auto_init = False, trigger_mode = TRIGGER_MODE_MANUAL, labels = ["flavors"], @@ -382,10 +387,19 @@ def deploy_worker_templates(template, substitutions): yaml = shlex.quote(yaml) flavor_name = os.path.basename(flavor) - flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\"" + flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM')" + flavor_cmd += "; export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -" + flavor_cmd += "; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\"" # wait for kubeconfig to be available - flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; chmod 600 ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; echo \"Waiting for kubeconfig to be available\"" + flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig" + flavor_cmd += "; chmod 600 ./${CLUSTER_NAME}.kubeconfig" + flavor_cmd += "; echo \"Kubeconfig for $CLUSTER_NAME created and saved in the local\"" + flavor_cmd += "; echo \"Waiting for $CLUSTER_NAME API Server to be accessible\"" + flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; echo \"API Server of $CLUSTER_NAME is accessible\"" # copy the kubeadm configmap to the calico-system namespace. # This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace. @@ -393,6 +407,9 @@ def deploy_worker_templates(template, substitutions): flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done" flavor_cmd += "; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -" + if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): + flavor_cmd += peer_vnets() + flavor_cmd += get_addons(flavor_name) local_resource( @@ -454,6 +471,63 @@ def waitforsystem(): local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system") local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system") +def peer_vnets(): + # TODO: check for az cli to be installed in local + # wait for AKS VNet to be in the state created + peering_cmd = "; echo \"--------Peering VNETs--------\"" + peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180" + peering_cmd += "; export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)" + peering_cmd += "; echo \" 1/8 ${AKS_MGMT_VNET_NAME} found \"" + + # wait for workload VNet to be created + peering_cmd += "; az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180" + peering_cmd += "; export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)" + peering_cmd += "; echo \" 2/8 ${CLUSTER_NAME}-vnet found \"" + + # peer mgmt vnet + peering_cmd += "; az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none" + peering_cmd += "; az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}\"" + + # peer workload vnet + peering_cmd += "; az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none" + peering_cmd += "; az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet\"" + + # create private DNS zone + peering_cmd += "; az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none" + peering_cmd += "; az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 5/8 ${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}\"" + + # link private DNS Zone to workload vnet + peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none" + peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone\"" + + # link private DNS Zone to mgmt vnet + peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none" + peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone\"" + + # create private DNS zone record + # TODO: 10.0.0.100 should be customizable + peering_cmd += "; az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --record-set-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} --ipv4-address 10.0.0.100 --only-show-errors --output none" + peering_cmd += "; echo \" 8/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} private DNS zone record created\n\"" + + return peering_cmd + +def clear_aks_vnet_peerings(): + delete_peering_cmd = "; echo \"--------Clearing AKS MGMT VNETs Peerings--------\"" + delete_peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180" + delete_peering_cmd += "; echo \" ${AKS_MGMT_VNET_NAME} found \"" + + # List all peering names and store them in an array + delete_peering_cmd += "; PEERING_NAMES=$(az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --query \"[].name\" --output tsv)" + delete_peering_cmd += "; for PEERING_NAME in ${PEERING_NAMES[@]}; do echo \"Deleting peering: ${PEERING_NAME}\"; az network vnet peering delete --name ${PEERING_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME}; done" + delete_peering_cmd += "; echo \"All VNETs Peerings deleted in ${AKS_MGMT_VNET_NAME}\"" + + return delete_peering_cmd + ############################## # Actual work happens here ############################## diff --git a/scripts/aks-as-mgmt.sh b/scripts/aks-as-mgmt.sh index 1d97708c420..1fffb9a08bc 100755 --- a/scripts/aks-as-mgmt.sh +++ b/scripts/aks-as-mgmt.sh @@ -30,7 +30,7 @@ make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${AZWI##*/}" export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # management cluster name export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # resource group name export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}" -export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}" +export AKS_MGMT_KUBERNETES_VERSION="${AKS_MGMT_KUBERNETES_VERSION:-v1.30.2}" export AZURE_LOCATION="${AZURE_LOCATION:-westus2}" export AKS_NODE_VM_SIZE="${AKS_NODE_VM_SIZE:-"Standard_B2s"}" export AKS_NODE_COUNT="${AKS_NODE_COUNT:-1}" @@ -42,6 +42,13 @@ export AZWI_STORAGE_CONTAINER="\$web" export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}" export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}" export REGISTRY="${REGISTRY:-}" +export AKS_MGMT_VNET_NAME="${AKS_MGMT_VNET_NAME:-"aks-mgmt-vnet-${RANDOM_SUFFIX}"}" +export AKS_MGMT_VNET_CIDR="${AKS_MGMT_VNET_CIDR:-"20.255.0.0/16"}" +export AKS_MGMT_SERVICE_CIDR="${AKS_MGMT_SERVICE_CIDR:-"20.255.254.0/24"}" +export AKS_MGMT_DNS_SERVICE_IP="${AKS_MGMT_DNS_SERVICE_IP:-"20.255.254.100"}" +export AKS_MGMT_SUBNET_NAME="${AKS_MGMT_SUBNET_NAME:-"aks-mgmt-subnet-${RANDOM_SUFFIX}"}" +export AKS_MGMT_SUBNET_CIDR="${AKS_MGMT_SUBNET_CIDR:-"20.255.0.0/24"}" + export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}" export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}" @@ -63,7 +70,7 @@ main() { echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME" echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP" echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP" - echo "KUBERNETES_VERSION: $KUBERNETES_VERSION" + echo "AKS_MGMT_KUBERNETES_VERSION: $AKS_MGMT_KUBERNETES_VERSION" echo "AZURE_LOCATION: $AZURE_LOCATION" echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE" echo "AZURE_NODE_MACHINE_TYPE: $AZURE_NODE_MACHINE_TYPE" @@ -76,6 +83,12 @@ main() { echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH" echo "REGISTRY: $REGISTRY" echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX" + echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME" + echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR" + echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR" + echo "AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP" + echo "AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME" + echo "AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR" echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID" echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID" @@ -102,6 +115,16 @@ create_aks_cluster() { --location "${AZURE_LOCATION}" \ --output none --only-show-errors \ --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + + echo "creating vnet for the resource group ${AKS_RESOURCE_GROUP}" + az network vnet create \ + --resource-group "${AKS_RESOURCE_GROUP}"\ + --name "${AKS_MGMT_VNET_NAME}" \ + --address-prefix "${AKS_MGMT_VNET_CIDR}" \ + --subnet-name "${AKS_MGMT_SUBNET_NAME}" \ + --subnet-prefix "${AKS_MGMT_SUBNET_CIDR}" \ + --output none --only-show-errors \ + --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" fi aks_exists=$(az aks show --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" 2>&1 || true) # true because we want to continue if the command fails @@ -110,13 +133,16 @@ create_aks_cluster() { az aks create --name "${MGMT_CLUSTER_NAME}" \ --resource-group "${AKS_RESOURCE_GROUP}" \ --location "${AZURE_LOCATION}" \ - --kubernetes-version "${KUBERNETES_VERSION}" \ + --kubernetes-version "${AKS_MGMT_KUBERNETES_VERSION}" \ --node-count "${AKS_NODE_COUNT}" \ --node-vm-size "${AKS_NODE_VM_SIZE}" \ --node-resource-group "${AKS_NODE_RESOURCE_GROUP}" \ --vm-set-type VirtualMachineScaleSets \ --generate-ssh-keys \ --network-plugin azure \ + --vnet-subnet-id "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_RESOURCE_GROUP}/providers/Microsoft.Network/virtualNetworks/${AKS_MGMT_VNET_NAME}/subnets/${AKS_MGMT_SUBNET_NAME}" \ + --service-cidr "${AKS_MGMT_SERVICE_CIDR}" \ + --dns-service-ip "${AKS_MGMT_DNS_SERVICE_IP}" \ --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" \ --output none --only-show-errors; elif echo "$aks_exists" | grep -q "${MGMT_CLUSTER_NAME}"; then @@ -127,6 +153,7 @@ create_aks_cluster() { fi # check and save kubeconfig + echo -e "\n" echo "saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \ --file "${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" --only-show-errors @@ -179,15 +206,10 @@ create_aks_cluster() { set_env_varaibles(){ cat < tilt-settings-temp.yaml kustomize_substitutions: - MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}" AKS_RESOURCE_GROUP: "${AKS_RESOURCE_GROUP}" AKS_NODE_RESOURCE_GROUP: "${AKS_NODE_RESOURCE_GROUP}" - MGMT_CLUSTER_KUBECONFIG: "${MGMT_CLUSTER_KUBECONFIG}" - AKS_MI_CLIENT_ID: "${AKS_MI_CLIENT_ID}" - AKS_MI_OBJECT_ID: "${AKS_MI_OBJECT_ID}" - AKS_MI_RESOURCE_ID: "${AKS_MI_RESOURCE_ID}" - MANAGED_IDENTITY_NAME: "${MANAGED_IDENTITY_NAME}" - MANAGED_IDENTITY_RG: "${MANAGED_IDENTITY_RG}" + AKS_MGMT_VNET_NAME: "${AKS_MGMT_VNET_NAME}" + MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}" AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: "${AKS_MI_CLIENT_ID}" CI_RG: "${MANAGED_IDENTITY_RG}" USER_IDENTITY: "${MANAGED_IDENTITY_NAME}" diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 44a5d2b4455..ccca1e7e081 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 30.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 30.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 30.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 30.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -203,7 +211,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '30.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index ba29dfeb37a..b8b496f8264 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -24,7 +24,10 @@ metadata: namespace: default spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 40.2.0.0/16 identityRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity @@ -37,12 +40,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 40.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 40.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 40.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 40.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -198,7 +209,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '40.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index 7704dcfc083..c714b78233f 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 50.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 50.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 50.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -205,7 +213,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '50.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index fc890e60836..6234f6a70c8 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -42,6 +42,8 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -236,5 +238,5 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index fe5b94184d0..6cbfcb87aca 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -38,12 +38,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 60.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 60.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 60.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -199,7 +207,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 8415a8f31b7..c12598320f1 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 100.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 100.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 100.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 100.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -202,7 +210,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '100.5.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 84750e8fe3a..a316fc2eabd 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 80.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 80.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 80.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 80.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -238,7 +246,7 @@ spec: preKubeadmCommands: - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' | jq -r .compute.name)/g" /etc/kubeadm.yml - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '80.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index d0bb0e43a47..c186bf66b9c 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -38,6 +38,8 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -252,5 +254,5 @@ spec: cluster-dns: '[fd00::10]' name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 5cb94eeb512..a854fe20a02 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 90.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 90.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 90.0.1.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 90.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -211,5 +219,5 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '90.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 5663fb7403d..a2be5af9905 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -24,7 +24,10 @@ metadata: namespace: default spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 70.2.0.0/16 identityRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity @@ -32,6 +35,9 @@ spec: location: ${AZURE_LOCATION} networkSpec: apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 70.0.11.100 name: ${CLUSTER_NAME}-internal-lb type: Internal controlPlaneOutboundLB: @@ -39,11 +45,17 @@ spec: nodeOutboundLB: frontendIPsCount: 1 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 70.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 70.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 70.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -203,7 +215,9 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - echo '70.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index ecee4ae0eaf..10913a19488 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -39,12 +39,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -298,7 +306,7 @@ spec: - powershell C:/defender-exclude-calico.ps1 preKubeadmCommands: - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' - -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + -Value '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" users: - groups: Administrators name: capi diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a18b181f968..0426726c5a4 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} diff --git a/templates/flavors/README.md b/templates/flavors/README.md index db10f8d1b2f..fe1d53158d0 100644 --- a/templates/flavors/README.md +++ b/templates/flavors/README.md @@ -91,3 +91,27 @@ worker-templates: KUBERNETES_VERSION: v1.22.1 WORKER_MACHINE_COUNT: "1" ``` + +#### Tilt flavors and their CIDRs +- Below VNet CIDRs are opininated and can be changed as per the requirement. +- AKS cluster created as part of [`aks-as-mgmt.sh`](../../hack/aks-as-mgmt.sh) script uses the `20.255.0.0/16` as the VNet CIDR. So the templates should have non overlapping CIDRs to that of AKS mgmt cluster. + +| Flavor | VNet | Control Plane Subnet | Private IP | Node Subnet | +|-----------------------------------------------------|---------------|----------------------|----------------|----------------| +| [`default`](default/kustomization.yaml) | `10.0.0.0/8` | `10.0.0.0/16` | `10.0.0.100` | `10.1.0.0/16` | +| [`aad`](aad/kustomization.yaml) | `30.0.0.0/8` | `30.0.0.0/16` | `30.0.11.100` | `30.1.0.0/16` | +| [`azure-bastion`](azure-bastion/kustomization.yaml) | `40.0.0.0/8` | `40.0.0.0/16` | `40.0.11.100` | `40.1.0.0/16` | +| [`azure-cni-v1`](azure-cni-v1/kustomization.yaml) | `50.0.0.0/8` | `50.0.0.0/16` | `50.0.11.100` | `50.1.0.0/16` | +| [`edgezone`](edgezone/kustomization.yaml) | `60.0.0.0/8` | `60.0.0.0/16` | `60.0.11.100` | `60.1.0.0/16` | +| [`private`](private/kustomization.yaml) | `70.0.0.0/8` | `70.0.0.0/16` | `70.0.11.100` | `70.1.0.0/16` | +| [`dual-stack`](dual-stack/kustomization.yaml) | `10.0.0.0/8` | `10.0.0.0/16` | `10.0.11.100` | `10.1.0.0/16` | +| [`ipv6`](ipv6/kustomization.yaml) | `10.0.0.0/8` | `10.0.0.0/16` | `10.0.11.100` | `10.1.0.0/16` | +| [`flatcar`](flatcar/kustomization.yaml) | `80.0.0.0/8` | `80.0.0.0/16` | `80.0.11.100` | `80.1.0.0/16` | +| [`nvdia-gpu`](nvidia-gpu/kustomization.yaml) | `90.0.0.0/8` | `90.0.0.0/16` | `90.0.11.100` | `90.1.0.0/16` | +| [`windows`](windows/kustomization.yaml) | `10.0.0.0/8` | `10.0.0.0/16` | `10.0.11.100` | `10.1.0.0/16` | +| [`ephemeral`](ephemeral/kustomization.yaml) | `100.0.0.0/8` | `100.0.0.0/16` | `100.0.11.100` | `100.1.0.0/16` | + +Note: +- Dual-stack has not been updated with non-overlapping CIDR and uses the `10.0.0.0` based CIDR. +- IPv6 has not been updated with non-overlapping CIDR and uses the `10.0.0.0` based CIDR. +- Windows has not been updated with non-overlapping CIDR and uses the `10.0.0.0` based CIDR. diff --git a/templates/flavors/aad/kustomization.yaml b/templates/flavors/aad/kustomization.yaml index de60f8706b0..1881743a533 100644 --- a/templates/flavors/aad/kustomization.yaml +++ b/templates/flavors/aad/kustomization.yaml @@ -8,8 +8,43 @@ resources: patches: - path: patches/kubeadm-controlplane.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '30.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 30.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 30.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 30.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 30.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/azure-bastion/kustomization.yaml b/templates/flavors/azure-bastion/kustomization.yaml index f39b9e3e32c..7e9e2d0db70 100644 --- a/templates/flavors/azure-bastion/kustomization.yaml +++ b/templates/flavors/azure-bastion/kustomization.yaml @@ -8,8 +8,43 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml - path: patches/azure-cluster.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '40.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 40.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 40.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 40.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 40.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/azure-bastion/patches/azure-cluster.yaml b/templates/flavors/azure-bastion/patches/azure-cluster.yaml index 74d517df341..ab6d1efd426 100644 --- a/templates/flavors/azure-bastion/patches/azure-cluster.yaml +++ b/templates/flavors/azure-bastion/patches/azure-cluster.yaml @@ -5,4 +5,7 @@ metadata: name: ${CLUSTER_NAME} spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 40.2.0.0/16 diff --git a/templates/flavors/azure-cni-v1/kustomization.yaml b/templates/flavors/azure-cni-v1/kustomization.yaml index 43779e89117..668eeff1dd3 100644 --- a/templates/flavors/azure-cni-v1/kustomization.yaml +++ b/templates/flavors/azure-cni-v1/kustomization.yaml @@ -11,8 +11,43 @@ patches: - path: patches/azure-machine-template.yaml - path: patches/kubeadm-control-plane.yaml - path: patches/kubeadm-worker-node.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '50.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 50.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 50.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 50.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 50.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index eb0e4743560..333ecdd55d6 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -8,7 +8,7 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml sortOptions: diff --git a/templates/flavors/dual-stack/kustomization.yaml b/templates/flavors/dual-stack/kustomization.yaml index f8c00538723..cdd43c9cb8e 100644 --- a/templates/flavors/dual-stack/kustomization.yaml +++ b/templates/flavors/dual-stack/kustomization.yaml @@ -10,8 +10,15 @@ patches: - path: patches/kubeadm-controlplane.yaml - path: patches/controlplane-azuremachinetemplate.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml -- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands + value: [] + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts sortOptions: order: fifo diff --git a/templates/flavors/dual-stack/patches/dual-stack.yaml b/templates/flavors/dual-stack/patches/dual-stack.yaml index 95aca4fd28f..5613adb731d 100644 --- a/templates/flavors/dual-stack/patches/dual-stack.yaml +++ b/templates/flavors/dual-stack/patches/dual-stack.yaml @@ -24,6 +24,16 @@ metadata: name: ${CLUSTER_NAME} spec: networkSpec: + apiServerLB: + # We pre-create this public IP and the DNS name to use it in the + # worker node's /etc/hosts. + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + name: ${CLUSTER_NAME}-api-lb + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 vnet: cidrBlocks: - "10.0.0.0/8" diff --git a/templates/flavors/edgezone/kustomization.yaml b/templates/flavors/edgezone/kustomization.yaml index 5e44b5bc717..40af34f39e0 100644 --- a/templates/flavors/edgezone/kustomization.yaml +++ b/templates/flavors/edgezone/kustomization.yaml @@ -10,8 +10,43 @@ patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml - path: patches/azure-extendedlocation.yaml - path: patches/azure-remove-natgateway.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '60.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 60.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 60.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 60.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 60.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/ephemeral/kustomization.yaml b/templates/flavors/ephemeral/kustomization.yaml index b66c5670a19..bb48be69ac7 100644 --- a/templates/flavors/ephemeral/kustomization.yaml +++ b/templates/flavors/ephemeral/kustomization.yaml @@ -20,8 +20,43 @@ patches: kind: AzureMachineTemplate name: .*-control-plane version: v1beta1 -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '100.5.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 100.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 100.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 100.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 100.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/flatcar/kustomization.yaml b/templates/flavors/flatcar/kustomization.yaml index 826dc2b11c5..f24ba26ef0d 100644 --- a/templates/flavors/flatcar/kustomization.yaml +++ b/templates/flavors/flatcar/kustomization.yaml @@ -9,13 +9,38 @@ resources: patches: - path: patches/kubeadm-controlplane.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 80.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 80.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 80.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 80.1.0.0/16 - target: kind: KubeadmConfigTemplate patch: |- - op: add path: /spec/template/spec/preKubeadmCommands/- - value: echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + value: echo '80.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + sortOptions: order: fifo diff --git a/templates/flavors/ipv6/kustomization.yaml b/templates/flavors/ipv6/kustomization.yaml index d663897e134..72230596d8a 100644 --- a/templates/flavors/ipv6/kustomization.yaml +++ b/templates/flavors/ipv6/kustomization.yaml @@ -11,8 +11,15 @@ patches: - path: patches/ipv6.yaml - path: patches/kubeadm-controlplane.yaml - path: patches/controlplane-azuremachinetemplate.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml -- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands + value: [] + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts sortOptions: order: fifo diff --git a/templates/flavors/ipv6/patches/ipv6.yaml b/templates/flavors/ipv6/patches/ipv6.yaml index ea0eff7286d..8bd72e4ab3d 100644 --- a/templates/flavors/ipv6/patches/ipv6.yaml +++ b/templates/flavors/ipv6/patches/ipv6.yaml @@ -18,6 +18,16 @@ metadata: name: ${CLUSTER_NAME} spec: networkSpec: + apiServerLB: + # We pre-create this public IP and the DNS name to use it in the + # worker node's /etc/hosts. + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + name: ${CLUSTER_NAME}-api-lb + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 vnet: cidrBlocks: - "10.0.0.0/8" diff --git a/templates/flavors/nvidia-gpu/kustomization.yaml b/templates/flavors/nvidia-gpu/kustomization.yaml index f8ebc7a9157..ea0166c9b52 100644 --- a/templates/flavors/nvidia-gpu/kustomization.yaml +++ b/templates/flavors/nvidia-gpu/kustomization.yaml @@ -16,8 +16,43 @@ generatorOptions: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '90.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 90.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 90.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 90.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 90.0.1.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml index e8b87f0ff6b..82fce965117 100644 --- a/templates/flavors/private/kustomization.yaml +++ b/templates/flavors/private/kustomization.yaml @@ -11,6 +11,48 @@ patches: - path: patches/private-lb.yaml - path: patches/apiserver-host-dns.yaml - path: patches/azure-bastion.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml +- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '70.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 70.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 70.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 70.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 70.1.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: remove + path: /spec/networkSpec/apiServerLB/frontendIPs/0 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/private/patches/azure-bastion.yaml b/templates/flavors/private/patches/azure-bastion.yaml index 74d517df341..6d9c3e8773f 100644 --- a/templates/flavors/private/patches/azure-bastion.yaml +++ b/templates/flavors/private/patches/azure-bastion.yaml @@ -5,4 +5,7 @@ metadata: name: ${CLUSTER_NAME} spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 70.2.0.0/16 diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index 452f43e02c5..93bb3dd3e90 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -8,6 +8,30 @@ resources: patches: - path: ../base-windows-containerd/cluster.yaml - path: patches/kubeadm-config-template.yaml +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.1.0.0/16 sortOptions: order: fifo diff --git a/templates/flavors/windows/patches/kubeadm-config-template.yaml b/templates/flavors/windows/patches/kubeadm-config-template.yaml index 3509792752a..5bf33864cbf 100644 --- a/templates/flavors/windows/patches/kubeadm-config-template.yaml +++ b/templates/flavors/windows/patches/kubeadm-config-template.yaml @@ -9,4 +9,4 @@ spec: # so that worker nodes can access the API server using the internal IP. # 10.0.0.100 is the default IP that gets assigned to a internal load balancer. preKubeadmCommands: - - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" diff --git a/templates/internal-load-balancer/azure-cluster-frontend-ip.yaml b/templates/internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml similarity index 57% rename from templates/internal-load-balancer/azure-cluster-frontend-ip.yaml rename to templates/internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml index 45a584e4a02..92e854c12eb 100644 --- a/templates/internal-load-balancer/azure-cluster-frontend-ip.yaml +++ b/templates/internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml @@ -12,3 +12,17 @@ spec: publicIP: name: ${CLUSTER_NAME}-api-lb dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.0.100 + vnet: + cidrBlocks: + - 10.0.0.0/8 + subnets: + - name: control-plane-subnet + role: control-plane + cidrBlocks: + - 10.0.0.0/16 + - name: node-subnet + role: node + cidrBlocks: + - 10.1.0.0/16 diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 760210ba5a5..1793536c3b6 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -41,12 +41,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 50.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 50.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 50.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -212,7 +220,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '50.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 0a3d54c620f..0222980a42b 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -45,6 +45,14 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -448,6 +456,8 @@ spec: preKubeadmCommands: - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index cbf94be6a1b..8dc7193f684 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -43,6 +43,14 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -466,6 +474,8 @@ spec: preKubeadmCommands: - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index e9b090f4ca5..cb774dd745a 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -42,18 +42,26 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - - name: ${AZURE_CUSTOM_VNET_NAME}-controlplane-subnet + - cidrBlocks: + - 10.0.0.0/16 + name: ${AZURE_CUSTOM_VNET_NAME}-controlplane-subnet role: control-plane securityGroup: name: control-plane-nsg - - name: ${AZURE_CUSTOM_VNET_NAME}-node-subnet + - cidrBlocks: + - 10.1.0.0/16 + name: ${AZURE_CUSTOM_VNET_NAME}-node-subnet role: node routeTable: name: node-routetable securityGroup: name: node-nsg vnet: + cidrBlocks: + - 10.0.0.0/8 name: ${AZURE_CUSTOM_VNET_NAME} resourceGroup: ${AZURE_CUSTOM_VNET_RESOURCE_GROUP} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -216,7 +224,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index b5ca8f3689f..79611e9de86 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -47,6 +47,8 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -245,7 +247,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index ef1b007edf2..4d434189e88 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -45,12 +45,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 60.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 60.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 60.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -215,7 +223,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index bffae1d36f2..6029979fc95 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 80.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 80.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 80.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 80.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -246,7 +254,7 @@ spec: preKubeadmCommands: - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' | jq -r .compute.name)/g" /etc/kubeadm.yml - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '80.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 6cd89fc25f0..7ec1025836f 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -45,6 +45,8 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.11.100 subnets: - cidrBlocks: - 10.0.0.0/16 @@ -260,7 +262,7 @@ spec: cluster-dns: '[fd00::10]' name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 0b7fde5f1b1..59eee95b1de 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 90.0.11.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 90.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 90.0.1.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 90.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -219,7 +227,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '90.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: addons.cluster.x-k8s.io/v1alpha1 diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 4f6613665e1..425fe50f438 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -236,7 +236,9 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 5dc7f601945..333347a3a16 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} diff --git a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml index 4d01fd9ab39..798d0702dbe 100644 --- a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml @@ -28,6 +28,13 @@ patches: - path: ../prow-dual-stack/patches/cluster-label-calico-dual-stack.yaml - path: patches/machine-deployment.yaml - path: ../patches/windows-addons-disabled.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + sortOptions: order: fifo diff --git a/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml b/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml index 7298ad5aaab..24a94fd2c8d 100644 --- a/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-ipv6/kustomization.yaml @@ -28,6 +28,13 @@ patches: - path: ../prow-ipv6/patches/cluster-label-calico-ipv6.yaml - path: patches/machine-deployment.yaml - path: ../patches/windows-addons-disabled.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + sortOptions: order: fifo diff --git a/templates/test/ci/prow-custom-vnet/kustomization.yaml b/templates/test/ci/prow-custom-vnet/kustomization.yaml index e078480baf6..9fd3b612e63 100644 --- a/templates/test/ci/prow-custom-vnet/kustomization.yaml +++ b/templates/test/ci/prow-custom-vnet/kustomization.yaml @@ -17,6 +17,47 @@ patches: - path: ../patches/uami-control-plane.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.11.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.0.11.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.0.0.0/8 +- target: + kind: AzureCluster + patch: |- + - op: add + path: /spec/networkSpec/subnets/0/cidrBlocks + value: [] + - op: add + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.0.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: add + path: /spec/networkSpec/subnets/1/cidrBlocks + value: [] + - op: add + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.1.0.0/16 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/test/ci/prow-private/kustomization.yaml b/templates/test/ci/prow-private/kustomization.yaml index 059607b987f..b79ec24d2b4 100644 --- a/templates/test/ci/prow-private/kustomization.yaml +++ b/templates/test/ci/prow-private/kustomization.yaml @@ -20,6 +20,16 @@ patches: - path: ../patches/uami-control-plane.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml +# we need to add the default private IP to the hosts file of the worker nodes +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + configMapGenerator: - files: - resources=../../../addons/calico.yaml