From 9362fb19edd24f47addae0b1c653ee2fa34ec103 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 30 Aug 2024 12:57:07 +0200 Subject: [PATCH] Static routing: support cks cluster --- .../cluster/KubernetesClusterManagerImpl.java | 30 ++++++++++++++----- .../cluster/KubernetesClusterService.java | 3 ++ .../KubernetesClusterActionWorker.java | 8 ++--- .../KubernetesClusterDestroyWorker.java | 2 +- .../KubernetesClusterScaleWorker.java | 4 +-- .../KubernetesClusterStartWorker.java | 10 +++---- ui/src/views/compute/KubernetesServiceTab.vue | 10 +++++-- 7 files changed, 45 insertions(+), 22 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index ce92924a53a3..39e4bbf935fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -71,6 +71,7 @@ import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -264,6 +265,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject private UserVmService userVmService; + @Inject + RoutedIpv4Manager routedIpv4Manager; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -429,6 +432,13 @@ private void validateNetwork(Network network, int clusterTotalNodeCount) { if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); + } + if (routedIpv4Manager.isRoutedNetwork(network)) { + logger.debug("No need to add firewall and port forwarding rules for ROUTED network. Assume the VMs are reachable."); + return; + } Long vpcId = network.getVpcId(); if (vpcId == null && !networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); @@ -436,9 +446,6 @@ private void validateNetwork(Network network, int clusterTotalNodeCount) { if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); - } if (network.getVpcId() != null) { validateVpcTier(network); return; @@ -592,7 +599,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes if (ntwk != null) { response.setNetworkId(ntwk.getUuid()); response.setAssociatedNetworkName(ntwk.getName()); - if (ntwk.getGuestType() == Network.GuestType.Isolated) { + if (!isDirectAccess(ntwk)) { List ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true); if (ipAddresses != null && ipAddresses.size() == 1) { response.setIpAddress(ipAddresses.get(0).getAddress().addr()); @@ -829,8 +836,9 @@ private void validateManagedKubernetesClusterCreateParameters(final CreateKubern if (network == null) { throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s parameter", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, ApiConstants.NETWORK_ID)); } - if (Network.GuestType.Shared.equals(network.getGuestType())) { - throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); + if (isDirectAccess(network)) { + throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s network or %s network", + ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared, NetworkOffering.NetworkMode.ROUTED)); } } @@ -851,7 +859,8 @@ private Network getKubernetesClusterNetworkIfMissing(final String clusterName, f } else { throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); } - } else if (Network.GuestType.Shared.equals(network.getGuestType())) { + } + if (isDirectAccess(network)) { if (controlNodesCount > 1 && StringUtils.isEmpty(externalLoadBalancerIpAddress)) { throw new InvalidParameterValueException(String.format("Multi-control nodes, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used", network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS)); @@ -893,7 +902,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { List details = new ArrayList<>(); long kubernetesClusterId = kubernetesCluster.getId(); - if ((network != null && Network.GuestType.Shared.equals(network.getGuestType())) || kubernetesCluster.getClusterType() == KubernetesCluster.ClusterType.ExternalManaged) { + if ((network != null && isDirectAccess(network)) || kubernetesCluster.getClusterType() == KubernetesCluster.ClusterType.ExternalManaged) { addKubernetesClusterDetailIfIsNotEmpty(details, kubernetesClusterId, ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true); } @@ -2015,6 +2024,11 @@ private void createNetworkOfferingForKubernetes(String offeringName, String offe } } + @Override + public boolean isDirectAccess(Network network) { + return Network.GuestType.Shared.equals(network.getGuestType()) || routedIpv4Manager.isRoutedNetwork(network); + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { _name = name; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 6acc876493ef..9d86c564de48 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.network.Network; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; @@ -122,4 +123,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd); + + boolean isDirectAccess(Network network); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 0ef3ee96d328..2d993ac3cb6f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -65,7 +65,6 @@ import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; -import com.cloud.network.Network.GuestType; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; import com.cloud.network.dao.IPAddressDao; @@ -429,13 +428,14 @@ protected Pair getKubernetesClusterServerIpSshPort(UserVm contr logger.warn(String.format("Network for Kubernetes cluster : %s cannot be found", kubernetesCluster.getName())); return new Pair<>(null, port); } + if (manager.isDirectAccess(network)) { + return getKubernetesClusterServerIpSshPortForSharedNetwork(controlVm); + } if (network.getVpcId() != null) { return getKubernetesClusterServerIpSshPortForVpcTier(network, acquireNewPublicIpForVpcTierIfNeeded); } if (Network.GuestType.Isolated.equals(network.getGuestType())) { return getKubernetesClusterServerIpSshPortForIsolatedNetwork(network); - } else if (Network.GuestType.Shared.equals(network.getGuestType())) { - return getKubernetesClusterServerIpSshPortForSharedNetwork(controlVm); } logger.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster : %s", kubernetesCluster.getName())); return new Pair<>(null, port); @@ -654,7 +654,7 @@ protected boolean taintControlNodes() { protected boolean deployProvider() { Network network = networkDao.findById(kubernetesCluster.getNetworkId()); // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it - if (network.getGuestType() != GuestType.Isolated) { + if (manager.isDirectAccess(network)) { logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network", kubernetesCluster.getName()), null); return true; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 047d32f1f9fa..50d7fb14085a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -223,7 +223,7 @@ private void validateClusterVMsDestroyed() { private void checkForRulesToDelete() throws ManagementServerException { NetworkVO kubernetesClusterNetwork = networkDao.findById(kubernetesCluster.getNetworkId()); - if (kubernetesClusterNetwork != null && kubernetesClusterNetwork.getGuestType() != Network.GuestType.Shared) { + if (kubernetesClusterNetwork != null && !manager.isDirectAccess(kubernetesClusterNetwork)) { deleteKubernetesClusterNetworkRules(); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index f5ac553733d6..aa500de91902 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -163,9 +163,9 @@ private void scaleKubernetesClusterVpcTierRules(final List clusterVMIds) t * @throws ManagementServerException */ private void scaleKubernetesClusterNetworkRules(final List clusterVMIds) throws ManagementServerException { - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (manager.isDirectAccess(network)) { if (logger.isDebugEnabled()) { - logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); + logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network or ROUTED network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); } return; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index b2fbbd31f6ba..6f345d00644e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -189,7 +189,7 @@ private UserVm createKubernetesControlNode(final Network network, String serverI Pair> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner); String controlNodeIp = ipAddresses.first(); Map requestedIps = ipAddresses.second(); - if (Network.GuestType.Shared.equals(network.getGuestType()) && StringUtils.isEmpty(serverIp)) { + if (StringUtils.isEmpty(serverIp) && manager.isDirectAccess(network)) { serverIp = controlNodeIp; } Network.IpAddresses addrs = new Network.IpAddresses(controlNodeIp, null); @@ -377,9 +377,9 @@ private Network startKubernetesClusterNetwork(final DeployDestination destinatio } protected void setupKubernetesClusterNetworkRules(Network network, List clusterVMs) throws ManagementServerException { - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (manager.isDirectAccess(network)) { if (logger.isDebugEnabled()) { - logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); + logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network or ROUTED network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); } return; } @@ -489,7 +489,7 @@ public boolean startKubernetesClusterOnCreate() { } publicIpAddress = publicIpSshPort.first(); if (StringUtils.isEmpty(publicIpAddress) && - (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getControlNodeCount() > 1)) { // Shared network, single-control node cluster won't have an IP yet + (!manager.isDirectAccess(network) || kubernetesCluster.getControlNodeCount() > 1)) { // Shared network, single-control node cluster won't have an IP yet logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } // Allow account creating the kubernetes cluster to access systemVM template @@ -534,7 +534,7 @@ public boolean startKubernetesClusterOnCreate() { attachIsoKubernetesVMs(clusterVMs); if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) { String msg = String.format("Failed to setup Kubernetes cluster : %s is not in usable state as the system is unable to access control node VMs of the cluster", kubernetesCluster.getName()); - if (kubernetesCluster.getControlNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { + if (kubernetesCluster.getControlNodeCount() > 1 && manager.isDirectAccess(network)) { msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", msg, CLUSTER_NODES_DEFAULT_START_SSH_PORT, diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index c2e8891b9a0a..d6f6880621b3 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -114,7 +114,12 @@