Skip to content

Commit

Permalink
Static routing: support cks cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
@weizhouapache
weizhouapache committed Aug 30, 2024
1 parent 618fa03 commit 9362fb1
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 22 deletions.
30 changes: 22 additions & 8 deletions ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.network.RoutedIpv4Manager;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -264,6 +265,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne

@Inject
private UserVmService userVmService;
@Inject
RoutedIpv4Manager routedIpv4Manager;

private void logMessage(final Level logLevel, final String message, final Exception e) {
if (logLevel == Level.WARN) {
Expand Down Expand Up @@ -429,16 +432,20 @@ private void validateNetwork(Network network, int clusterTotalNodeCount) {
if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) {
throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid()));
}
if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) {
throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid()));
}
if (routedIpv4Manager.isRoutedNetwork(network)) {
logger.debug("No need to add firewall and port forwarding rules for ROUTED network. Assume the VMs are reachable.");
return;
}
Long vpcId = network.getVpcId();
if (vpcId == null && !networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) {
throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid()));
}
if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) {
throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid()));
}
if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) {
throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid()));
}
if (network.getVpcId() != null) {
validateVpcTier(network);
return;
Expand Down Expand Up @@ -592,7 +599,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes
if (ntwk != null) {
response.setNetworkId(ntwk.getUuid());
response.setAssociatedNetworkName(ntwk.getName());
if (ntwk.getGuestType() == Network.GuestType.Isolated) {
if (!isDirectAccess(ntwk)) {
List<IPAddressVO> ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true);
if (ipAddresses != null && ipAddresses.size() == 1) {
response.setIpAddress(ipAddresses.get(0).getAddress().addr());
Expand Down Expand Up @@ -829,8 +836,9 @@ private void validateManagedKubernetesClusterCreateParameters(final CreateKubern
if (network == null) {
throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s parameter", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, ApiConstants.NETWORK_ID));
}
if (Network.GuestType.Shared.equals(network.getGuestType())) {
throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString()));
if (isDirectAccess(network)) {
throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s network or %s network",
ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared, NetworkOffering.NetworkMode.ROUTED));
}
}

Expand All @@ -851,7 +859,8 @@ private Network getKubernetesClusterNetworkIfMissing(final String clusterName, f
} else {
throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid()));
}
} else if (Network.GuestType.Shared.equals(network.getGuestType())) {
}
if (isDirectAccess(network)) {
if (controlNodesCount > 1 && StringUtils.isEmpty(externalLoadBalancerIpAddress)) {
throw new InvalidParameterValueException(String.format("Multi-control nodes, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used",
network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS));
Expand Down Expand Up @@ -893,7 +902,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
List<KubernetesClusterDetailsVO> details = new ArrayList<>();
long kubernetesClusterId = kubernetesCluster.getId();

if ((network != null && Network.GuestType.Shared.equals(network.getGuestType())) || kubernetesCluster.getClusterType() == KubernetesCluster.ClusterType.ExternalManaged) {
if ((network != null && isDirectAccess(network)) || kubernetesCluster.getClusterType() == KubernetesCluster.ClusterType.ExternalManaged) {
addKubernetesClusterDetailIfIsNotEmpty(details, kubernetesClusterId, ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true);
}

Expand Down Expand Up @@ -2015,6 +2024,11 @@ private void createNetworkOfferingForKubernetes(String offeringName, String offe
}
}

@Override
public boolean isDirectAccess(Network network) {
return Network.GuestType.Shared.equals(network.getGuestType()) || routedIpv4Manager.isRoutedNetwork(network);
}

@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
_name = name;
Expand Down
3 changes: 3 additions & 0 deletions ...bernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;

import com.cloud.network.Network;
import com.cloud.utils.component.PluggableService;
import com.cloud.utils.exception.CloudRuntimeException;

Expand Down Expand Up @@ -122,4 +123,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable
boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd);

List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);

boolean isDirectAccess(Network network);
}
8 changes: 4 additions & 4 deletions ...c/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
import com.cloud.network.Network.GuestType;
import com.cloud.network.NetworkModel;
import com.cloud.network.NetworkService;
import com.cloud.network.dao.IPAddressDao;
Expand Down Expand Up @@ -429,13 +428,14 @@ protected Pair<String, Integer> getKubernetesClusterServerIpSshPort(UserVm contr
logger.warn(String.format("Network for Kubernetes cluster : %s cannot be found", kubernetesCluster.getName()));
return new Pair<>(null, port);
}
if (manager.isDirectAccess(network)) {
return getKubernetesClusterServerIpSshPortForSharedNetwork(controlVm);
}
if (network.getVpcId() != null) {
return getKubernetesClusterServerIpSshPortForVpcTier(network, acquireNewPublicIpForVpcTierIfNeeded);
}
if (Network.GuestType.Isolated.equals(network.getGuestType())) {
return getKubernetesClusterServerIpSshPortForIsolatedNetwork(network);
} else if (Network.GuestType.Shared.equals(network.getGuestType())) {
return getKubernetesClusterServerIpSshPortForSharedNetwork(controlVm);
}
logger.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster : %s", kubernetesCluster.getName()));
return new Pair<>(null, port);
Expand Down Expand Up @@ -654,7 +654,7 @@ protected boolean taintControlNodes() {
protected boolean deployProvider() {
Network network = networkDao.findById(kubernetesCluster.getNetworkId());
// Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
if (network.getGuestType() != GuestType.Isolated) {
if (manager.isDirectAccess(network)) {
logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
kubernetesCluster.getName()), null);
return true;
Expand Down
2 changes: 1 addition & 1 deletion .../main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ private void validateClusterVMsDestroyed() {

private void checkForRulesToDelete() throws ManagementServerException {
NetworkVO kubernetesClusterNetwork = networkDao.findById(kubernetesCluster.getNetworkId());
if (kubernetesClusterNetwork != null && kubernetesClusterNetwork.getGuestType() != Network.GuestType.Shared) {
if (kubernetesClusterNetwork != null && !manager.isDirectAccess(kubernetesClusterNetwork)) {
deleteKubernetesClusterNetworkRules();
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,9 @@ private void scaleKubernetesClusterVpcTierRules(final List<Long> clusterVMIds) t
* @throws ManagementServerException
*/
private void scaleKubernetesClusterNetworkRules(final List<Long> clusterVMIds) throws ManagementServerException {
if (!Network.GuestType.Isolated.equals(network.getGuestType())) {
if (manager.isDirectAccess(network)) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network or ROUTED network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
}
return;
}
Expand Down
10 changes: 5 additions & 5 deletions ...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ private UserVm createKubernetesControlNode(final Network network, String serverI
Pair<String, Map<Long, Network.IpAddresses>> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner);
String controlNodeIp = ipAddresses.first();
Map<Long, Network.IpAddresses> requestedIps = ipAddresses.second();
if (Network.GuestType.Shared.equals(network.getGuestType()) && StringUtils.isEmpty(serverIp)) {
if (StringUtils.isEmpty(serverIp) && manager.isDirectAccess(network)) {
serverIp = controlNodeIp;
}
Network.IpAddresses addrs = new Network.IpAddresses(controlNodeIp, null);
Expand Down Expand Up @@ -377,9 +377,9 @@ private Network startKubernetesClusterNetwork(final DeployDestination destinatio
}

protected void setupKubernetesClusterNetworkRules(Network network, List<UserVm> clusterVMs) throws ManagementServerException {
if (!Network.GuestType.Isolated.equals(network.getGuestType())) {
if (manager.isDirectAccess(network)) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network or ROUTED network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
}
return;
}
Expand Down Expand Up @@ -489,7 +489,7 @@ public boolean startKubernetesClusterOnCreate() {
}
publicIpAddress = publicIpSshPort.first();
if (StringUtils.isEmpty(publicIpAddress) &&
(Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getControlNodeCount() > 1)) { // Shared network, single-control node cluster won't have an IP yet
(!manager.isDirectAccess(network) || kubernetesCluster.getControlNodeCount() > 1)) { // Shared network, single-control node cluster won't have an IP yet
logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed);
}
// Allow account creating the kubernetes cluster to access systemVM template
Expand Down Expand Up @@ -534,7 +534,7 @@ public boolean startKubernetesClusterOnCreate() {
attachIsoKubernetesVMs(clusterVMs);
if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) {
String msg = String.format("Failed to setup Kubernetes cluster : %s is not in usable state as the system is unable to access control node VMs of the cluster", kubernetesCluster.getName());
if (kubernetesCluster.getControlNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) {
if (kubernetesCluster.getControlNodeCount() > 1 && manager.isDirectAccess(network)) {
msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d",
msg,
CLUSTER_NODES_DEFAULT_START_SSH_PORT,
Expand Down
10 changes: 8 additions & 2 deletions ui/src/views/compute/KubernetesServiceTab.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,12 @@
<status :text="text ? text : ''" displayText />
</template>
<template v-if="column.key === 'port'" :name="text" :record="record">
{{ cksSshStartingPort + index }}
<div v-if="network.type === 'Shared' || network.ip4routing">
{{ cksSshPortSharedNetwork }}
</div>
<div v-else>
{{ cksSshStartingPort + index }}
</div>
</template>
<template v-if="column.key === 'actions'">
<a-tooltip placement="bottom" >
Expand Down Expand Up @@ -214,6 +219,7 @@ export default {
publicIpAddress: null,
currentTab: 'details',
cksSshStartingPort: 2222,
cksSshPortSharedNetwork: 22,
annotations: []
}
},
Expand Down Expand Up @@ -409,7 +415,7 @@ export default {
},
async fetchPublicIpAddress () {
await this.fetchNetwork()
if (this.network && this.network.type === 'Shared') {
if (this.network && (this.network.type === 'Shared' || this.network.ip4routing)) {
this.publicIpAddress = null
return
}
Expand Down

0 comments on commit 9362fb1

Please sign in to comment.