Skip to content

Commit

Permalink
Merge pull request rancher#47420 from slickwarren/cwarren/v2.8/airgap…
Browse files Browse the repository at this point in the history 
…-more-k8s

[2.8] adding full k8s tests to airgap
  • Loading branch information
@slickwarren
slickwarren authored Oct 9, 2024
2 parents f34338f + 16f72cb commit d0cdc69
Show file tree
Hide file tree
Showing 6 changed files with 335 additions and 122 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ require (
github.com/google/gnostic-models v0.6.8
github.com/rancher/cis-operator v1.0.11
github.com/rancher/rancher/pkg/apis v0.0.0-20240613212755-3021cf92ff9f
github.com/rancher/shepherd v0.0.0-20240927124804-e41d69ef115f
github.com/rancher/shepherd v0.0.0-20241004184305-e4fd58dfd3dc
github.com/rancher/wrangler v1.1.1
go.qase.io/client v0.0.0-20231114201952-65195ec001fa
)
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1547,8 +1547,8 @@ github.com/rancher/remotedialer v0.4.0 h1:T9yC5bFMsZFVQ6rK0dNrRg6rRb6Zr/4vsig8S0
github.com/rancher/remotedialer v0.4.0/go.mod h1:Ys004RpJuTLSm+k4aYUCoFiOOad37ubYev3TkOFg/5w=
github.com/rancher/rke v1.5.14-rc.2 h1:fPahxavkGSSqPGnf8aViLVlb/IEaZIA2Lz/v8/nZVPw=
github.com/rancher/rke v1.5.14-rc.2/go.mod h1:/z9oyKqYpFwgRBV9rfLxqUdjydz/VMCTcjld4uUt7uM=
github.com/rancher/shepherd v0.0.0-20240927124804-e41d69ef115f h1:wCY0Wm/UZ2lhFZ8YhKueW215JLVKBNUchUMHEQVoAoc=
github.com/rancher/shepherd v0.0.0-20240927124804-e41d69ef115f/go.mod h1:1TXkmbjCxMEp8Rzzw+ToyrhJYUGDC0lw6uXLe3Ie+M4=
github.com/rancher/shepherd v0.0.0-20241004184305-e4fd58dfd3dc h1:gsJ6sLq/cw9eRh571IxN1OIJmUL+XuQegBs7upn7MLw=
github.com/rancher/shepherd v0.0.0-20241004184305-e4fd58dfd3dc/go.mod h1:1TXkmbjCxMEp8Rzzw+ToyrhJYUGDC0lw6uXLe3Ie+M4=
github.com/rancher/steve v0.0.0-20241004194751-1d54556fdd0b h1:KEKYG23BZ3NdtpCB5R2xtm+B9zAn2yd8gQlbqqxy404=
github.com/rancher/steve v0.0.0-20241004194751-1d54556fdd0b/go.mod h1:o4vLBzMTKbHHhIiAcbgOiaN3aK1vIjL6ZTgaGxQYpsY=
github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20240301001845-4eacc2dabbde h1:x5VZI/0TUx1MeZirh6e0OMAInhCmq6yRvD6897458Ng=
Expand Down
93 changes: 42 additions & 51 deletions tests/v2/actions/provisioning/creates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,23 +561,29 @@ func CreateProvisioningRKE1CustomCluster(client *rancher.Client, externalNodePro
// CreateProvisioningAirgapCustomCluster provisions a non-rke1 cluster using corral to gather its nodes, then runs verify checks
func CreateProvisioningAirgapCustomCluster(client *rancher.Client, clustersConfig *clusters.ClusterConfig, corralPackages *corral.Packages) (*v1.SteveAPIObject, error) {
setLogrusFormatter()
rolesPerNode := map[int32]string{}
quantityPerPool := []int32{}
rolesPerPool := []string{}
for _, pool := range clustersConfig.MachinePools {
var finalRoleCommand string
if pool.MachinePoolConfig.ControlPlane {
finalRoleCommand += " --controlplane"
}

if pool.MachinePoolConfig.Etcd {
finalRoleCommand += " --etcd"
}

if pool.MachinePoolConfig.Worker {
finalRoleCommand += " --worker"
}

if pool.MachinePoolConfig.Windows {
finalRoleCommand += " --windows"
}

rolesPerNode[pool.MachinePoolConfig.Quantity] = finalRoleCommand
quantityPerPool = append(quantityPerPool, pool.MachinePoolConfig.Quantity)
rolesPerPool = append(rolesPerPool, finalRoleCommand)

}

if clustersConfig.PSACT == string(provisioninginput.RancherBaseline) {
Expand Down Expand Up @@ -618,32 +624,25 @@ func CreateProvisioningAirgapCustomCluster(client *rancher.Client, clustersConfi
}

logrus.Infof("Register Custom Cluster Through Corral")
for quantity, roles := range rolesPerNode {
err = corral.UpdateCorralConfig("node_count", fmt.Sprint(quantity))
if err != nil {
return nil, err
}
corralsArgs := []corral.Args{}

for poolIndex, poolRole := range rolesPerPool {

regCmd := fmt.Sprintf("%s %s", token.InsecureNodeCommand, poolRole)

// environment variables must be escaped inside original registration command
temp := strings.Replace(token.InsecureNodeCommand, "\"", "\\\"", -1)
command := fmt.Sprintf("%s %s", temp, roles)
logrus.Infof("registration command is %s", command)
err = corral.UpdateCorralConfig("registration_command", command)
if err != nil {
return nil, err
}
regCmd = strings.Replace(regCmd, "\"", "\\\"", -1)

corralName := namegen.AppendRandomString(rke2k3sNodeCorralName)
_, err = corral.CreateCorral(
client.Session,
corralName,
corralPackages.CorralPackageImages[corralPackageAirgapCustomClusterName],
corralPackages.HasDebug,
corralPackages.HasCleanup,
)
if err != nil {
return nil, err
}
corralsArgs = append(corralsArgs, corral.Args{
Name: namegen.AppendRandomString(rke2k3sNodeCorralName),
PackageName: corralPackages.CorralPackageImages[corralPackageAirgapCustomClusterName],
Updates: map[string]string{"registration_command": regCmd, "node_count": fmt.Sprint(quantityPerPool[poolIndex])},
})
}

_, err = corral.CreateMultipleCorrals(client.Session, corralsArgs, corralPackages.HasDebug, corralPackages.HasCleanup)
if err != nil {
return nil, err
}

createdCluster, err := client.Steve.SteveType(stevetypes.Provisioning).ByID(namespace + "/" + clusterName)
Expand All @@ -654,7 +653,8 @@ func CreateProvisioningAirgapCustomCluster(client *rancher.Client, clustersConfi
func CreateProvisioningRKE1AirgapCustomCluster(client *rancher.Client, clustersConfig *clusters.ClusterConfig, corralPackages *corral.Packages) (*management.Cluster, error) {
setLogrusFormatter()
clusterName := namegen.AppendRandomString(rke1AirgapCustomCluster)
rolesPerNode := map[int64]string{}
quantityPerPool := []int32{}
rolesPerPool := []string{}
for _, pool := range clustersConfig.NodePools {
var finalRoleCommand string
if pool.NodeRoles.ControlPlane {
Expand All @@ -667,7 +667,8 @@ func CreateProvisioningRKE1AirgapCustomCluster(client *rancher.Client, clustersC
finalRoleCommand += " --worker"
}

rolesPerNode[pool.NodeRoles.Quantity] = finalRoleCommand
quantityPerPool = append(quantityPerPool, int32(pool.NodeRoles.Quantity))
rolesPerPool = append(rolesPerPool, finalRoleCommand)
}

if clustersConfig.PSACT == string(provisioninginput.RancherBaseline) {
Expand Down Expand Up @@ -698,35 +699,25 @@ func CreateProvisioningRKE1AirgapCustomCluster(client *rancher.Client, clustersC
return nil, err
}

logrus.Infof("Register Custom Cluster Through Corral")
for quantity, roles := range rolesPerNode {
err = corral.UpdateCorralConfig("node_count", fmt.Sprint(quantity))
if err != nil {
return nil, err
}
corralsArgs := []corral.Args{}

logrus.Infof("Register Custom Cluster Through Corral")
for poolIndex, poolRole := range rolesPerPool {
// environment variables must be escaped inside original registration command
temp := strings.Replace(token.NodeCommand, "\"", "\\\"", -1)
command := fmt.Sprintf("%s %s", temp, roles)
logrus.Infof("registration command is %s", command)
err = corral.UpdateCorralConfig("registration_command", command)
if err != nil {
return nil, err
}
escapedCommand := strings.Replace(token.NodeCommand, "\"", "\\\"", -1)

corralName := namegen.AppendRandomString(rke1NodeCorralName)
corralsArgs = append(corralsArgs, corral.Args{
Name: namegen.AppendRandomString(rke1NodeCorralName),
PackageName: corralPackages.CorralPackageImages[corralPackageAirgapCustomClusterName],
Updates: map[string]string{"registration_command": fmt.Sprintf("%s %s", escapedCommand, poolRole), "node_count": fmt.Sprint(quantityPerPool[poolIndex])},
})
}

_, err = corral.CreateCorral(
client.Session,
corralName,
corralPackages.CorralPackageImages[corralPackageAirgapCustomClusterName],
corralPackages.HasDebug,
corralPackages.HasCleanup,
)
if err != nil {
return nil, err
}
_, err = corral.CreateMultipleCorrals(client.Session, corralsArgs, corralPackages.HasDebug, corralPackages.HasCleanup)
if err != nil {
return nil, err
}

createdCluster, err := client.Management.Cluster.ByID(clusterResp.ID)
return createdCluster, err
}
Expand Down
119 changes: 97 additions & 22 deletions tests/v2/validation/provisioning/airgap/k3s_custom_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package airgap
import (
"testing"

apisV1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
"github.com/rancher/rancher/tests/v2/actions/clusters"
provisioning "github.com/rancher/rancher/tests/v2/actions/provisioning"
"github.com/rancher/rancher/tests/v2/actions/provisioninginput"
Expand All @@ -14,8 +15,14 @@ import (
"github.com/rancher/rancher/tests/v2/validation/provisioning/registries"
"github.com/rancher/shepherd/clients/corral"
"github.com/rancher/shepherd/clients/rancher"
management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"
steveV1 "github.com/rancher/shepherd/clients/rancher/v1"
extensionscluster "github.com/rancher/shepherd/extensions/clusters"
"github.com/rancher/shepherd/extensions/clusters/kubernetesversions"
"github.com/rancher/shepherd/extensions/users"
password "github.com/rancher/shepherd/extensions/users/passwordgenerator"
"github.com/rancher/shepherd/pkg/config"
namegen "github.com/rancher/shepherd/pkg/namegenerator"
"github.com/rancher/shepherd/pkg/session"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -51,7 +58,25 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() {
client, err := rancher.NewClient("", testSession)
require.NoError(a.T(), err)

a.client = client
var testuser = namegen.AppendRandomString("testuser-")
var testpassword = password.GenerateUserPassword("testpass-")
enabled := true

user := &management.User{
Username: testuser,
Password: testpassword,
Name: testuser,
Enabled: &enabled,
}

newUser, err := users.CreateUserWithRole(client, user, "user")
require.NoError(a.T(), err)

standardUserClient, err := client.AsUser(newUser)
require.NoError(a.T(), err)

a.client = standardUserClient

listOfCorrals, err := corral.ListCorral()
require.NoError(a.T(), err)

Expand Down Expand Up @@ -83,44 +108,94 @@ func (a *AirGapK3SCustomClusterTestSuite) SetupSuite() {
}

func (a *AirGapK3SCustomClusterTestSuite) TestProvisioningAirGapK3SCustomCluster() {
a.clustersConfig.MachinePools = []provisioninginput.MachinePools{provisioninginput.AllRolesMachinePool}
nodeRolesAll := []provisioninginput.MachinePools{provisioninginput.AllRolesMachinePool}
nodeRolesShared := []provisioninginput.MachinePools{provisioninginput.EtcdControlPlaneMachinePool, provisioninginput.WorkerMachinePool}
nodeRolesDedicated := []provisioninginput.MachinePools{provisioninginput.EtcdMachinePool, provisioninginput.ControlPlaneMachinePool, provisioninginput.WorkerMachinePool}

tests := []struct {
name string
client *rancher.Client
name string
client *rancher.Client
machinePool []provisioninginput.MachinePools
}{
{provisioninginput.AdminClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client},
{"1 Node All Roles " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesAll},
{"2 nodes - etcd|cp roles per 1 node " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesShared},
{"3 nodes - 1 role per node " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesDedicated},
}
for _, tt := range tests {

a.clustersConfig.MachinePools = tt.machinePool

if a.clustersConfig.K3SKubernetesVersions == nil {
k3sVersions, err := kubernetesversions.ListK3SAllVersions(a.client)
require.NoError(a.T(), err)

a.clustersConfig.K3SKubernetesVersions = k3sVersions
}

permutations.RunTestPermutations(&a.Suite, tt.name, tt.client, a.clustersConfig, permutations.K3SAirgapCluster, nil, a.corralPackage)
}

}

func (a *AirGapK3SCustomClusterTestSuite) TestProvisioningUpgradeAirGapK3SCustomCluster() {
a.clustersConfig.MachinePools = []provisioninginput.MachinePools{provisioninginput.AllRolesMachinePool}
nodeRolesAll := []provisioninginput.MachinePools{provisioninginput.AllRolesMachinePool}
nodeRolesShared := []provisioninginput.MachinePools{provisioninginput.EtcdControlPlaneMachinePool, provisioninginput.WorkerMachinePool}
nodeRolesDedicated := []provisioninginput.MachinePools{provisioninginput.EtcdMachinePool, provisioninginput.ControlPlaneMachinePool, provisioninginput.WorkerMachinePool}

k3sVersions, err := kubernetesversions.ListK3SAllVersions(a.client)
require.NoError(a.T(), err)
tests := []struct {
name string
client *rancher.Client
machinePool []provisioninginput.MachinePools
}{
{"1 Node All Roles " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesAll},
{"2 nodes - etcd|cp roles per 1 node " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesShared},
{"3 nodes - 1 role per node " + provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", a.client, nodeRolesDedicated},
}

numOfK3SVersions := len(k3sVersions)
require.Equal(a.T(), len(a.clustersConfig.CNIs), 1)
for _, tt := range tests {

a.clustersConfig.K3SKubernetesVersions[0] = k3sVersions[numOfK3SVersions-2]
testConfig := clusters.ConvertConfigToClusterConfig(a.clustersConfig)
testConfig.KubernetesVersion = a.clustersConfig.K3SKubernetesVersions[0]
testConfig.CNI = a.clustersConfig.CNIs[0]
clusterObject, err := provisioning.CreateProvisioningAirgapCustomCluster(a.client, testConfig, a.corralPackage)
reports.TimeoutClusterReport(clusterObject, err)
require.NoError(a.T(), err)
a.clustersConfig.MachinePools = tt.machinePool
k3sVersions, err := kubernetesversions.ListK3SAllVersions(a.client)
require.NoError(a.T(), err)

provisioning.VerifyCluster(a.T(), a.client, testConfig, clusterObject)
require.Equal(a.T(), len(a.clustersConfig.CNIs), 1)

upgradedCluster, err := provisioning.UpgradeClusterK8sVersion(a.client, &clusterObject.Name, &k3sVersions[numOfK3SVersions-1])
reports.TimeoutClusterReport(clusterObject, err)
require.NoError(a.T(), err)
if a.clustersConfig.K3SKubernetesVersions != nil {
k3sVersions = a.clustersConfig.K3SKubernetesVersions
}
numOfK3SVersions := len(k3sVersions)

testConfig := clusters.ConvertConfigToClusterConfig(a.clustersConfig)
testConfig.KubernetesVersion = k3sVersions[numOfK3SVersions-2]
testConfig.CNI = a.clustersConfig.CNIs[0]

versionToUpgrade := k3sVersions[numOfK3SVersions-1]
tt.name += testConfig.KubernetesVersion + " to " + versionToUpgrade
a.Run(tt.name, func() {
clusterObject, err := provisioning.CreateProvisioningAirgapCustomCluster(a.client, testConfig, a.corralPackage)
require.NoError(a.T(), err)

provisioning.VerifyUpgrade(a.T(), upgradedCluster, k3sVersions[numOfK3SVersions-1])
reports.TimeoutClusterReport(clusterObject, err)
require.NoError(a.T(), err)

provisioning.VerifyCluster(a.T(), a.client, testConfig, clusterObject)

updatedClusterObject := new(apisV1.Cluster)
err = steveV1.ConvertToK8sType(clusterObject, &updatedClusterObject)
require.NoError(a.T(), err)

updatedClusterObject.Spec.KubernetesVersion = versionToUpgrade
testConfig.KubernetesVersion = versionToUpgrade

a.client, err = a.client.ReLogin()
require.NoError(a.T(), err)

upgradedCluster, err := extensionscluster.UpdateK3SRKE2Cluster(a.client, clusterObject, updatedClusterObject)
require.NoError(a.T(), err)

provisioning.VerifyCluster(a.T(), a.client, testConfig, upgradedCluster)
})
}
}

// In order for 'go test' to run this suite, we need to create
Expand Down
Loading

0 comments on commit d0cdc69

Please sign in to comment.