forked from rancher/rancher
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added proxy tests for node_driver and custom clusters
- Loading branch information
1 parent
b35702e
commit ea088c6
Showing
14 changed files
with
887 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
|
|
||
| # Proxy Provisioning Configs | ||
|
|
||
| Please refer to [RKE1 Provisioning](../rke1/README.md) and [RKE2 Provisioning](../rke2/README.md) to build config file with basic `provisioningInput` parameters. | ||
|
|
||
| Your GO test_package should be set to `provisioning/proxy`. | ||
| Your GO suite should be set to `-run ^TestRKE2ProxyTestSuite$`. | ||
| Please see below for more details for your proxy config. Please note that the config can be in either JSON or YAML (all examples are illustrated in YAML). | ||
|
|
||
| For provisioning tests, include the following parameters into `agentEnvVars` and/or `agentEnvVarsRKE1` inside your `provisioningInput` | ||
|
|
||
| ```yaml | ||
| provisioningInput: | ||
| agentEnvVars: | ||
| - name: HTTPS_PROXY | ||
| value: #proxy server internal ip address:port | ||
| - name: HTTP_PROXY | ||
| value: #proxy server internal ip address:port | ||
| - name: NO_PROXY | ||
| value: localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,172.16.0.0/12,cattle-system.svc | ||
| agentEnvVarsRKE1: | ||
| - name: HTTPS_PROXY | ||
| value: #proxy server internal ip address:port | ||
| - name: HTTP_PROXY | ||
| value: #proxy server internal ip address:port | ||
| - name: NO_PROXY | ||
| value: localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,172.16.0.0/12,cattle-system.svc | ||
| ``` | ||
| You should have a basic understanding of Corral before running the custom cluster tests. | ||
| In your config file, set the following: | ||
| ```yaml | ||
| corralPackages: | ||
| corralPackageImages: | ||
| airgapCustomCluster: "/dist/aws-rancher-custom-cluster-false-true" | ||
| rancherHA: "/dist/aws-aws-proxy-standalone-rke2-rancher-proxy-calico-true-2.15.1-1.11.0" # the name of the corral rancher is configurable with config entry above | ||
| ... | ||
| hasDebug: <bool, default=false> | ||
| hasCleanup: <bool, default=true> | ||
| corralConfigs: | ||
| corralConfigUser: <string, default="jenkauto"> | ||
| corralConfigVars: | ||
| <var1>: <string, "val1"> # for now only aws is supported, so use the appropriate aws vars | ||
| registry_ip: <addr> # if the proxied rancher instance is created beforehand (not in the same job) set this to the registry public IP, otherwise it is automatically done in the job. | ||
| registry_private_ip: <addr> # if the proxied rancher instance is created beforehand (not in the same job) set this to the registry private IP, otherwise it is automatically done in the job. | ||
| rancher_chart_repo: <val> # | ||
| rancher_version: <val> # | ||
| kubernetes_version: <val> # | ||
| corral_private_key: <val> # only set this if you have created the proxied rancher instance beforehand. By doing `corral vars <corral> corral_private_key` | ||
| corral_public_key: <val> # only set this if you have created the proxied rancher instance beforehand. By doing `corral vars <corral> corral_private_key` | ||
| ... | ||
| corralSSHPath: <string, optional, mostly for local testing> | ||
| corralRancherHA: | ||
| name: rancherha # this is the name of your aigap corral package if it hasn't been created beforehand | ||
| ``` | ||
| Note: `corralConfigUser` will be the prefix for all resources created in your provider. | ||
| From there, your `corralConfigVars` should contain the parameters necessary to run the test. You can see what variables need to be set by navigating to your corral package folder and checking the `manifest.yaml` variables. | ||
|
|
||
| In order to run the entire proxy package set the package to `proxy/...` Your GO suite should be set to blank. | ||
|
|
||
| Formatting `corral_private_key`: | ||
| 1. Output key to file `corral vars <corral> corral_private_key > temp` | ||
| 2. Copy single line version of key `awk -v ORS='\\n' '1' temp | pbcopy` | ||
| 3. Paste into config (example yaml format `corral_private_key: '"<key>"'`) |
137 changes: 137 additions & 0 deletions
137
tests/v2/validation/provisioning/proxy/k3s_proxy_custom_cluster_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| package proxy | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| rkev1 "github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1" | ||
| "github.com/rancher/rancher/tests/v2/actions/provisioninginput" | ||
| "github.com/rancher/rancher/tests/v2/validation/pipeline/rancherha/corralha" | ||
| "github.com/rancher/rancher/tests/v2/validation/provisioning/permutations" | ||
| "github.com/rancher/shepherd/clients/corral" | ||
| "github.com/rancher/shepherd/clients/rancher" | ||
| management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" | ||
| "github.com/rancher/shepherd/extensions/clusters" | ||
| "github.com/rancher/shepherd/extensions/clusters/kubernetesversions" | ||
| "github.com/rancher/shepherd/extensions/users" | ||
| password "github.com/rancher/shepherd/extensions/users/passwordgenerator" | ||
| "github.com/rancher/shepherd/pkg/config" | ||
| "github.com/rancher/shepherd/pkg/environmentflag" | ||
| namegen "github.com/rancher/shepherd/pkg/namegenerator" | ||
| "github.com/rancher/shepherd/pkg/session" | ||
| "github.com/stretchr/testify/require" | ||
| "github.com/stretchr/testify/suite" | ||
| ) | ||
|
|
||
| type ProxyK3SCustomClusterTestSuite struct { | ||
| suite.Suite | ||
| client *rancher.Client | ||
| standardUserClient *rancher.Client | ||
| session *session.Session | ||
| corralPackage *corral.Packages | ||
| clustersConfig *provisioninginput.Config | ||
| EnvVar rkev1.EnvVar | ||
| corralImage string | ||
| corralAutoCleanup bool | ||
| } | ||
|
|
||
| func (k *ProxyK3SCustomClusterTestSuite) TearDownSuite() { | ||
| k.session.Cleanup() | ||
| } | ||
|
|
||
| func (k *ProxyK3SCustomClusterTestSuite) SetupSuite() { | ||
| testSession := session.NewSession() | ||
| k.session = testSession | ||
|
|
||
| corralRancherHA := new(corralha.CorralRancherHA) | ||
| config.LoadConfig(corralha.CorralRancherHAConfigConfigurationFileKey, corralRancherHA) | ||
|
|
||
| k.clustersConfig = new(provisioninginput.Config) | ||
| config.LoadConfig(provisioninginput.ConfigurationFileKey, k.clustersConfig) | ||
|
|
||
| client, err := rancher.NewClient("", testSession) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| k.client = client | ||
|
|
||
| enabled := true | ||
| var testuser = namegen.AppendRandomString("testuser-") | ||
| var testpassword = password.GenerateUserPassword("testpass-") | ||
| user := &management.User{ | ||
| Username: testuser, | ||
| Password: testpassword, | ||
| Name: testuser, | ||
| Enabled: &enabled, | ||
| } | ||
|
|
||
| newUser, err := users.CreateUserWithRole(client, user, "user") | ||
| require.NoError(k.T(), err) | ||
|
|
||
| newUser.Password = user.Password | ||
|
|
||
| standardUserClient, err := client.AsUser(newUser) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| k.standardUserClient = standardUserClient | ||
|
|
||
| k.clustersConfig.K3SKubernetesVersions, err = kubernetesversions.Default( | ||
| k.client, clusters.K3SClusterType.String(), k.clustersConfig.K3SKubernetesVersions) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| listOfCorrals, err := corral.ListCorral() | ||
| require.NoError(k.T(), err) | ||
|
|
||
| corralConfig := corral.Configurations() | ||
| err = corral.SetupCorralConfig(corralConfig.CorralConfigVars, corralConfig.CorralConfigUser, corralConfig.CorralSSHPath) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| k.corralPackage = corral.PackagesConfig() | ||
| k.corralImage = k.corralPackage.CorralPackageImages[corralPackageAirgapCustomClusterName] | ||
| k.corralAutoCleanup = k.corralPackage.HasCleanup | ||
|
|
||
| _, corralExist := listOfCorrals[corralRancherHA.Name] | ||
| if corralExist { | ||
| bastionIP, err := corral.GetCorralEnvVar(corralRancherHA.Name, corralRegistryPrivateIP) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| k.EnvVar.Name = "HTTP_PROXY" | ||
| k.EnvVar.Value = bastionIP + ":3219" | ||
| k.clustersConfig.AgentEnvVars = append(k.clustersConfig.AgentEnvVars, k.EnvVar) | ||
|
|
||
| k.EnvVar.Name = "HTTPS_PROXY" | ||
| k.EnvVar.Value = bastionIP + ":3219" | ||
| k.clustersConfig.AgentEnvVars = append(k.clustersConfig.AgentEnvVars, k.EnvVar) | ||
|
|
||
| k.EnvVar.Name = "NO_PROXY" | ||
| k.EnvVar.Value = "localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,cattle-system.svc" | ||
| k.clustersConfig.AgentEnvVars = append(k.clustersConfig.AgentEnvVars, k.EnvVar) | ||
|
|
||
| err = corral.SetCorralSSHKeys(corralRancherHA.Name) | ||
| require.NoError(k.T(), err) | ||
|
|
||
| err = corral.SetCorralBastion(corralRancherHA.Name) | ||
| require.NoError(k.T(), err) | ||
| } else { | ||
| k.T().Logf("Using AgentEnvVars from config: %v", k.clustersConfig.AgentEnvVars) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| func (k *ProxyK3SCustomClusterTestSuite) TestProxyK3SCustomClusterProvisioning() { | ||
| k.clustersConfig.MachinePools = []provisioninginput.MachinePools{provisioninginput.AllRolesMachinePool} | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| client *rancher.Client | ||
| runFlag bool | ||
| }{ | ||
| {provisioninginput.StandardClientName.String() + "-" + permutations.K3SAirgapCluster + "-", k.standardUserClient, k.standardUserClient.Flags.GetValue(environmentflag.Short)}, | ||
| } | ||
| for _, tt := range tests { | ||
| provisioningConfig := *k.clustersConfig | ||
| permutations.RunTestPermutations(&k.Suite, tt.name, tt.client, &provisioningConfig, permutations.K3SAirgapCluster, nil, k.corralPackage) | ||
| } | ||
| } | ||
|
|
||
| func TestProxyK3SCustomClusterTestSuite(t *testing.T) { | ||
| suite.Run(t, new(ProxyK3SCustomClusterTestSuite)) | ||
| } |
Oops, something went wrong.