From 186958fcd68943be19e7669d69b37f6e97a3c642 Mon Sep 17 00:00:00 2001 From: Anupama Upadhyayula Date: Mon, 11 Nov 2024 17:32:44 -0800 Subject: [PATCH] Stackstate P0 checks (#47623) * Initial PR * Stack state p0 cases automation * Adding the rbac tests for stackstate * New changes to accomodate node driver cleanup * Updates from PR reviews * Updates from PR reviews * Adding REadme * More updates after pr reviews * Additional updates based on the offline discussion * Latest review updates * Debug * Last few PR comment updates * Skipping namespace creation if it exists --- tests/v2/actions/charts/charts.go | 7 +- tests/v2/actions/charts/stackstate.go | 287 +++++++++++++++ tests/v2/actions/observability/config.go | 8 + tests/v2/actions/observability/stackstate.go | 157 ++++++++ tests/v2/actions/uiplugins/payload.go | 37 ++ tests/v2/actions/uiplugins/stackstate.go | 180 +++++++++ tests/v2/validation/observability/README.md | 30 ++ .../rbac/stackstate_rbac_test.go | 267 ++++++++++++++ .../observability/stackstate_test.go | 348 ++++++++++++++++++ 9 files changed, 1318 insertions(+), 3 deletions(-) create mode 100644 tests/v2/actions/charts/stackstate.go create mode 100644 tests/v2/actions/observability/config.go create mode 100644 tests/v2/actions/observability/stackstate.go create mode 100644 tests/v2/actions/uiplugins/payload.go create mode 100644 tests/v2/actions/uiplugins/stackstate.go create mode 100644 tests/v2/validation/observability/README.md create mode 100644 tests/v2/validation/observability/rbac/stackstate_rbac_test.go create mode 100644 tests/v2/validation/observability/stackstate_test.go diff --git a/tests/v2/actions/charts/charts.go b/tests/v2/actions/charts/charts.go index 83d2404d2c8..b3150794c51 100644 --- a/tests/v2/actions/charts/charts.go +++ b/tests/v2/actions/charts/charts.go @@ -8,9 +8,10 @@ const ( // defaultRegistrySettingID is a private constant string that contains the ID of system default registry setting. defaultRegistrySettingID = "system-default-registry" // serverURLSettingID is a private constant string that contains the ID of server URL setting. - serverURLSettingID = "server-url" - rancherChartsName = "rancher-charts" - active = "active" + serverURLSettingID = "server-url" + rancherChartsName = "rancher-charts" + rancherPartnerCharts = "rancher-partner-charts" + active = "active" ) // InstallOptions is a struct of the required options to install a chart. diff --git a/tests/v2/actions/charts/stackstate.go b/tests/v2/actions/charts/stackstate.go new file mode 100644 index 00000000000..8c5a59b3978 --- /dev/null +++ b/tests/v2/actions/charts/stackstate.go @@ -0,0 +1,287 @@ +package charts + +import ( + "context" + "fmt" + + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" + kubenamespaces "github.com/rancher/rancher/tests/v2/actions/kubeapi/namespaces" + "github.com/rancher/rancher/tests/v2/actions/namespaces" + "github.com/rancher/rancher/tests/v2/actions/observability" + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/api/steve/catalog/types" + "github.com/rancher/shepherd/pkg/wait" + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" +) + +const ( + // Public constants + StackstateExtensionNamespace = "cattle-ui-plugin-system" + StackstateExtensionsName = "observability" + UIPluginName = "rancher-ui-plugins" + StackstateK8sAgent = "stackstate-k8s-agent" + StackstateNamespace = "stackstate" + StackstateCRD = "observability.rancher.io.configuration" + RancherPartnerChartRepo = "rancher-partner-charts" +) + +var ( + timeoutSeconds = int64(defaults.TwoMinuteTimeout) +) + +// InstallStackstateAgentChart is a private helper function that returns chart install action with stack state agent and payload options. +func InstallStackstateAgentChart(client *rancher.Client, installOptions *InstallOptions, stackstateConfigs *observability.StackStateConfig, systemProjectID string) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + log.Info("Error getting server setting.") + return err + } + + stackstateAgentChartInstallActionPayload := &payloadOpts{ + InstallOptions: *installOptions, + Name: StackstateK8sAgent, + Namespace: StackstateNamespace, + Host: serverSetting.Value, + } + + chartInstallAction := newStackstateAgentChartInstallAction(stackstateAgentChartInstallActionPayload, stackstateConfigs, systemProjectID) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + log.Info("Error getting catalogClient") + return err + } + + // register uninstall stackstate agent as a cleanup function + client.Session.RegisterCleanupFunc(func() error { + defaultChartUninstallAction := newChartUninstallAction() + + err := catalogClient.UninstallChart(StackstateK8sAgent, StackstateNamespace, defaultChartUninstallAction) + if err != nil { + return err + } + + watchAppInterface, err := catalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + StackstateK8sAgent, + TimeoutSeconds: &timeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + chart := event.Object.(*catalogv1.App) + if event.Type == watch.Error { + return false, fmt.Errorf("there was an error uninstalling stackstate agent chart") + } else if event.Type == watch.Deleted { + return true, nil + } else if chart == nil { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + + steveclient, err := client.Steve.ProxyDownstream(installOptions.Cluster.ID) + if err != nil { + return err + } + namespaceClient := steveclient.SteveType(namespaces.NamespaceSteveType) + + namespace, err := namespaceClient.ByID(StackstateNamespace) + if err != nil { + return err + } + + err = namespaceClient.Delete(namespace) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminDynamicClient, err := adminClient.GetDownStreamClusterClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + adminNamespaceResource := adminDynamicClient.Resource(kubenamespaces.NamespaceGroupVersionResource).Namespace("") + + watchNamespaceInterface, err := adminNamespaceResource.Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + StackstateNamespace, + TimeoutSeconds: &timeoutSeconds, + }) + if err != nil { + return err + } + + return wait.WatchWait(watchNamespaceInterface, func(event watch.Event) (ready bool, err error) { + if event.Type == watch.Deleted { + return true, nil + } + return false, nil + }) + }) + + err = catalogClient.InstallChart(chartInstallAction, RancherPartnerChartRepo) + if err != nil { + log.Info("Errored installing the chart") + return err + } + + // wait for chart to be fully deployed + watchAppInterface, err := catalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + StackstateK8sAgent, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + log.Info("Unable to obtain the installed app ") + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + return false, nil + }) + + if err != nil { + log.Info("Unable to obtain the status of the installed app ") + return err + } + return nil +} + +// newStackstateAgentChartInstallAction is a helper function that returns an array of newChartInstallActions for installing the stackstate agent charts +func newStackstateAgentChartInstallAction(p *payloadOpts, stackstateConfigs *observability.StackStateConfig, systemProjectID string) *types.ChartInstallAction { + stackstateValues := map[string]interface{}{ + "stackstate": map[string]interface{}{ + "cluster": map[string]interface{}{ + "name": p.Cluster.Name, + }, + "apiKey": stackstateConfigs.ClusterApiKey, + "url": stackstateConfigs.Url, + }, + } + + chartInstall := newChartInstall(p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, rancherPartnerCharts, systemProjectID, p.DefaultRegistry, stackstateValues) + + chartInstalls := []types.ChartInstall{*chartInstall} + chartInstallAction := newChartInstallAction(p.Namespace, p.ProjectID, chartInstalls) + + return chartInstallAction +} + +// UpgradeStackstateAgentChart is a helper function that upgrades the stackstate agent chart. +func UpgradeStackstateAgentChart(client *rancher.Client, installOptions *InstallOptions, stackstateConfigs *observability.StackStateConfig, systemProjectID string) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + stackstateAgentChartUpgradeActionPayload := &payloadOpts{ + InstallOptions: *installOptions, + Name: StackstateK8sAgent, + Namespace: StackstateNamespace, + Host: serverSetting.Value, + } + + chartUpgradeAction := newStackstateAgentChartUpgradeAction(stackstateAgentChartUpgradeActionPayload, stackstateConfigs) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + err = catalogClient.UpgradeChart(chartUpgradeAction, RancherPartnerChartRepo) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + // wait for chart to be in status pending upgrade + watchAppInterface, err := adminCatalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + StackstateK8sAgent, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(StackstateNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + StackstateK8sAgent, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + + return nil +} + +// newStackstateAgentChartUpgradeAction is a private helper function that returns chart upgrade action. +func newStackstateAgentChartUpgradeAction(p *payloadOpts, stackstateConfigs *observability.StackStateConfig) *types.ChartUpgradeAction { + + stackstateValues := map[string]interface{}{ + "stackstate": map[string]interface{}{ + "cluster": map[string]interface{}{ + "name": p.Cluster.Name, + }, + "apiKey": stackstateConfigs.ClusterApiKey, + "url": stackstateConfigs.Url, + }, + } + + chartUpgrade := newChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, stackstateValues) + chartUpgrades := []types.ChartUpgrade{*chartUpgrade} + chartUpgradeAction := newChartUpgradeAction(p.Namespace, chartUpgrades) + + return chartUpgradeAction +} diff --git a/tests/v2/actions/observability/config.go b/tests/v2/actions/observability/config.go new file mode 100644 index 00000000000..d64159a2f69 --- /dev/null +++ b/tests/v2/actions/observability/config.go @@ -0,0 +1,8 @@ +package observability + +type StackStateConfig struct { + ServiceToken string `json:"serviceToken" yaml:"serviceToken"` + Url string `json:"url" yaml:"url"` + ClusterApiKey string `json:"clusterApiKey" yaml:"clusterApiKey"` + UpgradeVersion string `json:"upgradeVersion" yaml:"upgradeVersion"` +} diff --git a/tests/v2/actions/observability/stackstate.go b/tests/v2/actions/observability/stackstate.go new file mode 100644 index 00000000000..40f6dd7a8a3 --- /dev/null +++ b/tests/v2/actions/observability/stackstate.go @@ -0,0 +1,157 @@ +package observability + +import ( + "context" + "time" + + "github.com/rancher/shepherd/clients/rancher" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + "github.com/rancher/shepherd/extensions/defaults" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + // Public Constants + StackstateName = "stackstate" + ObservabilitySteveType = "configurations.observability.rancher.io" + CrdGroup = "observability.rancher.io" + ApiExtenisonsCRD = "apiextensions.k8s.io.customresourcedefinition" + + // Private Constants + localURL = "local://" + inactiveState = "inactive" + activeState = "active" +) + +// NewStackstateCRDConfiguration is a constructor that takes in the configuration and creates an unstructured type to install the CRD +func NewStackstateCRDConfiguration(namespace, name string, stackstateCRDConfig *StackStateConfig) *unstructured.Unstructured { + + crdConfig := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "url": stackstateCRDConfig.Url, + "serviceToken": stackstateCRDConfig.ServiceToken, + }, + }, + } + return crdConfig +} + +// WhitelistStackstateDomains is a helper that utilizes the rancher client and add the stackstate domains to whitelist them. +// This is a temporary solution from upstream and will need to be removed once this has been fixed. +func WhitelistStackstateDomains(client *rancher.Client, whitelistDomains []string) error { + + nodedriver := &management.NodeDriver{ + Name: StackstateName, + Active: false, + WhitelistDomains: whitelistDomains, + URL: localURL, + State: inactiveState, + } + + stackstateNodeDriver, err := client.Management.NodeDriver.Create(nodedriver) + if err != nil { + return err + } + + err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + resp, err := client.Management.NodeDriver.ByID(stackstateNodeDriver.ID) + if err != nil { + return false, err + } + + if resp.State == inactiveState { + return true, nil + } + return false, nil + }) + + return err +} + +// InstallStackstateCRD is a helper that utilizes the rancher client and installs the stackstate crds. +func InstallStackstateCRD(client *rancher.Client) error { + stackstateCRDConfig := apiextv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{Kind: "CustomResourceDefinition", APIVersion: "apiextensions.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: ObservabilitySteveType}, + Spec: apiextv1.CustomResourceDefinitionSpec{ + Group: CrdGroup, + Versions: []apiextv1.CustomResourceDefinitionVersion{ + 0: {Name: "v1beta1", + Served: true, + Storage: true, + Schema: &apiextv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "spec": { + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "url": {Type: "string"}, + "serviceToken": { + Type: "string", + }, + "apiToken": { + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + Names: apiextv1.CustomResourceDefinitionNames{ + Plural: "configurations", + Singular: "configuration", + Kind: "Configuration", + ListKind: "ConfigurationList", + }, + Scope: "Namespaced", + }, + } + + crd, err := client.Steve.SteveType(ApiExtenisonsCRD).Create(stackstateCRDConfig) + if err != nil { + return err + } + + client.Session.RegisterCleanupFunc(func() error { + err := client.Steve.SteveType(ApiExtenisonsCRD).Delete(crd) + if err != nil { + return err + } + + err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.TenSecondTimeout, true, func(ctx context.Context) (done bool, err error) { + _, err = client.Steve.SteveType(ApiExtenisonsCRD).ByID(crd.ID) + if err != nil { + return false, nil + } + return done, nil + }) + + return err + }) + + err = kwait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + resp, err := client.Steve.SteveType(ApiExtenisonsCRD).ByID(ObservabilitySteveType) + if err != nil { + return false, err + } + + if resp.ObjectMeta.State.Name == activeState { + return true, nil + } + return false, nil + }) + + return err + +} diff --git a/tests/v2/actions/uiplugins/payload.go b/tests/v2/actions/uiplugins/payload.go new file mode 100644 index 00000000000..efd6206507a --- /dev/null +++ b/tests/v2/actions/uiplugins/payload.go @@ -0,0 +1,37 @@ +package uiplugins + +import "github.com/rancher/shepherd/pkg/api/steve/catalog/types" + +type ExtensionOptions struct { + ChartName string + Version string + ReleaseName string +} + +// newExtensionsInstall is a private constructor that creates a chart install with given chart values that can be used for chart install action. +func newPluginsInstall(name, version string, chartValues map[string]interface{}) *types.ChartInstall { + chartInstall := types.ChartInstall{ + ChartName: name, + ReleaseName: name, + Version: version, + Values: nil, + } + + for k, v := range chartValues { + chartInstall.Values[k] = v + } + + return &chartInstall +} + + +// newPluginUninstallAction is a private constructor that creates a default payload for chart uninstall action with all disabled options. +func newPluginUninstallAction() *types.ChartUninstallAction { + return &types.ChartUninstallAction{ + DisableHooks: false, + DryRun: false, + KeepHistory: false, + Timeout: nil, + Description: "", + } +} diff --git a/tests/v2/actions/uiplugins/stackstate.go b/tests/v2/actions/uiplugins/stackstate.go new file mode 100644 index 00000000000..73dd1ffa097 --- /dev/null +++ b/tests/v2/actions/uiplugins/stackstate.go @@ -0,0 +1,180 @@ +package uiplugins + +import ( + "context" + "fmt" + + "github.com/rancher/machine/libmachine/log" + v1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/api/steve/catalog/types" + "github.com/rancher/shepherd/pkg/wait" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" +) + +const ( + stackstateExtensionNamespace = "cattle-ui-plugin-system" + stackstateExtensionsName = "observability" + uiPluginName = "rancher-ui-plugins" + local = "local" +) + +var ( + timeoutSeconds = int64(defaults.TwoMinuteTimeout) +) + +// InstallObservabilityUiPlugin is a helper function that installs the observability extension chart in the local cluster of rancher. +func InstallObservabilityUiPlugin(client *rancher.Client, installExtensionOptions *ExtensionOptions) error { + + extensionInstallAction := newStackstateUiPluginInstallAction(installExtensionOptions) + + catalogClient, err := client.GetClusterCatalogClient(local) + if err != nil { + return err + } + + // register uninstall stackstate extension as a cleanup function + client.Session.RegisterCleanupFunc(func() error { + defaultChartUninstallAction := newPluginUninstallAction() + + err := catalogClient.UninstallChart(stackstateExtensionsName, stackstateExtensionNamespace, defaultChartUninstallAction) + if err != nil { + return err + } + + watchAppInterface, err := catalogClient.Apps(stackstateExtensionNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + stackstateExtensionsName, + TimeoutSeconds: &timeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + chart := event.Object.(*v1.App) + if event.Type == watch.Error { + return false, fmt.Errorf("there was an error uninstalling stackstate extension") + } else if event.Type == watch.Deleted { + log.Info("Uninstalled observability extension successfully.") + return true, nil + } else if chart == nil { + return true, nil + } + return false, nil + + }) + + return err + + }) + + err = catalogClient.InstallChart(extensionInstallAction, uiPluginName) + if err != nil { + return err + } + + watchAppInterface, err := catalogClient.Apps(stackstateExtensionNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + stackstateExtensionsName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*v1.App) + + state := app.Status.Summary.State + if state == string(v1.StatusDeployed) { + return true, nil + } + return false, nil + }) + + return err +} + +// newStackstateUiPluginInstallAction is a private helper function that returns chart install action with stackstate extension payload options. +func newStackstateUiPluginInstallAction(p *ExtensionOptions) *types.ChartInstallAction { + + chartInstall := newPluginsInstall(p.ChartName, p.Version, nil) + chartInstalls := []types.ChartInstall{*chartInstall} + + chartInstallAction := &types.ChartInstallAction{ + Namespace: stackstateExtensionNamespace, + Charts: chartInstalls, + } + + return chartInstallAction +} + +// CreateExtensionsRepo is a helper that utilizes the rancher client and add the ui extensions repo to the list if repositories in the local cluster. +func CreateExtensionsRepo(client *rancher.Client, rancherUiPluginsName, uiExtensionGitRepoURL, uiExtensionsRepoBranch string) error { + log.Info("Adding ui extensions repo to rancher chart repositories in the local cluster.") + + clusterRepoObj := v1.ClusterRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: rancherUiPluginsName, + }, + Spec: v1.RepoSpec{ + GitRepo: uiExtensionGitRepoURL, + GitBranch: uiExtensionsRepoBranch, + }, + } + + repoObject, err := client.Catalog.ClusterRepos().Create(context.TODO(), &clusterRepoObj, metav1.CreateOptions{}) + if err != nil { + return err + } + client.Session.RegisterCleanupFunc(func() error { + err := client.Catalog.ClusterRepos().Delete(context.TODO(), repoObject.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + + watchAppInterface, err := client.Catalog.ClusterRepos().Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + repoObject.Name, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + if event.Type == watch.Error { + return false, fmt.Errorf("there was an error deleting the cluster repo") + } else if event.Type == watch.Deleted { + log.Info("Removed extensions repo successfully.") + return true, nil + } + return false, nil + }) + + return err + }) + + watchAppInterface, err := client.Catalog.ClusterRepos().Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + clusterRepoObj.Name, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + repo := event.Object.(*v1.ClusterRepo) + + state := repo.Status.Conditions + for _, condition := range state { + if condition.Type == string(v1.RepoDownloaded) && condition.Status == "True" { + return true, nil + } + } + return false, nil + }) + + return err +} diff --git a/tests/v2/validation/observability/README.md b/tests/v2/validation/observability/README.md new file mode 100644 index 00000000000..561c22bf499 --- /dev/null +++ b/tests/v2/validation/observability/README.md @@ -0,0 +1,30 @@ +# Stackstate rancher integration +The purpose of these tests is to ensure that the StackState integration in Rancher functions correctly. Specifically, when the StackState agent chart is installed, Rancher should be able to successfully communicate with the StackState server. + +## Pre-requisites + +- Ensure you have an existing cluster that the user can access. If no downstream cluster is available in Rancher, it is required that the user creates one before running this test. +- A StackState API key in the StackState UI is essential to proceed with the tests. Navigate to the StackState UI and create an instance using the cluster name from the previous step which in turn returns an api key. +- For the tests TestDynamicUpgradeStackstateAgentChart, make sure to provide the version stackstate needs to be upgraded to. Otherwise the test will be skipped. + +## Test Setup + +Your GO suite should be set to `-run ^Test$`. For example to run the stackstate_test.go, set the GO suite to `-run ^TestStackStateTestSuite$` You can find specific tests by checking the test file you plan to run. + + +In your config file, set the following: + +```yaml +rancher: + host: "rancher_server_address" + adminToken: "rancher_admin_token" + insecure: True #optional + cleanup: True #optional + clusterName: "downstream_cluster_name" + +stackstateConfigs: + url: "stackstate_ui_server_address" + serviceToken: "stackstate_user_service_token" + clusterApiKey: "stackstate_cluster_apikey" + upgradeVersion: "Stackstate_agent_version" #optional +``` diff --git a/tests/v2/validation/observability/rbac/stackstate_rbac_test.go b/tests/v2/validation/observability/rbac/stackstate_rbac_test.go new file mode 100644 index 00000000000..712be8540fb --- /dev/null +++ b/tests/v2/validation/observability/rbac/stackstate_rbac_test.go @@ -0,0 +1,267 @@ +package rbac + +import ( + "context" + "strings" + + "testing" + + provv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1" + "github.com/rancher/rancher/tests/v2/actions/charts" + "github.com/rancher/rancher/tests/v2/actions/fleet" + "github.com/rancher/rancher/tests/v2/actions/kubeapi/namespaces" + kubeprojects "github.com/rancher/rancher/tests/v2/actions/kubeapi/projects" + "github.com/rancher/rancher/tests/v2/actions/observability" + "github.com/rancher/rancher/tests/v2/actions/projects" + "github.com/rancher/rancher/tests/v2/actions/rbac" + "github.com/rancher/rancher/tests/v2/actions/uiplugins" + "github.com/rancher/shepherd/clients/rancher" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + extencharts "github.com/rancher/shepherd/extensions/charts" + extensionscluster "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/users" + "github.com/rancher/shepherd/extensions/workloads/pods" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + uiExtensionsRepo = "https://github.com/rancher/ui-plugin-charts" + uiGitBranch = "main" + rancherUIPlugins = "rancher-ui-plugins" + project = "management.cattle.io.project" + rancherPartnerCharts = "rancher-partner-charts" + systemProject = "System" + localCluster = "local" + stackStateConfigFileKey = "stackstateConfigs" +) + +type StackStateRBACTestSuite struct { + suite.Suite + client *rancher.Client + session *session.Session + cluster *management.Cluster + projectID string + stackstateAgentInstallOptions *charts.InstallOptions + stackstateConfigs *observability.StackStateConfig +} + +func (rb *StackStateRBACTestSuite) TearDownSuite() { + rb.session.Cleanup() +} + +func (rb *StackStateRBACTestSuite) SetupSuite() { + testSession := session.NewSession() + rb.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(rb.T(), err) + + rb.client = client + + clusterName := client.RancherConfig.ClusterName + require.NotEmptyf(rb.T(), clusterName, "Cluster name to install should be set") + cluster, err := extensionscluster.NewClusterMeta(rb.client, clusterName) + require.NoError(rb.T(), err) + rb.cluster, err = rb.client.Management.Cluster.ByID(cluster.ID) + require.NoError(rb.T(), err) + + projectTemplate := kubeprojects.NewProjectTemplate(cluster.ID) + projectTemplate.Name = charts.StackstateNamespace + project, err := client.Steve.SteveType(project).Create(projectTemplate) + require.NoError(rb.T(), err) + rb.projectID = project.ID + + ssNamespaceExists, err := namespaces.GetNamespaceByName(client, cluster.ID, charts.StackstateNamespace) + if ssNamespaceExists == nil && k8sErrors.IsNotFound(err) { + _, err = namespaces.CreateNamespace(client, cluster.ID, project.Name, charts.StackstateNamespace, "", map[string]string{}, map[string]string{}) + } + require.NoError(rb.T(), err) + + _, err = rb.client.Catalog.ClusterRepos().Get(context.TODO(), rancherUIPlugins, meta.GetOptions{}) + if k8sErrors.IsNotFound(err) { + err = uiplugins.CreateExtensionsRepo(rb.client, rancherUIPlugins, uiExtensionsRepo, uiGitBranch) + log.Info("Created extensions repo for ui plugins.") + } + require.NoError(rb.T(), err) + + var stackstateConfigs observability.StackStateConfig + config.LoadConfig(stackStateConfigFileKey, &stackstateConfigs) + rb.stackstateConfigs = &stackstateConfigs + + _, err = client.Management.NodeDriver.ByID(observability.StackstateName) + if strings.Contains(err.Error(), "Not Found") { + err = observability.WhitelistStackstateDomains(rb.client, []string{rb.stackstateConfigs.Url}) + } + require.NoError(rb.T(), err) + + crdsExists, err := rb.client.Steve.SteveType(observability.ApiExtenisonsCRD).ByID(observability.ObservabilitySteveType) + if crdsExists == nil && strings.Contains(err.Error(), "Not Found") { + err = observability.InstallStackstateCRD(rb.client) + log.Info("Installed stackstate crd.") + } + require.NoError(rb.T(), err) + + client, err = client.ReLogin() + require.NoError(rb.T(), err) + + initialStackstateExtension, err := extencharts.GetChartStatus(client, localCluster, charts.StackstateExtensionNamespace, charts.StackstateExtensionsName) + require.NoError(rb.T(), err) + + if !initialStackstateExtension.IsAlreadyInstalled { + latestUIPluginVersion, err := rb.client.Catalog.GetLatestChartVersion(charts.StackstateExtensionsName, charts.UIPluginName) + require.NoError(rb.T(), err) + + extensionOptions := &uiplugins.ExtensionOptions{ + ChartName: charts.StackstateExtensionsName, + ReleaseName: charts.StackstateExtensionsName, + Version: latestUIPluginVersion, + } + + err = uiplugins.InstallObservabilityUiPlugin(client, extensionOptions) + require.NoError(rb.T(), err) + log.Info("Installed stackstate ui plugin.") + } + + steveAdminClient, err := client.Steve.ProxyDownstream(localCluster) + require.NoError(rb.T(), err) + + crdConfig := observability.NewStackstateCRDConfiguration(charts.StackstateNamespace, observability.StackstateName, rb.stackstateConfigs) + _, err = steveAdminClient.SteveType(charts.StackstateCRD).Create(crdConfig) + require.NoError(rb.T(), err, "Unable to install stackstate CRD configuration.") + + latestSSVersion, err := rb.client.Catalog.GetLatestChartVersion(charts.StackstateK8sAgent, rancherPartnerCharts) + require.NoError(rb.T(), err) + rb.stackstateAgentInstallOptions = &charts.InstallOptions{ + Cluster: cluster, + Version: latestSSVersion, + ProjectID: rb.projectID, + } +} + +func (rb *StackStateRBACTestSuite) TestClusterOwnerInstallStackstate() { + subSession := rb.session.NewSession() + defer subSession.Cleanup() + + client, err := rb.client.WithSession(subSession) + require.NoError(rb.T(), err) + + initialStackstateAgent, err := extencharts.GetChartStatus(client, rb.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(rb.T(), err) + + if initialStackstateAgent.IsAlreadyInstalled { + rb.T().Skip("Stack state agent is already installed, skipping the tests.") + } + + var newUser *management.User + user, err := users.CreateUserWithRole(rb.client, users.UserConfig(), rbac.StandardUser.String()) + require.NoError(rb.T(), err) + newUser = user + rb.T().Logf("Created user: %v", newUser.Username) + + standardClient, err := rb.client.AsUser(user) + require.NoError(rb.T(), err) + + err = users.AddClusterRoleToUser(rb.client, rb.cluster, user, rbac.ClusterOwner.String(), nil) + require.NoError(rb.T(), err) + + systemProject, err := projects.GetProjectByName(client, rb.cluster.ID, systemProject) + require.NoError(rb.T(), err) + require.NotNil(rb.T(), systemProject.ID, "System project is nil.") + systemProjectID := strings.Split(systemProject.ID, ":")[1] + + rb.Run(charts.StackstateK8sAgent+" "+rb.stackstateAgentInstallOptions.Version, func() { + err = charts.InstallStackstateAgentChart(standardClient, rb.stackstateAgentInstallOptions, rb.stackstateConfigs, systemProjectID) + require.NoError(rb.T(), err) + log.Info("Stackstate agent chart installed successfully") + + rb.T().Log("Verifying the deployments of stackstate agent chart to have expected number of available replicas") + err = extencharts.WatchAndWaitDeployments(client, rb.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(rb.T(), err) + + rb.T().Log("Verifying the daemonsets of stackstate agent chart to have expected number of available replicas nodes") + err = extencharts.WatchAndWaitDaemonSets(client, rb.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(rb.T(), err) + + clusterObject, _, _ := extensionscluster.GetProvisioningClusterByName(rb.client, rb.client.RancherConfig.ClusterName, fleet.Namespace) + + var clusterName string + if clusterObject != nil { + status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(rb.T(), err) + clusterName = status.ClusterName + } else { + clusterName, err = extensionscluster.GetClusterIDByName(rb.client, rb.client.RancherConfig.ClusterName) + require.NoError(rb.T(), err) + } + podErrors := pods.StatusPods(client, clusterName) + require.Empty(rb.T(), podErrors) + }) +} + +func (rb *StackStateRBACTestSuite) TestMembersCannotInstallStackstate() { + subSession := rb.session.NewSession() + defer subSession.Cleanup() + + client, err := rb.client.WithSession(subSession) + require.NoError(rb.T(), err) + + initialStackstateAgent, err := extencharts.GetChartStatus(client, rb.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(rb.T(), err) + + if initialStackstateAgent.IsAlreadyInstalled { + rb.T().Skip("Stack state agent is already installed, skipping the tests.") + } + + tests := []struct { + name string + role rbac.Role + member string + }{ + {"Cluster Member", rbac.ClusterMember, rbac.StandardUser.String()}, + {"Project Owner", rbac.ProjectOwner, rbac.StandardUser.String()}, + {"Project Member", rbac.ProjectMember, rbac.StandardUser.String()}, + } + + for _, tt := range tests { + var newUser *management.User + user, err := users.CreateUserWithRole(rb.client, users.UserConfig(), tt.member) + require.NoError(rb.T(), err) + newUser = user + rb.T().Logf("Created user: %v", newUser.Username) + + standardClient, err := rb.client.AsUser(user) + require.NoError(rb.T(), err) + + if strings.Contains(tt.role.String(), "project") { + stackstateProject, err := client.Management.Project.ByID(rb.projectID) + require.NoError(rb.T(), err) + err = users.AddProjectMember(rb.client, stackstateProject, user, tt.role.String(), nil) + require.NoError(rb.T(), err) + } else { + err := users.AddClusterRoleToUser(rb.client, rb.cluster, user, tt.role.String(), nil) + require.NoError(rb.T(), err) + } + + systemProject, err := projects.GetProjectByName(client, rb.cluster.ID, systemProject) + require.NoError(rb.T(), err) + require.NotNil(rb.T(), systemProject.ID, "System project is nil.") + systemProjectID := strings.Split(systemProject.ID, ":")[1] + rb.Run(charts.StackstateK8sAgent+" "+rb.stackstateAgentInstallOptions.Version, func() { + err = charts.InstallStackstateAgentChart(standardClient, rb.stackstateAgentInstallOptions, rb.stackstateConfigs, systemProjectID) + require.Error(rb.T(), err) + k8sErrors.IsForbidden(err) + }) + } +} + +func TestStackStateRBACTestSuite(t *testing.T) { + suite.Run(t, new(StackStateRBACTestSuite)) +} diff --git a/tests/v2/validation/observability/stackstate_test.go b/tests/v2/validation/observability/stackstate_test.go new file mode 100644 index 00000000000..93de36b4452 --- /dev/null +++ b/tests/v2/validation/observability/stackstate_test.go @@ -0,0 +1,348 @@ +//go:build (validation || infra.any || cluster.any || sanity) && !stress && !extended + +package observability + +import ( + "context" + "strings" + + "testing" + + provv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1" + "github.com/rancher/rancher/tests/v2/actions/charts" + "github.com/rancher/rancher/tests/v2/actions/fleet" + "github.com/rancher/rancher/tests/v2/actions/kubeapi/namespaces" + kubeprojects "github.com/rancher/rancher/tests/v2/actions/kubeapi/projects" + "github.com/rancher/rancher/tests/v2/actions/observability" + "github.com/rancher/rancher/tests/v2/actions/projects" + "github.com/rancher/rancher/tests/v2/actions/uiplugins" + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + extencharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + extensionscluster "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/workloads/pods" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + project = "management.cattle.io.project" + rancherPartnerCharts = "rancher-partner-charts" + systemProject = "System" + localCluster = "local" + stackStateConfigFileKey = "stackstateConfigs" + uiExtensionsRepo = "https://github.com/rancher/ui-plugin-charts" + uiGitBranch = "main" + rancherUIPlugins = "rancher-ui-plugins" +) + +type StackStateTestSuite struct { + suite.Suite + client *rancher.Client + session *session.Session + cluster *clusters.ClusterMeta + projectID string + stackstateAgentInstallOptions *charts.InstallOptions + stackstateConfigs *observability.StackStateConfig +} + +func (ss *StackStateTestSuite) TearDownSuite() { + ss.session.Cleanup() +} + +func (ss *StackStateTestSuite) SetupSuite() { + testSession := session.NewSession() + ss.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(ss.T(), err) + + ss.client = client + + clusterName := client.RancherConfig.ClusterName + require.NotEmptyf(ss.T(), clusterName, "Cluster name to install should be set") + cluster, err := clusters.NewClusterMeta(ss.client, clusterName) + require.NoError(ss.T(), err) + ss.cluster = cluster + + projectTemplate := kubeprojects.NewProjectTemplate(cluster.ID) + projectTemplate.Name = charts.StackstateNamespace + project, err := client.Steve.SteveType(project).Create(projectTemplate) + require.NoError(ss.T(), err) + ss.projectID = project.ID + + ssNamespaceExists, err := namespaces.GetNamespaceByName(client, cluster.ID, charts.StackstateNamespace) + if ssNamespaceExists == nil && k8sErrors.IsNotFound(err) { + _, err = namespaces.CreateNamespace(client, cluster.ID, project.Name, charts.StackstateNamespace, "", map[string]string{}, map[string]string{}) + } + require.NoError(ss.T(), err) + + _, err = ss.client.Catalog.ClusterRepos().Get(context.TODO(), rancherUIPlugins, meta.GetOptions{}) + + if k8sErrors.IsNotFound(err) { + err = uiplugins.CreateExtensionsRepo(ss.client, rancherUIPlugins, uiExtensionsRepo, uiGitBranch) + log.Info("Created an extensions repo for ui plugins.") + } + require.NoError(ss.T(), err) + + var stackstateConfigs observability.StackStateConfig + config.LoadConfig(stackStateConfigFileKey, &stackstateConfigs) + ss.stackstateConfigs = &stackstateConfigs + + err = observability.WhitelistStackstateDomains(ss.client, []string{ss.stackstateConfigs.Url}) + require.NoError(ss.T(), err) + log.Info("Node driver installed with stackstate extensions ui to whitelist stackstate URL") + + crdsExists, err := ss.client.Steve.SteveType(observability.ApiExtenisonsCRD).ByID(observability.ObservabilitySteveType) + if crdsExists == nil && strings.Contains(err.Error(), "Not Found") { + err = observability.InstallStackstateCRD(ss.client) + log.Info("Installed stackstate CRD") + } + require.NoError(ss.T(), err) + + client, err = client.ReLogin() + require.NoError(ss.T(), err) + + initialStackstateExtension, err := extencharts.GetChartStatus(client, localCluster, charts.StackstateExtensionNamespace, charts.StackstateExtensionsName) + require.NoError(ss.T(), err) + + if !initialStackstateExtension.IsAlreadyInstalled { + latestUIPluginVersion, err := ss.client.Catalog.GetLatestChartVersion(charts.StackstateExtensionsName, charts.UIPluginName) + require.NoError(ss.T(), err) + + extensionOptions := &uiplugins.ExtensionOptions{ + ChartName: charts.StackstateExtensionsName, + ReleaseName: charts.StackstateExtensionsName, + Version: latestUIPluginVersion, + } + + err = uiplugins.InstallObservabilityUiPlugin(client, extensionOptions) + require.NoError(ss.T(), err) + log.Info("Installed stackstate ui extensions") + } + + steveAdminClient, err := client.Steve.ProxyDownstream(localCluster) + require.NoError(ss.T(), err) + + crdConfig := observability.NewStackstateCRDConfiguration(charts.StackstateNamespace, observability.StackstateName, ss.stackstateConfigs) + crd, err := steveAdminClient.SteveType(charts.StackstateCRD).Create(crdConfig) + require.NoError(ss.T(), err) + log.Info("Created stackstate ui extensions configuration") + + _, err = steveAdminClient.SteveType(charts.StackstateCRD).ByID(crd.ID) + require.NoError(ss.T(), err) + + latestSSVersion, err := ss.client.Catalog.GetLatestChartVersion(charts.StackstateK8sAgent, rancherPartnerCharts) + require.NoError(ss.T(), err) + + ss.stackstateAgentInstallOptions = &charts.InstallOptions{ + Cluster: cluster, + Version: latestSSVersion, + ProjectID: project.ID, + } +} + +func (ss *StackStateTestSuite) TestStackStateAgentChart() { + subSession := ss.session.NewSession() + defer subSession.Cleanup() + + client, err := ss.client.WithSession(subSession) + require.NoError(ss.T(), err) + + initialStackstateAgent, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + if initialStackstateAgent.IsAlreadyInstalled { + ss.T().Skip("Stack state agent is already installed, skipping the tests.") + } + + log.Info("Installing stack state agent on the provided cluster") + + systemProject, err := projects.GetProjectByName(client, ss.cluster.ID, systemProject) + require.NoError(ss.T(), err) + require.NotNil(ss.T(), systemProject.ID) + systemProjectID := strings.Split(systemProject.ID, ":")[1] + + ss.Run(charts.StackstateK8sAgent+" "+ss.stackstateAgentInstallOptions.Version, func() { + err = charts.InstallStackstateAgentChart(ss.client, ss.stackstateAgentInstallOptions, ss.stackstateConfigs, systemProjectID) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the deployments of stackstate agent chart to have expected number of available replicas") + err = extencharts.WatchAndWaitDeployments(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the daemonsets of stackstate agent chart to have expected number of available replicas nodes") + err = extencharts.WatchAndWaitDaemonSets(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + clusterObject, _, _ := extensionscluster.GetProvisioningClusterByName(ss.client, ss.client.RancherConfig.ClusterName, fleet.Namespace) + if clusterObject != nil { + status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(ss.T(), err) + + podErrors := pods.StatusPods(client, status.ClusterName) + require.Empty(ss.T(), podErrors) + } + }) +} + +func (ss *StackStateTestSuite) TestUpgradeStackstateAgentChart() { + subSession := ss.session.NewSession() + defer subSession.Cleanup() + + client, err := ss.client.WithSession(subSession) + require.NoError(ss.T(), err) + + versionsList, err := client.Catalog.GetListChartVersions(charts.StackstateK8sAgent, rancherPartnerCharts) + require.NoError(ss.T(), err) + + if len(versionsList) < 2 { + ss.T().Skip("Skipping the upgrade case, only one version of stackstate agent chart is available") + } + + versionLatest := versionsList[0] + versionBeforeLatest := versionsList[1] + ss.stackstateAgentInstallOptions.Version = versionBeforeLatest + + ss.Run(charts.StackstateK8sAgent+" "+ss.stackstateAgentInstallOptions.Version, func() { + initialStackstateAgent, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + if initialStackstateAgent.IsAlreadyInstalled { + ss.T().Skip("Skipping the upgrade case, stackstate agent chart is already installed.") + } + + systemProject, err := projects.GetProjectByName(client, ss.cluster.ID, systemProject) + require.NoError(ss.T(), err) + require.NotNil(ss.T(), systemProject.ID) + systemProjectID := strings.Split(systemProject.ID, ":")[1] + + ss.T().Logf("Installing stackstate agent chart with the version before the latest version %v", ss.stackstateAgentInstallOptions.Version) + err = charts.InstallStackstateAgentChart(client, ss.stackstateAgentInstallOptions, ss.stackstateConfigs, systemProjectID) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the deployments of stackstate agent chart to have expected number of available replicas") + err = extencharts.WatchAndWaitDeployments(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the daemonsets of stackstate agent chart to have expected number of available replicas nodes") + err = extencharts.WatchAndWaitDaemonSets(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + stackstateAgentChartPreUpgrade, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + // Validate current version of stackstate agent is one of the versions before latest + chartVersionPreUpgrade := stackstateAgentChartPreUpgrade.ChartDetails.Spec.Chart.Metadata.Version + require.Contains(ss.T(), versionsList[1:], chartVersionPreUpgrade) + + ss.stackstateAgentInstallOptions.Version = versionLatest + + ss.T().Logf("Upgrading stackstate agent chart to the latest version %v", ss.stackstateAgentInstallOptions.Version) + err = charts.UpgradeStackstateAgentChart(client, ss.stackstateAgentInstallOptions, ss.stackstateConfigs, systemProject.ID) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the deployments of stackstate agent chart to have expected number of available replicas") + err = extencharts.WatchAndWaitDeployments(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the daemonsets of stackstate agent chart to have expected number of available replicas nodes") + err = extencharts.WatchAndWaitDaemonSets(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + clusterObject, _, err := extensionscluster.GetProvisioningClusterByName(ss.client, ss.client.RancherConfig.ClusterName, fleet.Namespace) + require.NoError(ss.T(), err) + + var clusterName string + + if clusterObject != nil { + status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(ss.T(), err) + clusterName = status.ClusterName + } else { + clusterName, err = extensionscluster.GetClusterIDByName(ss.client, ss.client.RancherConfig.ClusterName) + require.NoError(ss.T(), err) + } + podErrors := pods.StatusPods(client, clusterName) + require.Empty(ss.T(), podErrors) + + stackstateAgentChartPostUpgrade, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + ss.T().Log("Comparing installed and desired stackstate agent versions") + chartVersionPostUpgrade := stackstateAgentChartPostUpgrade.ChartDetails.Spec.Chart.Metadata.Version + require.Equal(ss.T(), ss.stackstateAgentInstallOptions.Version, chartVersionPostUpgrade) + }) +} + +func (ss *StackStateTestSuite) TestDynamicUpgradeStackstateAgentChart() { + + subSession := ss.session.NewSession() + defer subSession.Cleanup() + + client, err := ss.client.WithSession(subSession) + require.NoError(ss.T(), err) + + versionToUpgrade := ss.stackstateConfigs.UpgradeVersion + if versionToUpgrade == "" { + ss.T().Skip("Skipping the test as no user version provided") + } + + ss.Run(charts.StackstateK8sAgent+" "+ss.stackstateAgentInstallOptions.Version, func() { + initialStackstateAgent, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + if !initialStackstateAgent.IsAlreadyInstalled || initialStackstateAgent.ChartDetails.Spec.Chart.Metadata.Version == versionToUpgrade { + ss.T().Skip("Skipping the test, as stackstate agent chart is already installed with the provided version or stackstate agent is not installed.") + } + + ss.stackstateAgentInstallOptions.Version = ss.stackstateConfigs.UpgradeVersion + require.NoError(ss.T(), err) + + ss.T().Logf("Upgrading stackstate agent chart to the user provided version %v", ss.stackstateAgentInstallOptions.Version) + err = charts.UpgradeStackstateAgentChart(client, ss.stackstateAgentInstallOptions, ss.stackstateConfigs, systemProject) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the deployments of stackstate agent chart to have expected number of available replicas") + err = extencharts.WatchAndWaitDeployments(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + ss.T().Log("Verifying the daemonsets of stackstate agent chart to have expected number of available replicas nodes") + err = extencharts.WatchAndWaitDaemonSets(client, ss.cluster.ID, charts.StackstateNamespace, meta.ListOptions{}) + require.NoError(ss.T(), err) + + clusterObject, _, _ := extensionscluster.GetProvisioningClusterByName(ss.client, ss.client.RancherConfig.ClusterName, fleet.Namespace) + + var clusterName string + if clusterObject != nil { + status := &provv1.ClusterStatus{} + err := steveV1.ConvertToK8sType(clusterObject.Status, status) + require.NoError(ss.T(), err) + clusterName = status.ClusterName + } else { + clusterName, err = extensionscluster.GetClusterIDByName(ss.client, ss.client.RancherConfig.ClusterName) + require.NoError(ss.T(), err) + } + podErrors := pods.StatusPods(client, clusterName) + require.Empty(ss.T(), podErrors) + + stackstateAgentChartPostUpgrade, err := extencharts.GetChartStatus(client, ss.cluster.ID, charts.StackstateNamespace, charts.StackstateK8sAgent) + require.NoError(ss.T(), err) + + ss.T().Log("Comparing installed and desired stackstate agent versions") + chartVersionPostUpgrade := stackstateAgentChartPostUpgrade.ChartDetails.Spec.Chart.Metadata.Version + require.Equal(ss.T(), ss.stackstateAgentInstallOptions.Version, chartVersionPostUpgrade) + }) +} + +func TestStackStateTestSuite(t *testing.T) { + suite.Run(t, new(StackStateTestSuite)) +}