From c9915ace056d516ed3987b8fd5fdf7fe67ba8a71 Mon Sep 17 00:00:00 2001 From: Diogo Recharte Date: Mon, 29 Jan 2024 10:31:53 +0000 Subject: [PATCH] EVEREST-633 Add multi-namespace support (#271) * EVEREST-633 force everest install to percona-everest namespace * EVEREST-633 remove deprecated name flag * EVEREST-633 add multi-namespace support * EVEREST-633 remove operator channel flags * EVEREST-633 update backend and operator go mods * EVEREST-633 shut up linter, we know this code is dead --- cli-tests/Makefile | 1 + cli-tests/tests/flow/all-operators.spec.ts | 50 +-- cli-tests/tests/flow/mongodb-operator.spec.ts | 11 +- cli-tests/tests/flow/pg-operator.spec.ts | 13 +- cli-tests/tests/flow/pxc-operator.spec.ts | 11 +- commands/delete.go | 2 +- commands/install.go | 19 +- commands/uninstall.go | 2 - commands/upgrade.go | 4 +- pkg/install/install.go | 298 +++++++++++++----- pkg/kubernetes/client/client.go | 75 ++--- pkg/kubernetes/client/kubeclient_interface.go | 7 +- .../client/mock_kube_client_connector.go | 79 ++++- pkg/kubernetes/kubernetes.go | 182 +++++++++-- pkg/kubernetes/olm_operator_test.go | 8 +- pkg/monitoring/enable.go | 9 +- pkg/uninstall/uninstall.go | 39 +-- pkg/upgrade/upgrade.go | 87 +++-- 18 files changed, 578 insertions(+), 319 deletions(-) diff --git a/cli-tests/Makefile b/cli-tests/Makefile index 59935e6d..ccbb94b4 100644 --- a/cli-tests/Makefile +++ b/cli-tests/Makefile @@ -4,6 +4,7 @@ init: ## Install dependencies install-operators: ## Install operators to k8s ../bin/everest install \ + --namespace percona-everest-operators \ --skip-wizard \ test-cli: ## Run all tests diff --git a/cli-tests/tests/flow/all-operators.spec.ts b/cli-tests/tests/flow/all-operators.spec.ts index b3be40ad..43753b0f 100644 --- a/cli-tests/tests/flow/all-operators.spec.ts +++ b/cli-tests/tests/flow/all-operators.spec.ts @@ -27,13 +27,18 @@ test.describe('Everest CLI install', async () => { test('install all operators', async ({ page, cli, request }) => { const verifyClusterResources = async () => { await test.step('verify installed operators in k8s', async () => { + const perconaEverestPodsOut = await cli.exec('kubectl get pods --namespace=percona-everest'); + + await perconaEverestPodsOut.outContainsNormalizedMany([ + 'everest-operator-controller-manager', + ]); + const out = await cli.exec('kubectl get pods --namespace=percona-everest-all'); await out.outContainsNormalizedMany([ 'percona-xtradb-cluster-operator', 'percona-server-mongodb-operator', 'percona-postgresql-operator', - 'everest-operator-controller-manager', ]); }); }; @@ -41,7 +46,7 @@ test.describe('Everest CLI install', async () => { await test.step('run everest install command', async () => { const out = await cli.everestExecSkipWizard( - `install --name=${clusterName} --namespace=percona-everest-all`, + `install --namespace=percona-everest-all`, ); await out.assertSuccess(); @@ -64,7 +69,7 @@ test.describe('Everest CLI install', async () => { await out.outContains( 'name: DISABLE_TELEMETRY\n value: "false"', ); - out = await cli.exec(`kubectl patch service everest --patch '{"spec": {"type": "LoadBalancer"}}' --namespace=percona-everest-all`) + out = await cli.exec(`kubectl patch service everest --patch '{"spec": {"type": "LoadBalancer"}}' --namespace=percona-everest`) await out.assertSuccess(); @@ -81,42 +86,20 @@ test.describe('Everest CLI install', async () => { 'name: DISABLE_TELEMETRY\n value: "true"', ); // check that the spec.type is not overrided - out = await cli.exec('kubectl get service/everest --namespace=percona-everest-all -o yaml'); + out = await cli.exec('kubectl get service/everest --namespace=percona-everest -o yaml'); await out.outContains( 'type: LoadBalancer', ); }); - await test.step('run everest install command using a different namespace', async () => { - const install = await cli.everestExecSkipWizard( - `install --namespace=different-everest`, - ); - - await install.assertSuccess(); - - let out = await cli.exec('kubectl get clusterrolebinding everest-admin-cluster-role-binding -o yaml'); - await out.assertSuccess(); - - await out.outContainsNormalizedMany([ - 'namespace: percona-everest-all', - 'namespace: different-everest', - ]); - await cli.everestExec('uninstall --namespace=different-everest --assume-yes'); - // Check that uninstall will fail because there's no everest deployment - out = await cli.everestExec('uninstall --namespace=different-everest --assume-yes'); - await out.outErrContainsNormalizedMany([ - 'no Everest deployment in different-everest namespace', - ]); - - }); await test.step('uninstall Everest', async () => { let out = await cli.everestExec( - `uninstall --namespace=percona-everest-all --assume-yes`, + `uninstall --assume-yes`, ); await out.assertSuccess(); // check that the deployment does not exist - out = await cli.exec('kubectl get deploy percona-everest -n percona-everest-all'); + out = await cli.exec('kubectl get deploy percona-everest -n percona-everest'); await out.outErrContainsNormalizedMany([ 'Error from server (NotFound): deployments.apps "percona-everest" not found', @@ -124,16 +107,5 @@ test.describe('Everest CLI install', async () => { }); - await test.step('uninstall Everest non existent namespace', async () => { - let out = await cli.everestExec( - `uninstall --namespace=not-exist --assume-yes`, - ); - - await out.outErrContainsNormalizedMany([ - 'namespace not-exist is not found', - ]); - - }); - }); }); diff --git a/cli-tests/tests/flow/mongodb-operator.spec.ts b/cli-tests/tests/flow/mongodb-operator.spec.ts index 53108fa2..01a55009 100644 --- a/cli-tests/tests/flow/mongodb-operator.spec.ts +++ b/cli-tests/tests/flow/mongodb-operator.spec.ts @@ -27,11 +27,16 @@ test.describe('Everest CLI install', async () => { test('install only mongodb-operator', async ({ page, cli, request }) => { const verifyClusterResources = async () => { await test.step('verify installed operators in k8s', async () => { - const out = await cli.exec('kubectl get pods --namespace=percona-everest'); + const perconaEverestPodsOut = await cli.exec('kubectl get pods --namespace=percona-everest'); + + await perconaEverestPodsOut.outContainsNormalizedMany([ + 'everest-operator-controller-manager', + ]); + + const out = await cli.exec('kubectl get pods --namespace=percona-everest-operators'); await out.outContainsNormalizedMany([ 'percona-server-mongodb-operator', - 'everest-operator-controller-manager', ]); await out.outNotContains([ @@ -44,7 +49,7 @@ test.describe('Everest CLI install', async () => { await test.step('run everest install command', async () => { const out = await cli.everestExecSkipWizard( - `install --operator.mongodb=true --operator.postgresql=false --operator.xtradb-cluster=false --name=${clusterName}`, + `install --operator.mongodb=true --operator.postgresql=false --operator.xtradb-cluster=false --namespace=percona-everest-operators`, ); await out.assertSuccess(); diff --git a/cli-tests/tests/flow/pg-operator.spec.ts b/cli-tests/tests/flow/pg-operator.spec.ts index 6be0f71d..72c13ca5 100644 --- a/cli-tests/tests/flow/pg-operator.spec.ts +++ b/cli-tests/tests/flow/pg-operator.spec.ts @@ -27,11 +27,16 @@ test.describe('Everest CLI install', async () => { test('install only postgresql-operator', async ({ page, cli, request }) => { const verifyClusterResources = async () => { await test.step('verify installed operators in k8s', async () => { - const out = await cli.exec('kubectl get pods --namespace=percona-everest'); + const perconaEverestPodsOut = await cli.exec('kubectl get pods --namespace=percona-everest'); + + await perconaEverestPodsOut.outContainsNormalizedMany([ + 'everest-operator-controller-manager', + ]); + + const out = await cli.exec('kubectl get pods --namespace=percona-everest-operators'); await out.outContainsNormalizedMany([ 'percona-postgresql-operator', - 'everest-operator-controller-manager', ]); await out.outNotContains([ @@ -43,7 +48,7 @@ test.describe('Everest CLI install', async () => { await test.step('run everest install command', async () => { const out = await cli.everestExecSkipWizard( - `install --operator.mongodb=false --operator.postgresql=true --operator.xtradb-cluster=false --name=${clusterName}`, + `install --operator.mongodb=false --operator.postgresql=true --operator.xtradb-cluster=false --namespace=percona-everest-operators`, ); await out.assertSuccess(); @@ -63,7 +68,7 @@ test.describe('Everest CLI install', async () => { await operator.assertSuccess(); const out = await cli.everestExecSkipWizard( - `install --operator.mongodb=false --operator.postgresql=true --operator.xtradb-cluster=true --name=${clusterName}`, + `install --operator.mongodb=false --operator.postgresql=true --operator.xtradb-cluster=true --namespace=percona-everest-operators`, ); const restartedOperator = await cli.exec(`kubectl -n percona-everest get po | grep everest|awk {'print $1'}`); await restartedOperator.assertSuccess(); diff --git a/cli-tests/tests/flow/pxc-operator.spec.ts b/cli-tests/tests/flow/pxc-operator.spec.ts index 142bcd8b..580502bf 100644 --- a/cli-tests/tests/flow/pxc-operator.spec.ts +++ b/cli-tests/tests/flow/pxc-operator.spec.ts @@ -27,11 +27,16 @@ test.describe('Everest CLI install', async () => { test('install only xtradb-cluster-operator', async ({ page, cli, request }) => { const verifyClusterResources = async () => { await test.step('verify installed operators in k8s', async () => { - const out = await cli.exec('kubectl get pods --namespace=percona-everest'); + const perconaEverestPodsOut = await cli.exec('kubectl get pods --namespace=percona-everest'); + + await perconaEverestPodsOut.outContainsNormalizedMany([ + 'everest-operator-controller-manager', + ]); + + const out = await cli.exec('kubectl get pods --namespace=percona-everest-operators'); await out.outContainsNormalizedMany([ 'percona-xtradb-cluster-operator', - 'everest-operator-controller-manager', ]); await out.outNotContains([ @@ -44,7 +49,7 @@ test.describe('Everest CLI install', async () => { await test.step('run everest install command', async () => { const out = await cli.everestExecSkipWizard( - `install --operator.mongodb=false --operator.postgresql=false --operator.xtradb-cluster=true --name=${clusterName}`, + `install --operator.mongodb=false --operator.postgresql=false --operator.xtradb-cluster=true --namespace=percona-everest-operators`, ); await out.assertSuccess(); diff --git a/commands/delete.go b/commands/delete.go index 2bb599e6..ae16f198 100644 --- a/commands/delete.go +++ b/commands/delete.go @@ -23,7 +23,7 @@ import ( "github.com/percona/percona-everest-cli/commands/delete" ) -func newDeleteCmd(l *zap.SugaredLogger) *cobra.Command { +func newDeleteCmd(l *zap.SugaredLogger) *cobra.Command { //nolint:deadcode,unused cmd := &cobra.Command{ Use: "delete", } diff --git a/commands/install.go b/commands/install.go index 24a5c223..cdc59bba 100644 --- a/commands/install.go +++ b/commands/install.go @@ -29,7 +29,8 @@ import ( func newInstallCmd(l *zap.SugaredLogger) *cobra.Command { cmd := &cobra.Command{ - Use: "install", + Use: "install", + Example: "everestctl install --namespace dev --namespace staging --namespace prod --operator.mongodb=true --operator.postgresql=true --operator.xtradb-cluster=true --skip-wizard", Run: func(cmd *cobra.Command, args []string) { initInstallViperFlags(cmd) c := &install.Config{} @@ -57,19 +58,12 @@ func newInstallCmd(l *zap.SugaredLogger) *cobra.Command { func initInstallFlags(cmd *cobra.Command) { cmd.Flags().StringP("kubeconfig", "k", "~/.kube/config", "Path to a kubeconfig") - cmd.Flags().StringP("name", "n", "", "Kubernetes cluster name") - cmd.Flags().String("namespace", "percona-everest", "Namespace into which Percona Everest components are deployed to") + cmd.Flags().StringArray("namespace", []string{}, "Namespaces list Percona Everest can manage") cmd.Flags().Bool("skip-wizard", false, "Skip installation wizard") cmd.Flags().Bool("operator.mongodb", true, "Install MongoDB operator") cmd.Flags().Bool("operator.postgresql", true, "Install PostgreSQL operator") cmd.Flags().Bool("operator.xtradb-cluster", true, "Install XtraDB Cluster operator") - - cmd.Flags().String("channel.everest", "stable-v0", "Channel for Everest operator") - cmd.Flags().String("channel.victoria-metrics", "stable-v0", "Channel for VictoriaMetrics operator") - cmd.Flags().String("channel.xtradb-cluster", "stable-v1", "Channel for XtraDB Cluster operator") - cmd.Flags().String("channel.mongodb", "stable-v1", "Channel for MongoDB operator") - cmd.Flags().String("channel.postgresql", "fast-v2", "Channel for PostgreSQL operator") } func initInstallViperFlags(cmd *cobra.Command) { @@ -77,16 +71,9 @@ func initInstallViperFlags(cmd *cobra.Command) { viper.BindEnv("kubeconfig") //nolint:errcheck,gosec viper.BindPFlag("kubeconfig", cmd.Flags().Lookup("kubeconfig")) //nolint:errcheck,gosec - viper.BindPFlag("name", cmd.Flags().Lookup("name")) //nolint:errcheck,gosec viper.BindPFlag("namespace", cmd.Flags().Lookup("namespace")) //nolint:errcheck,gosec viper.BindPFlag("operator.mongodb", cmd.Flags().Lookup("operator.mongodb")) //nolint:errcheck,gosec viper.BindPFlag("operator.postgresql", cmd.Flags().Lookup("operator.postgresql")) //nolint:errcheck,gosec viper.BindPFlag("operator.xtradb-cluster", cmd.Flags().Lookup("operator.xtradb-cluster")) //nolint:errcheck,gosec - - viper.BindPFlag("channel.victoria-metrics", cmd.Flags().Lookup("channel.victoria-metrics")) //nolint:errcheck,gosec - viper.BindPFlag("channel.xtradb-cluster", cmd.Flags().Lookup("channel.xtradb-cluster")) //nolint:errcheck,gosec - viper.BindPFlag("channel.mongodb", cmd.Flags().Lookup("channel.mongodb")) //nolint:errcheck,gosec - viper.BindPFlag("channel.postgresql", cmd.Flags().Lookup("channel.postgresql")) //nolint:errcheck,gosec - viper.BindPFlag("channel.everest", cmd.Flags().Lookup("channel.everest")) //nolint:errcheck,gosec } diff --git a/commands/uninstall.go b/commands/uninstall.go index 8523cabb..f6b389d9 100644 --- a/commands/uninstall.go +++ b/commands/uninstall.go @@ -58,7 +58,6 @@ func newUninstallCmd(l *zap.SugaredLogger) *cobra.Command { func initUninstallFlags(cmd *cobra.Command) { cmd.Flags().StringP("kubeconfig", "k", "~/.kube/config", "Path to a kubeconfig") - cmd.Flags().String("namespace", "percona-everest", "Namespace into which Percona Everest components are deployed to") cmd.Flags().BoolP("assume-yes", "y", false, "Assume yes to all questions") cmd.Flags().BoolP("force", "f", false, "Force removal in case there are database clusters running") } @@ -66,7 +65,6 @@ func initUninstallFlags(cmd *cobra.Command) { func initUninstallViperFlags(cmd *cobra.Command) { viper.BindEnv("kubeconfig") //nolint:errcheck,gosec viper.BindPFlag("kubeconfig", cmd.Flags().Lookup("kubeconfig")) //nolint:errcheck,gosec - viper.BindPFlag("namespace", cmd.Flags().Lookup("namespace")) //nolint:errcheck,gosec viper.BindPFlag("assume-yes", cmd.Flags().Lookup("assume-yes")) //nolint:errcheck,gosec viper.BindPFlag("force", cmd.Flags().Lookup("force")) //nolint:errcheck,gosec } diff --git a/commands/upgrade.go b/commands/upgrade.go index 7118de5a..3c6f5f54 100644 --- a/commands/upgrade.go +++ b/commands/upgrade.go @@ -59,8 +59,7 @@ func newUpgradeCmd(l *zap.SugaredLogger) *cobra.Command { func initUpgradeFlags(cmd *cobra.Command) { cmd.Flags().StringP("kubeconfig", "k", "~/.kube/config", "Path to a kubeconfig") - cmd.Flags().StringP("name", "n", "", "Kubernetes cluster name") - cmd.Flags().String("namespace", "percona-everest", "Namespace into which Percona Everest components are deployed to") + cmd.Flags().StringArray("namespace", []string{}, "Namespaces list Percona Everest can manage") cmd.Flags().Bool("upgrade-olm", false, "Upgrade OLM distribution") cmd.Flags().Bool("skip-wizard", false, "Skip installation wizard") } @@ -68,7 +67,6 @@ func initUpgradeFlags(cmd *cobra.Command) { func initUpgradeViperFlags(cmd *cobra.Command) { viper.BindEnv("kubeconfig") //nolint:errcheck,gosec viper.BindPFlag("kubeconfig", cmd.Flags().Lookup("kubeconfig")) //nolint:errcheck,gosec - viper.BindPFlag("name", cmd.Flags().Lookup("name")) //nolint:errcheck,gosec viper.BindPFlag("namespace", cmd.Flags().Lookup("namespace")) //nolint:errcheck,gosec viper.BindPFlag("upgrade-olm", cmd.Flags().Lookup("upgrade-olm")) //nolint:errcheck,gosec viper.BindPFlag("skip-wizard", cmd.Flags().Lookup("skip-wizard")) //nolint:errcheck,gosec diff --git a/pkg/install/install.go b/pkg/install/install.go index dd42c6ff..7fa051f3 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -22,11 +22,14 @@ import ( "errors" "fmt" "net/url" + "strings" "github.com/AlecAivazis/survey/v2" "github.com/operator-framework/api/pkg/operators/v1alpha1" "go.uber.org/zap" "golang.org/x/sync/errgroup" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "github.com/percona/percona-everest-cli/pkg/kubernetes" @@ -42,26 +45,45 @@ type Install struct { } const ( - everestOperatorName = "everest-operator" - pxcOperatorName = "percona-xtradb-cluster-operator" - psmdbOperatorName = "percona-server-mongodb-operator" - pgOperatorName = "percona-postgresql-operator" - operatorInstallThreads = 1 + everestBackendServiceName = "percona-everest-backend" + everestOperatorName = "everest-operator" + pxcOperatorName = "percona-xtradb-cluster-operator" + psmdbOperatorName = "percona-server-mongodb-operator" + pgOperatorName = "percona-postgresql-operator" + operatorInstallThreads = 1 + + everestServiceAccount = "everest-admin" + everestServiceAccountRole = "everest-admin-role" + everestServiceAccountRoleBinding = "everest-admin-role-binding" + everestServiceAccountClusterRoleBinding = "everest-admin-cluster-role-binding" + + everestOperatorChannel = "stable-v0" + pxcOperatorChannel = "stable-v1" + psmdbOperatorChannel = "stable-v1" + pgOperatorChannel = "fast-v2" + // VMOperatorChannel is the catalog channel for the VM Operator. + VMOperatorChannel = "stable-v0" + + // CatalogSourceNamespace is the namespace where the catalog source is installed. + CatalogSourceNamespace = "olm" + // CatalogSource is the name of the catalog source. + CatalogSource = "percona-everest-catalog" + // OperatorGroup is the name of the operator group. + OperatorGroup = "percona-operators-group" + // EverestNamespace is the namespace where everest is installed. + EverestNamespace = "percona-everest" ) type ( // Config stores configuration for the operators. Config struct { - // Name of the Kubernetes Cluster - Name string - // Namespace defines the namespace operators shall be installed to. - Namespace string + // Namespaces defines namespaces that everest can operate in. + Namespaces []string `mapstructure:"namespace"` // SkipWizard skips wizard during installation. SkipWizard bool `mapstructure:"skip-wizard"` // KubeconfigPath is a path to a kubeconfig KubeconfigPath string `mapstructure:"kubeconfig"` - Channel ChannelConfig Operator OperatorConfig } @@ -74,19 +96,6 @@ type ( // PXC stores if XtraDB Cluster shall be installed. PXC bool `mapstructure:"xtradb-cluster"` } - // ChannelConfig stores configuration for operator channels. - ChannelConfig struct { - // Everest stores channel for Everest. - Everest string - // PG stores channel for PostgreSQL. - PG string `mapstructure:"postgresql"` - // PSMDB stores channel for MongoDB. - PSMDB string `mapstructure:"mongodb"` - // PXC stores channel for xtradb cluster. - PXC string `mapstructure:"xtradb-cluster"` - // VictoriaMetrics stores channel for VictoriaMetrics. - VictoriaMetrics string `mapstructure:"victoria-metrics"` - } ) // NewInstall returns a new Install struct. @@ -110,17 +119,42 @@ func NewInstall(c Config, l *zap.SugaredLogger) (*Install, error) { } // Run runs the operators installation process. -func (o *Install) Run(ctx context.Context) error { +func (o *Install) Run(ctx context.Context) error { //nolint:cyclop if err := o.populateConfig(); err != nil { return err } - if err := o.provisionNamespace(); err != nil { + + if len(o.config.Namespaces) == 0 { + return errors.New("namespace list is empty. Specify at least one namespace using the --namespace flag") + } + for _, ns := range o.config.Namespaces { + if ns == EverestNamespace { + return fmt.Errorf("'%s' namespace is reserved for Everest internals. Please specify another namespace", ns) + } + } + + if err := o.createNamespace(EverestNamespace); err != nil { + return err + } + if err := o.provisionOLM(ctx); err != nil { + return err + } + o.l.Info("Creating operator group for the everest") + if err := o.kubeClient.CreateOperatorGroup(ctx, OperatorGroup, EverestNamespace, o.config.Namespaces); err != nil { return err } - if err := o.performProvisioning(ctx); err != nil { + if err := o.provisionAllNamespaces(ctx); err != nil { return err } - _, err := o.kubeClient.GetSecret(ctx, token.SecretName, o.config.Namespace) + if err := o.installEverest(ctx); err != nil { + return err + } + o.l.Info("Updating cluster role bindings for the everest-admin") + if err := o.kubeClient.UpdateClusterRoleBinding(ctx, everestServiceAccountClusterRoleBinding, o.config.Namespaces); err != nil { + return err + } + + _, err := o.kubeClient.GetSecret(ctx, token.SecretName, EverestNamespace) if err != nil && !k8serrors.IsNotFound(err) { return errors.Join(err, errors.New("could not get the everest token secret")) } @@ -142,18 +176,14 @@ func (o *Install) populateConfig() error { } } - if o.config.Name == "" { - o.config.Name = o.kubeClient.ClusterName() - } - return nil } -func (o *Install) performProvisioning(ctx context.Context) error { - if err := o.provisionAllOperators(ctx); err != nil { +func (o *Install) installEverest(ctx context.Context) error { + if err := o.installOperator(ctx, everestOperatorChannel, everestOperatorName, EverestNamespace)(); err != nil { return err } - d, err := o.kubeClient.GetDeployment(ctx, kubernetes.PerconaEverestDeploymentName, o.config.Namespace) + d, err := o.kubeClient.GetDeployment(ctx, kubernetes.PerconaEverestDeploymentName, EverestNamespace) var everestExists bool if err != nil && !k8serrors.IsNotFound(err) { return err @@ -163,15 +193,58 @@ func (o *Install) performProvisioning(ctx context.Context) error { } if !everestExists { - o.l.Info(fmt.Sprintf("Deploying Everest to %s", o.config.Namespace)) - err = o.kubeClient.InstallEverest(ctx, o.config.Namespace) + o.l.Info(fmt.Sprintf("Deploying Everest to %s", EverestNamespace)) + err = o.kubeClient.InstallEverest(ctx, EverestNamespace) if err != nil { return err } + } else { + o.l.Info("Restarting Everest") + if err := o.kubeClient.RestartEverest(ctx, everestOperatorName, EverestNamespace); err != nil { + return err + } + if err := o.kubeClient.RestartEverest(ctx, everestBackendServiceName, EverestNamespace); err != nil { + return err + } } return nil } +func (o *Install) provisionAllNamespaces(ctx context.Context) error { + for _, namespace := range o.config.Namespaces { + namespace := namespace + if err := o.createNamespace(namespace); err != nil { + return err + } + if err := o.kubeClient.CreateOperatorGroup(ctx, OperatorGroup, namespace, []string{}); err != nil { + return err + } + + o.l.Infof("Installing operators into %s namespace", namespace) + if err := o.provisionOperators(ctx, namespace); err != nil { + return err + } + o.l.Info("Creating role for the Everest service account") + err := o.kubeClient.CreateRole(namespace, everestServiceAccountRole, o.serviceAccountRolePolicyRules()) + if err != nil { + return errors.Join(err, errors.New("could not create role")) + } + + o.l.Info("Binding role to the Everest Service account") + err = o.kubeClient.CreateRoleBinding( + namespace, + everestServiceAccountRoleBinding, + everestServiceAccountRole, + everestServiceAccount, + ) + if err != nil { + return errors.Join(err, errors.New("could not create role binding")) + } + } + + return nil +} + // runWizard runs installation wizard. func (o *Install) runWizard() error { if err := o.runEverestWizard(); err != nil { @@ -182,11 +255,34 @@ func (o *Install) runWizard() error { } func (o *Install) runEverestWizard() error { + var namespaces string pNamespace := &survey.Input{ - Message: "Namespace to deploy Everest to", - Default: o.config.Namespace, + Message: "Namespaces managed by Everest (comma separated)", + Default: namespaces, + } + if err := survey.AskOne(pNamespace, &namespaces); err != nil { + return err + } + + nsList := strings.Split(namespaces, ",") + for _, ns := range nsList { + ns = strings.TrimSpace(ns) + if ns == "" { + continue + } + + if ns == EverestNamespace { + return fmt.Errorf("'%s' namespace is reserved for Everest internals. Please specify another namespace", ns) + } + + o.config.Namespaces = append(o.config.Namespaces, ns) + } + + if len(o.config.Namespaces) == 0 { + return errors.New("namespace list is empty. Specify at least one namespace") } - return survey.AskOne(pNamespace, &o.config.Namespace) + + return nil } func (o *Install) runInstallWizard() error { @@ -241,30 +337,15 @@ func (o *Install) runInstallWizard() error { return nil } -// provisionNamespace provisions a namespace for Everest. -func (o *Install) provisionNamespace() error { - o.l.Infof("Creating namespace %s", o.config.Namespace) - err := o.kubeClient.CreateNamespace(o.config.Namespace) +// createNamespace provisions a namespace for Everest. +func (o *Install) createNamespace(namespace string) error { + o.l.Infof("Creating namespace %s", namespace) + err := o.kubeClient.CreateNamespace(namespace) if err != nil { return errors.Join(err, errors.New("could not provision namespace")) } - o.l.Infof("Namespace %s has been created", o.config.Namespace) - return nil -} - -// provisionAllOperators provisions all configured operators to a k8s cluster. -func (o *Install) provisionAllOperators(ctx context.Context) error { - o.l.Info("Started provisioning the cluster") - - if err := o.provisionOLM(ctx); err != nil { - return err - } - - if err := o.provisionInstall(ctx); err != nil { - return err - } - + o.l.Infof("Namespace %s has been created", namespace) return nil } @@ -285,11 +366,7 @@ func (o *Install) provisionOLM(ctx context.Context) error { return nil } -func (o *Install) provisionInstall(ctx context.Context) error { - deploymentsBefore, err := o.kubeClient.ListEngineDeploymentNames(ctx, o.config.Namespace) - if err != nil { - return err - } +func (o *Install) provisionOperators(ctx context.Context, namespace string) error { g, gCtx := errgroup.WithContext(ctx) // We set the limit to 1 since operator installation // requires an update to the same installation plan which @@ -298,32 +375,22 @@ func (o *Install) provisionInstall(ctx context.Context) error { g.SetLimit(operatorInstallThreads) if o.config.Operator.PXC { - g.Go(o.installOperator(gCtx, o.config.Channel.PXC, pxcOperatorName)) + g.Go(o.installOperator(gCtx, pxcOperatorChannel, pxcOperatorName, namespace)) } if o.config.Operator.PSMDB { - g.Go(o.installOperator(gCtx, o.config.Channel.PSMDB, psmdbOperatorName)) + g.Go(o.installOperator(gCtx, psmdbOperatorChannel, psmdbOperatorName, namespace)) } if o.config.Operator.PG { - g.Go(o.installOperator(gCtx, o.config.Channel.PG, pgOperatorName)) + g.Go(o.installOperator(gCtx, pgOperatorChannel, pgOperatorName, namespace)) } if err := g.Wait(); err != nil { return err } - if err := o.installOperator(ctx, o.config.Channel.Everest, everestOperatorName)(); err != nil { - return err - } - deploymentsAfter, err := o.kubeClient.ListEngineDeploymentNames(ctx, o.config.Namespace) - if err != nil { - return err - } - if len(deploymentsBefore) != 0 && len(deploymentsBefore) != len(deploymentsAfter) { - return o.restartEverestOperatorPod(ctx) - } return nil } -func (o *Install) installOperator(ctx context.Context, channel, operatorName string) func() error { +func (o *Install) installOperator(ctx context.Context, channel, operatorName, namespace string) func() error { return func() error { // We check if the context has not been cancelled yet to return early if err := ctx.Err(); err != nil { @@ -334,14 +401,25 @@ func (o *Install) installOperator(ctx context.Context, channel, operatorName str o.l.Infof("Installing %s operator", operatorName) params := kubernetes.InstallOperatorRequest{ - Namespace: o.config.Namespace, + Namespace: namespace, Name: operatorName, - OperatorGroup: kubernetes.OperatorGroup, - CatalogSource: kubernetes.CatalogSource, - CatalogSourceNamespace: kubernetes.CatalogSourceNamespace, + OperatorGroup: OperatorGroup, + CatalogSource: CatalogSource, + CatalogSourceNamespace: CatalogSourceNamespace, Channel: channel, InstallPlanApproval: v1alpha1.ApprovalManual, } + if len(o.config.Namespaces) != 0 && operatorName == everestOperatorName { + params.TargetNamespaces = o.config.Namespaces + params.SubscriptionConfig = &v1alpha1.SubscriptionConfig{ + Env: []corev1.EnvVar{ + { + Name: kubernetes.EverestWatchNamespacesEnvVar, + Value: strings.Join(o.config.Namespaces, ","), + }, + }, + } + } if err := o.kubeClient.InstallOperator(ctx, params); err != nil { o.l.Errorf("failed installing %s operator", operatorName) @@ -353,13 +431,63 @@ func (o *Install) installOperator(ctx context.Context, channel, operatorName str } } +func (o *Install) serviceAccountRolePolicyRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"databaseclusters"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"databaseengines"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"databaseclusterrestores"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"databaseclusterbackups"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"backupstorages"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"everest.percona.com"}, + Resources: []string{"monitoringconfigs"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"operator.victoriametrics.com"}, + Resources: []string{"vmagents"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"*"}, + }, + } +} + func (o *Install) generateToken(ctx context.Context) (*token.ResetResponse, error) { o.l.Info("Creating token for Everest") r, err := token.NewReset( token.ResetConfig{ KubeconfigPath: o.config.KubeconfigPath, - Namespace: o.config.Namespace, + Namespace: EverestNamespace, }, o.l, ) @@ -374,7 +502,3 @@ func (o *Install) generateToken(ctx context.Context) (*token.ResetResponse, erro return res, nil } - -func (o *Install) restartEverestOperatorPod(ctx context.Context) error { - return o.kubeClient.RestartEverest(ctx, "everest-operator", o.config.Namespace) -} diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go index 18f4d5b7..7a8ccb3e 100644 --- a/pkg/kubernetes/client/client.go +++ b/pkg/kubernetes/client/client.go @@ -41,6 +41,7 @@ import ( "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -84,8 +85,6 @@ const ( defaultAPIURIPath = "/api" defaultAPIsURIPath = "/apis" - - disableTelemetryEnvVar = "DISABLE_TELEMETRY" ) // Each level has 2 spaces for PrefixWriter. @@ -822,21 +821,6 @@ func (c *Client) setEverestServiceType(u *unstructured.Unstructured, namespace s } func (c *Client) updateClusterRoleBinding(u *unstructured.Unstructured, namespace string) error { - cl, err := c.kubeClient() - if err != nil { - return err - } - binding := &unstructured.Unstructured{} - binding.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "rbac.authorization.k8s.io", - Kind: "ClusterRoleBinding", - Version: "v1", - }) - err = cl.Get(context.Background(), types.NamespacedName{Name: "everest-admin-cluster-role-binding"}, binding) - if err != nil && !apierrors.IsNotFound(err) { - return err - } - sub, ok, err := unstructured.NestedFieldNoCopy(u.Object, "subjects") if err != nil { return err @@ -861,22 +845,6 @@ func (c *Client) updateClusterRoleBinding(u *unstructured.Unstructured, namespac return err } } - if binding.GetName() == "" { - return nil - } - - bindingSub, ok, err := unstructured.NestedFieldNoCopy(binding.Object, "subjects") - if err != nil { - return err - } - if !ok { - return nil - } - bindingSubjects, ok := bindingSub.([]interface{}) - if !ok { - return nil - } - subjects = append(subjects, bindingSubjects...) return unstructured.SetNestedSlice(u.Object, subjects, "subjects") } @@ -1178,7 +1146,7 @@ func (c *Client) GetOperatorGroup(ctx context.Context, namespace, name string) ( } // CreateOperatorGroup creates an operator group to be used as part of a subscription. -func (c *Client) CreateOperatorGroup(ctx context.Context, namespace, name string) (*v1.OperatorGroup, error) { +func (c *Client) CreateOperatorGroup(ctx context.Context, namespace, name string, targetNamespaces []string) (*v1.OperatorGroup, error) { operatorClient, err := versioned.NewForConfig(c.restConfig) if err != nil { return nil, errors.Join(err, errors.New("cannot create an operator client instance")) @@ -1193,7 +1161,7 @@ func (c *Client) CreateOperatorGroup(ctx context.Context, namespace, name string Namespace: namespace, }, Spec: v1.OperatorGroupSpec{ - TargetNamespaces: []string{namespace}, + TargetNamespaces: targetNamespaces, }, Status: v1.OperatorGroupStatus{ LastUpdated: &metav1.Time{ @@ -1205,6 +1173,25 @@ func (c *Client) CreateOperatorGroup(ctx context.Context, namespace, name string return operatorClient.OperatorsV1().OperatorGroups(namespace).Create(ctx, og, metav1.CreateOptions{}) } +// CreateSubscription creates an OLM subscription. +func (c *Client) CreateSubscription(ctx context.Context, namespace string, subscription *v1alpha1.Subscription) (*v1alpha1.Subscription, error) { + operatorClient, err := versioned.NewForConfig(c.restConfig) + if err != nil { + return nil, errors.Join(err, errors.New("cannot create an operator client instance")) + } + sub, err := operatorClient. + OperatorsV1alpha1(). + Subscriptions(namespace). + Create(ctx, subscription, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return sub, nil + } + return sub, err + } + return sub, nil +} + // CreateSubscriptionForCatalog creates an OLM subscription. func (c *Client) CreateSubscriptionForCatalog(ctx context.Context, namespace, name, catalogNamespace, catalog, packageName, channel, startingCSV string, approval v1alpha1.Approval, @@ -1214,11 +1201,6 @@ func (c *Client) CreateSubscriptionForCatalog(ctx context.Context, namespace, na return nil, errors.Join(err, errors.New("cannot create an operator client instance")) } - disableTelemetry, ok := os.LookupEnv(disableTelemetryEnvVar) - if !ok || disableTelemetry != "true" { - disableTelemetry = "false" - } - subscription := &v1alpha1.Subscription{ TypeMeta: metav1.TypeMeta{ Kind: v1alpha1.SubscriptionKind, @@ -1235,14 +1217,6 @@ func (c *Client) CreateSubscriptionForCatalog(ctx context.Context, namespace, na Channel: channel, StartingCSV: startingCSV, InstallPlanApproval: approval, - Config: &v1alpha1.SubscriptionConfig{ - Env: []corev1.EnvVar{ - { - Name: disableTelemetryEnvVar, - Value: disableTelemetry, - }, - }, - }, }, } sub, err := operatorClient. @@ -1414,3 +1388,8 @@ func (c *Client) DeleteFile(fileBytes []byte) error { func (c *Client) GetService(ctx context.Context, namespace, name string) (*corev1.Service, error) { return c.clientset.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) } + +// GetClusterRoleBinding returns cluster role binding by given name. +func (c *Client) GetClusterRoleBinding(ctx context.Context, name string) (*rbacv1.ClusterRoleBinding, error) { + return c.clientset.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) +} diff --git a/pkg/kubernetes/client/kubeclient_interface.go b/pkg/kubernetes/client/kubeclient_interface.go index e5c55768..7eadd8fd 100644 --- a/pkg/kubernetes/client/kubeclient_interface.go +++ b/pkg/kubernetes/client/kubeclient_interface.go @@ -11,6 +11,7 @@ import ( everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -87,7 +88,9 @@ type KubeClientConnector interface { // GetOperatorGroup retrieves an operator group details by namespace and name. GetOperatorGroup(ctx context.Context, namespace, name string) (*v1.OperatorGroup, error) // CreateOperatorGroup creates an operator group to be used as part of a subscription. - CreateOperatorGroup(ctx context.Context, namespace, name string) (*v1.OperatorGroup, error) + CreateOperatorGroup(ctx context.Context, namespace, name string, targetNamespaces []string) (*v1.OperatorGroup, error) + // CreateSubscription creates an OLM subscription. + CreateSubscription(ctx context.Context, namespace string, subscription *v1alpha1.Subscription) (*v1alpha1.Subscription, error) // CreateSubscriptionForCatalog creates an OLM subscription. CreateSubscriptionForCatalog(ctx context.Context, namespace, name, catalogNamespace, catalog, packageName, channel, startingCSV string, approval v1alpha1.Approval) (*v1alpha1.Subscription, error) // GetSubscription retrieves an OLM subscription by namespace and name. @@ -115,6 +118,8 @@ type KubeClientConnector interface { DeleteFile(fileBytes []byte) error // GetService returns k8s service by provided namespace and name. GetService(ctx context.Context, namespace, name string) (*corev1.Service, error) + // GetClusterRoleBinding returns cluster role binding by given name. + GetClusterRoleBinding(ctx context.Context, name string) (*rbacv1.ClusterRoleBinding, error) // DeleteAllMonitoringResources deletes all resources related to monitoring from k8s cluster. DeleteAllMonitoringResources(ctx context.Context, namespace string) error // GetNamespace returns a namespace. diff --git a/pkg/kubernetes/client/mock_kube_client_connector.go b/pkg/kubernetes/client/mock_kube_client_connector.go index 2f3fef35..d1b24962 100644 --- a/pkg/kubernetes/client/mock_kube_client_connector.go +++ b/pkg/kubernetes/client/mock_kube_client_connector.go @@ -12,6 +12,7 @@ import ( mock "github.com/stretchr/testify/mock" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -138,9 +139,9 @@ func (_m *MockKubeClientConnector) CreateNamespace(name string) error { return r0 } -// CreateOperatorGroup provides a mock function with given fields: ctx, namespace, name -func (_m *MockKubeClientConnector) CreateOperatorGroup(ctx context.Context, namespace string, name string) (*v1.OperatorGroup, error) { - ret := _m.Called(ctx, namespace, name) +// CreateOperatorGroup provides a mock function with given fields: ctx, namespace, name, targetNamespaces +func (_m *MockKubeClientConnector) CreateOperatorGroup(ctx context.Context, namespace string, name string, targetNamespaces []string) (*v1.OperatorGroup, error) { + ret := _m.Called(ctx, namespace, name, targetNamespaces) if len(ret) == 0 { panic("no return value specified for CreateOperatorGroup") @@ -148,19 +149,49 @@ func (_m *MockKubeClientConnector) CreateOperatorGroup(ctx context.Context, name var r0 *v1.OperatorGroup var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (*v1.OperatorGroup, error)); ok { - return rf(ctx, namespace, name) + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) (*v1.OperatorGroup, error)); ok { + return rf(ctx, namespace, name, targetNamespaces) } - if rf, ok := ret.Get(0).(func(context.Context, string, string) *v1.OperatorGroup); ok { - r0 = rf(ctx, namespace, name) + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string) *v1.OperatorGroup); ok { + r0 = rf(ctx, namespace, name, targetNamespaces) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*v1.OperatorGroup) } } - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, namespace, name) + if rf, ok := ret.Get(1).(func(context.Context, string, string, []string) error); ok { + r1 = rf(ctx, namespace, name, targetNamespaces) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateSubscription provides a mock function with given fields: ctx, namespace, subscription +func (_m *MockKubeClientConnector) CreateSubscription(ctx context.Context, namespace string, subscription *v1alpha1.Subscription) (*v1alpha1.Subscription, error) { + ret := _m.Called(ctx, namespace, subscription) + + if len(ret) == 0 { + panic("no return value specified for CreateSubscription") + } + + var r0 *v1alpha1.Subscription + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, *v1alpha1.Subscription) (*v1alpha1.Subscription, error)); ok { + return rf(ctx, namespace, subscription) + } + if rf, ok := ret.Get(0).(func(context.Context, string, *v1alpha1.Subscription) *v1alpha1.Subscription); ok { + r0 = rf(ctx, namespace, subscription) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.Subscription) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, *v1alpha1.Subscription) error); ok { + r1 = rf(ctx, namespace, subscription) } else { r1 = ret.Error(1) } @@ -372,6 +403,36 @@ func (_m *MockKubeClientConnector) GenerateKubeConfigWithToken(user string, secr return r0, r1 } +// GetClusterRoleBinding provides a mock function with given fields: ctx, name +func (_m *MockKubeClientConnector) GetClusterRoleBinding(ctx context.Context, name string) (*rbacv1.ClusterRoleBinding, error) { + ret := _m.Called(ctx, name) + + if len(ret) == 0 { + panic("no return value specified for GetClusterRoleBinding") + } + + var r0 *rbacv1.ClusterRoleBinding + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*rbacv1.ClusterRoleBinding, error)); ok { + return rf(ctx, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *rbacv1.ClusterRoleBinding); ok { + r0 = rf(ctx, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*rbacv1.ClusterRoleBinding) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetClusterServiceVersion provides a mock function with given fields: ctx, key func (_m *MockKubeClientConnector) GetClusterServiceVersion(ctx context.Context, key types.NamespacedName) (*v1alpha1.ClusterServiceVersion, error) { ret := _m.Called(ctx, key) diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index 1781e443..625c3a6e 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -26,15 +26,18 @@ import ( "io/fs" "log" "net/http" + "os" "strings" "time" - "github.com/operator-framework/api/pkg/operators/v1alpha1" + olmv1 "github.com/operator-framework/api/pkg/operators/v1" + olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" "go.uber.org/zap" yamlv3 "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -67,15 +70,12 @@ const ( // PerconaEverestDeploymentName stores the name of everest backend deployment. PerconaEverestDeploymentName = "percona-everest" - // CatalogSourceNamespace defines a namespace to use to find a catalog source. - CatalogSourceNamespace = "olm" - // CatalogSource is the name of OLM catalog source. - CatalogSource = "percona-everest-catalog" - // OperatorGroup defines the name of the configuration for subscriptions. - OperatorGroup = "percona-operators-group" // EverestOperatorDeploymentName is the name of the deployment for everest operator. EverestOperatorDeploymentName = "everest-operator-controller-manager" + // EverestWatchNamespacesEnvVar is the name of the environment variable. + EverestWatchNamespacesEnvVar = "WATCH_NAMESPACES" + pxcDeploymentName = "percona-xtradb-cluster-operator" psmdbDeploymentName = "percona-server-mongodb-operator" postgresDeploymentName = "percona-postgresql-operator" @@ -86,6 +86,7 @@ const ( databaseClusterAPIVersion = "everest.percona.com/v1alpha1" restartAnnotationKey = "everest.percona.com/restart" managedByKey = "everest.percona.com/managed-by" + disableTelemetryEnvVar = "DISABLE_TELEMETRY" // ContainerStateWaiting represents a state when container requires some // operations being done in order to complete start up. ContainerStateWaiting ContainerState = "waiting" @@ -473,9 +474,9 @@ func (k *Kubernetes) InstallOLMOperator(ctx context.Context, upgrade bool) error func (k *Kubernetes) applyCSVs(ctx context.Context, resources []unstructured.Unstructured) error { subscriptions := filterResources(resources, func(r unstructured.Unstructured) bool { return r.GroupVersionKind() == schema.GroupVersionKind{ - Group: v1alpha1.GroupName, - Version: v1alpha1.GroupVersion, - Kind: v1alpha1.SubscriptionKind, + Group: olmv1alpha1.GroupName, + Version: olmv1alpha1.GroupVersion, + Kind: olmv1alpha1.SubscriptionKind, } }) @@ -621,20 +622,46 @@ type InstallOperatorRequest struct { CatalogSource string CatalogSourceNamespace string Channel string - InstallPlanApproval v1alpha1.Approval + InstallPlanApproval olmv1alpha1.Approval StartingCSV string + TargetNamespaces []string + SubscriptionConfig *olmv1alpha1.SubscriptionConfig } // InstallOperator installs an operator via OLM. -func (k *Kubernetes) InstallOperator(ctx context.Context, req InstallOperatorRequest) error { - if err := createOperatorGroupIfNeeded(ctx, k.client, req.OperatorGroup, req.Namespace); err != nil { - return err +func (k *Kubernetes) InstallOperator(ctx context.Context, req InstallOperatorRequest) error { //nolint:funlen + disableTelemetry, ok := os.LookupEnv(disableTelemetryEnvVar) + if !ok || disableTelemetry != "true" { + disableTelemetry = "false" + } + config := &olmv1alpha1.SubscriptionConfig{Env: []corev1.EnvVar{}} + if req.SubscriptionConfig != nil { + config = req.SubscriptionConfig + } + config.Env = append(config.Env, corev1.EnvVar{ + Name: disableTelemetryEnvVar, + Value: disableTelemetry, + }) + subscription := &olmv1alpha1.Subscription{ + TypeMeta: metav1.TypeMeta{ + Kind: olmv1alpha1.SubscriptionKind, + APIVersion: olmv1alpha1.SubscriptionCRDAPIVersion, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: req.Name, + }, + Spec: &olmv1alpha1.SubscriptionSpec{ + CatalogSource: req.CatalogSource, + CatalogSourceNamespace: req.CatalogSourceNamespace, + Package: req.Name, + Channel: req.Channel, + StartingCSV: req.StartingCSV, + InstallPlanApproval: req.InstallPlanApproval, + Config: config, + }, } - - subs, err := k.client.CreateSubscriptionForCatalog( - ctx, req.Namespace, req.Name, "olm", req.CatalogSource, - req.Name, req.Channel, req.StartingCSV, v1alpha1.ApprovalManual, - ) + subs, err := k.client.CreateSubscription(ctx, req.Namespace, subscription) if err != nil { return errors.Join(err, errors.New("cannot create a subscription to install the operator")) } @@ -691,23 +718,38 @@ func (k *Kubernetes) approveInstallPlan(ctx context.Context, namespace, installP return true, nil } -func createOperatorGroupIfNeeded( - ctx context.Context, - client client.KubeClientConnector, - name, namespace string, -) error { - _, err := client.GetOperatorGroup(ctx, namespace, name) - if err == nil { +// CreateOperatorGroup creates operator group in the given namespace. +func (k *Kubernetes) CreateOperatorGroup(ctx context.Context, name, namespace string, targetNamespaces []string) error { + targetNamespaces = append(targetNamespaces, namespace) + og, err := k.client.GetOperatorGroup(ctx, namespace, name) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err != nil && apierrors.IsNotFound(err) { + _, err = k.client.CreateOperatorGroup(ctx, namespace, name, targetNamespaces) + if err != nil { + return err + } return nil } - - _, err = client.CreateOperatorGroup(ctx, namespace, name) - - return err + og.Kind = olmv1.OperatorGroupKind + og.APIVersion = "operators.coreos.com/v1" + var update bool + for _, namespace := range targetNamespaces { + namespace := namespace + if !arrayContains(og.Spec.TargetNamespaces, namespace) { + update = true + } + } + if update { + og.Spec.TargetNamespaces = targetNamespaces + return k.client.ApplyObject(og) + } + return nil } // ListSubscriptions all the subscriptions in the namespace. -func (k *Kubernetes) ListSubscriptions(ctx context.Context, namespace string) (*v1alpha1.SubscriptionList, error) { +func (k *Kubernetes) ListSubscriptions(ctx context.Context, namespace string) (*olmv1alpha1.SubscriptionList, error) { return k.client.ListSubscriptions(ctx, namespace) } @@ -729,8 +771,8 @@ func (k *Kubernetes) UpgradeOperator(ctx context.Context, namespace, name string return err } -func (k *Kubernetes) getInstallPlan(ctx context.Context, namespace, name string) (*v1alpha1.InstallPlan, error) { - var subs *v1alpha1.Subscription +func (k *Kubernetes) getInstallPlan(ctx context.Context, namespace, name string) (*olmv1alpha1.InstallPlan, error) { + var subs *olmv1alpha1.Subscription // If the subscription was recently created, the install plan might not be ready yet. err := wait.PollUntilContextTimeout(ctx, pollInterval, pollDuration, false, func(ctx context.Context) (bool, error) { @@ -769,7 +811,7 @@ func (k *Kubernetes) GetServerVersion() (*version.Info, error) { func (k *Kubernetes) GetClusterServiceVersion( ctx context.Context, key types.NamespacedName, -) (*v1alpha1.ClusterServiceVersion, error) { +) (*olmv1alpha1.ClusterServiceVersion, error) { return k.client.GetClusterServiceVersion(ctx, key) } @@ -777,7 +819,7 @@ func (k *Kubernetes) GetClusterServiceVersion( func (k *Kubernetes) ListClusterServiceVersion( ctx context.Context, namespace string, -) (*v1alpha1.ClusterServiceVersionList, error) { +) (*olmv1alpha1.ClusterServiceVersionList, error) { return k.client.ListClusterServiceVersion(ctx, namespace) } @@ -956,6 +998,28 @@ func (k *Kubernetes) DeleteEverest(ctx context.Context, namespace string) error return nil } +// GetWatchedNamespaces returns list of watched namespaces. +func (k *Kubernetes) GetWatchedNamespaces(ctx context.Context, namespace string) ([]string, error) { + deployment, err := k.GetDeployment(ctx, EverestOperatorDeploymentName, namespace) + if err != nil { + return nil, err + } + + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name != everestOperatorContainerName { + continue + } + for _, envVar := range container.Env { + if envVar.Name != EverestWatchNamespacesEnvVar { + continue + } + return strings.Split(envVar.Value, ","), nil + } + } + + return nil, errors.New("failed to get watched namespaces") +} + // GetDeployment returns k8s deployment by provided name and namespace. func (k *Kubernetes) GetDeployment(ctx context.Context, name, namespace string) (*appsv1.Deployment, error) { return k.client.GetDeployment(ctx, name, namespace) @@ -965,3 +1029,51 @@ func (k *Kubernetes) GetDeployment(ctx context.Context, name, namespace string) func (k *Kubernetes) WaitForRollout(ctx context.Context, name, namespace string) error { return k.client.DoRolloutWait(ctx, types.NamespacedName{Name: name, Namespace: namespace}) } + +// UpdateClusterRoleBinding updates namespaces list for the cluster role by provided name. +func (k *Kubernetes) UpdateClusterRoleBinding(ctx context.Context, name string, namespaces []string) error { + binding, err := k.client.GetClusterRoleBinding(ctx, name) + if err != nil { + return err + } + if len(binding.Subjects) == 0 { + return fmt.Errorf("no subjects available for the cluster role binding %s", name) + } + var needsUpdate bool + for _, namespace := range namespaces { + namespace := namespace + if !subjectsContains(binding.Subjects, namespace) { + subject := binding.Subjects[0] + subject.Namespace = namespace + binding.Subjects = append(binding.Subjects, subject) + needsUpdate = true + } + } + if needsUpdate { + binding.Kind = "ClusterRoleBinding" + binding.APIVersion = "rbac.authorization.k8s.io/v1" + return k.client.ApplyObject(binding) + } + + return nil +} + +func arrayContains(s []string, e string) bool { + for _, a := range s { + a := a + if a == e { + return true + } + } + return false +} + +func subjectsContains(s []rbacv1.Subject, n string) bool { + for _, a := range s { + a := a + if a.Namespace == n { + return true + } + } + return false +} diff --git a/pkg/kubernetes/olm_operator_test.go b/pkg/kubernetes/olm_operator_test.go index 2b40bcf5..e82863d4 100644 --- a/pkg/kubernetes/olm_operator_test.go +++ b/pkg/kubernetes/olm_operator_test.go @@ -46,8 +46,7 @@ func TestInstallOlmOperator(t *testing.T) { //nolint:paralleltest t.Run("Install OLM Operator", func(t *testing.T) { k8sclient.On( - "CreateSubscriptionForCatalog", mock.Anything, mock.Anything, mock.Anything, - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, + "CreateSubscription", mock.Anything, mock.Anything, mock.Anything, ).Return(&v1alpha1.Subscription{}, nil) k8sclient.On("GetDeployment", ctx, mock.Anything, "olm").Return(&appsv1.Deployment{}, nil) k8sclient.On("ApplyFile", mock.Anything).Return(nil) @@ -85,9 +84,8 @@ func TestInstallOlmOperator(t *testing.T) { }, } k8sclient.On( - "CreateSubscriptionForCatalog", - mock.Anything, subscriptionNamespace, operatorName, "olm", - catalogSource, operatorName, "stable", "", v1alpha1.ApprovalManual, + "CreateSubscription", + mock.Anything, subscriptionNamespace, mockSubscription, ).Return(mockSubscription, nil) k8sclient.On("GetSubscription", mock.Anything, subscriptionNamespace, operatorName).Return(mockSubscription, nil) mockInstallPlan := &v1alpha1.InstallPlan{} diff --git a/pkg/monitoring/enable.go b/pkg/monitoring/enable.go index 7134151c..30a6607f 100644 --- a/pkg/monitoring/enable.go +++ b/pkg/monitoring/enable.go @@ -33,6 +33,7 @@ import ( "github.com/percona/percona-everest-cli/commands/common" everestClient "github.com/percona/percona-everest-cli/pkg/everest/client" + "github.com/percona/percona-everest-cli/pkg/install" "github.com/percona/percona-everest-cli/pkg/kubernetes" ) @@ -181,10 +182,10 @@ func (m *Monitoring) installVMOperator(ctx context.Context) error { params := kubernetes.InstallOperatorRequest{ Namespace: m.config.Namespace, Name: vmOperatorName, - OperatorGroup: kubernetes.OperatorGroup, - CatalogSource: kubernetes.CatalogSource, - CatalogSourceNamespace: kubernetes.CatalogSourceNamespace, - Channel: "stable-v0", + OperatorGroup: install.OperatorGroup, + CatalogSource: install.CatalogSource, + CatalogSourceNamespace: install.CatalogSourceNamespace, + Channel: install.VMOperatorChannel, InstallPlanApproval: v1alpha1.ApprovalManual, } diff --git a/pkg/uninstall/uninstall.go b/pkg/uninstall/uninstall.go index 8e4fc53a..e5b29293 100644 --- a/pkg/uninstall/uninstall.go +++ b/pkg/uninstall/uninstall.go @@ -25,6 +25,7 @@ import ( "go.uber.org/zap" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/percona/percona-everest-cli/pkg/install" "github.com/percona/percona-everest-cli/pkg/kubernetes" ) @@ -39,8 +40,6 @@ type Uninstall struct { type Config struct { // KubeconfigPath is a path to a kubeconfig KubeconfigPath string `mapstructure:"kubeconfig"` - // Namespace defines the namespace operators shall be installed to. - Namespace string // AssumeYes is true when all questions can be skipped. AssumeYes bool `mapstructure:"assume-yes"` // Force is true when we shall not prompt for removal. @@ -62,32 +61,8 @@ func NewUninstall(c Config, l *zap.SugaredLogger) (*Uninstall, error) { return cli, nil } -func (u *Uninstall) runEverestWizard() error { - if !u.config.AssumeYes { - pNamespace := &survey.Input{ - Message: "Please select namespace", - Default: u.config.Namespace, - } - if err := survey.AskOne( - pNamespace, - &u.config.Namespace, - ); err != nil { - return err - } - } - - return nil -} - // Run runs the cluster command. func (u *Uninstall) Run(ctx context.Context) error { - if err := u.runEverestWizard(); err != nil { - return err - } - if u.config.Namespace == "" { - return errors.New("namespace is not provided") - } - if !u.config.AssumeYes { msg := `You are about to uninstall Everest from the Kubernetes cluster. This will uninstall Everest and all monitoring resources deployed by it. All other resources such as Databases and Database Backups will not be affected.` @@ -112,7 +87,7 @@ This will uninstall Everest and all monitoring resources deployed by it. All oth if err := u.uninstallK8sResources(ctx); err != nil { return err } - if err := u.kubeClient.DeleteEverest(ctx, u.config.Namespace); err != nil { + if err := u.kubeClient.DeleteEverest(ctx, install.EverestNamespace); err != nil { return err } @@ -120,23 +95,23 @@ This will uninstall Everest and all monitoring resources deployed by it. All oth } func (u *Uninstall) checkResourcesExist(ctx context.Context) error { - _, err := u.kubeClient.GetNamespace(ctx, u.config.Namespace) + _, err := u.kubeClient.GetNamespace(ctx, install.EverestNamespace) if err != nil && k8serrors.IsNotFound(err) { - return fmt.Errorf("namespace %s is not found", u.config.Namespace) + return fmt.Errorf("namespace %s is not found", install.EverestNamespace) } if err != nil && !k8serrors.IsNotFound(err) { return err } - _, err = u.kubeClient.GetDeployment(ctx, kubernetes.PerconaEverestDeploymentName, u.config.Namespace) + _, err = u.kubeClient.GetDeployment(ctx, kubernetes.PerconaEverestDeploymentName, install.EverestNamespace) if err != nil && k8serrors.IsNotFound(err) { - return fmt.Errorf("no Everest deployment in %s namespace", u.config.Namespace) + return fmt.Errorf("no Everest deployment in %s namespace", install.EverestNamespace) } return err } func (u *Uninstall) uninstallK8sResources(ctx context.Context) error { u.l.Info("Deleting all Kubernetes monitoring resources in Kubernetes cluster") - if err := u.kubeClient.DeleteAllMonitoringResources(ctx, u.config.Namespace); err != nil { + if err := u.kubeClient.DeleteAllMonitoringResources(ctx, install.EverestNamespace); err != nil { return errors.Join(err, errors.New("could not uninstall monitoring resources from the Kubernetes cluster")) } diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index fb613598..3dd52b3d 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -19,27 +19,25 @@ package upgrade import ( "context" "errors" + "fmt" "net/url" "os" "github.com/AlecAivazis/survey/v2" goversion "github.com/hashicorp/go-version" - "github.com/operator-framework/api/pkg/operators/v1alpha1" "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "github.com/percona/percona-everest-cli/data" + "github.com/percona/percona-everest-cli/pkg/install" "github.com/percona/percona-everest-cli/pkg/kubernetes" ) type ( // Config defines configuration required for upgrade command. Config struct { - // Name of the Kubernetes Cluster - Name string - // Namespace defines the namespace operators shall be installed to. - Namespace string + // Namespaces defines namespaces that everest can operate in. + Namespaces []string `mapstructure:"namespace"` // KubeconfigPath is a path to a kubeconfig KubeconfigPath string `mapstructure:"kubeconfig"` // UpgradeOLM defines do we need to upgrade OLM or not. @@ -78,6 +76,12 @@ func NewUpgrade(c Config, l *zap.SugaredLogger) (*Upgrade, error) { // Run runs the operators installation process. func (u *Upgrade) Run(ctx context.Context) error { + if err := u.runEverestWizard(ctx); err != nil { + return err + } + if len(u.config.Namespaces) == 0 { + return errors.New("namespace list is empty. Specify at least one namespace") + } if err := u.upgradeOLM(ctx); err != nil { return err } @@ -92,37 +96,66 @@ func (u *Upgrade) Run(ctx context.Context) error { } u.l.Info("Subscriptions have been patched") u.l.Info("Upgrading Everest") - if err := u.kubeClient.InstallEverest(ctx, u.config.Namespace); err != nil { + if err := u.kubeClient.InstallEverest(ctx, install.EverestNamespace); err != nil { return err } u.l.Info("Everest has been upgraded") return nil } -func (u *Upgrade) patchSubscriptions(ctx context.Context) error { - subList, err := u.kubeClient.ListSubscriptions(ctx, u.config.Namespace) - if err != nil { - return err - } - disableTelemetryEnvVar := "DISABLE_TELEMETRY" - disableTelemetry, ok := os.LookupEnv(disableTelemetryEnvVar) - if !ok || disableTelemetry != "true" { - disableTelemetry = "false" - } - for _, subscription := range subList.Items { - subscription := subscription - subscription.Spec.Config = &v1alpha1.SubscriptionConfig{ - Env: []corev1.EnvVar{ - { - Name: disableTelemetryEnvVar, - Value: disableTelemetry, - }, - }, +func (u *Upgrade) runEverestWizard(ctx context.Context) error { + if !u.config.SkipWizard { + namespaces, err := u.kubeClient.GetWatchedNamespaces(ctx, install.EverestNamespace) + if err != nil { + return err } - if err := u.kubeClient.ApplyObject(&subscription); err != nil { + pNamespace := &survey.MultiSelect{ + Message: "Please select namespaces", + Options: namespaces, + } + if err := survey.AskOne( + pNamespace, + &u.config.Namespaces, + survey.WithValidator(survey.MinItems(1)), + ); err != nil { return err } } + + return nil +} + +func (u *Upgrade) patchSubscriptions(ctx context.Context) error { + for _, namespace := range u.config.Namespaces { + namespace := namespace + subList, err := u.kubeClient.ListSubscriptions(ctx, namespace) + if err != nil { + return err + } + if len(subList.Items) == 0 { + u.l.Warn(fmt.Sprintf("No subscriptions found in '%s' namespace", namespace)) + continue + } + disableTelemetryEnvVar := "DISABLE_TELEMETRY" + disableTelemetry, ok := os.LookupEnv(disableTelemetryEnvVar) + if !ok || disableTelemetry != "true" { + disableTelemetry = "false" + } + for _, subscription := range subList.Items { + u.l.Info(fmt.Sprintf("Patching %s subscription in '%s' namespace", subscription.Name, subscription.Namespace)) + subscription := subscription + for i := range subscription.Spec.Config.Env { + env := subscription.Spec.Config.Env[i] + if env.Name == disableTelemetryEnvVar { + env.Value = disableTelemetry + subscription.Spec.Config.Env[i] = env + } + } + if err := u.kubeClient.ApplyObject(&subscription); err != nil { + return err + } + } + } return nil }