From 2d77f5429989c62e0b0116cf25e7d58fa2f0ecf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= Date: Fri, 13 Oct 2023 15:06:00 +0200 Subject: [PATCH] Split apply code into multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Weiße --- cli/internal/cmd/BUILD.bazel | 3 + cli/internal/cmd/apply.go | 409 ----------------------------- cli/internal/cmd/applyhelm.go | 124 +++++++++ cli/internal/cmd/applyinit.go | 238 +++++++++++++++++ cli/internal/cmd/applyterraform.go | 115 ++++++++ 5 files changed, 480 insertions(+), 409 deletions(-) create mode 100644 cli/internal/cmd/applyhelm.go create mode 100644 cli/internal/cmd/applyinit.go create mode 100644 cli/internal/cmd/applyterraform.go diff --git a/cli/internal/cmd/BUILD.bazel b/cli/internal/cmd/BUILD.bazel index e7937a9558..40fe21ed9d 100644 --- a/cli/internal/cmd/BUILD.bazel +++ b/cli/internal/cmd/BUILD.bazel @@ -5,6 +5,9 @@ go_library( name = "cmd", srcs = [ "apply.go", + "applyhelm.go", + "applyinit.go", + "applyterraform.go", "cloud.go", "cmd.go", "config.go", diff --git a/cli/internal/cmd/apply.go b/cli/internal/cmd/apply.go index 0d032740f4..998e639d8c 100644 --- a/cli/internal/cmd/apply.go +++ b/cli/internal/cmd/apply.go @@ -9,19 +9,14 @@ package cmd import ( "bytes" "context" - "encoding/hex" "errors" "fmt" "io" "net" - "net/url" "os" "path/filepath" - "strconv" - "text/tabwriter" "time" - "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/helm" "github.com/edgelesssys/constellation/v2/cli/internal/kubecmd" @@ -33,19 +28,14 @@ import ( "github.com/edgelesssys/constellation/v2/internal/compatibility" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" - grpcRetry "github.com/edgelesssys/constellation/v2/internal/grpc/retry" - "github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/license" - "github.com/edgelesssys/constellation/v2/internal/retry" "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/tools/clientcmd" ) type applyFlags struct { @@ -411,304 +401,6 @@ func (a *applyCmd) apply(cmd *cobra.Command, configFetcher attestationconfigapi. return nil } -// runTerraformApply checks if changes to Terraform are required and applies them. -func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string) error { - a.log.Debugf("Checking if Terraform migrations are required") - migrationRequired, err := a.planTerraformMigration(cmd, conf) - if err != nil { - return fmt.Errorf("planning Terraform migrations: %w", err) - } - - if !migrationRequired { - a.log.Debugf("No changes to infrastructure required, skipping Terraform migrations") - return nil - } - - a.log.Debugf("Migrating terraform resources for infrastructure changes") - postMigrationInfraState, err := a.migrateTerraform(cmd, conf, upgradeDir) - if err != nil { - return fmt.Errorf("performing Terraform migrations: %w", err) - } - - // Merge the pre-upgrade state with the post-migration infrastructure values - a.log.Debugf("Updating state file with new infrastructure state") - if _, err := stateFile.Merge( - // temporary state with post-migration infrastructure values - state.New().SetInfrastructure(postMigrationInfraState), - ); err != nil { - return fmt.Errorf("merging pre-upgrade state with post-migration infrastructure values: %w", err) - } - - // Write the post-migration state to disk - if err := stateFile.WriteToFile(a.fileHandler, constants.StateFilename); err != nil { - return fmt.Errorf("writing state file: %w", err) - } - return nil -} - -// planTerraformMigration checks if the Constellation version the cluster is being upgraded to requires a migration. -func (a *applyCmd) planTerraformMigration(cmd *cobra.Command, conf *config.Config) (bool, error) { - a.log.Debugf("Planning Terraform migrations") - vars, err := cloudcmd.TerraformUpgradeVars(conf) - if err != nil { - return false, fmt.Errorf("parsing upgrade variables: %w", err) - } - a.log.Debugf("Using Terraform variables:\n%+v", vars) - - // Check if there are any Terraform migrations to apply - - // Add manual migrations here if required - // - // var manualMigrations []terraform.StateMigration - // for _, migration := range manualMigrations { - // u.log.Debugf("Adding manual Terraform migration: %s", migration.DisplayName) - // u.upgrader.AddManualStateMigration(migration) - // } - - return a.clusterUpgrader.PlanClusterUpgrade(cmd.Context(), cmd.OutOrStdout(), vars, conf.GetProvider()) -} - -// migrateTerraform migrates an existing Terraform state and the post-migration infrastructure state is returned. -func (a *applyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string) (state.Infrastructure, error) { - // Ask for confirmation first - fmt.Fprintln(cmd.OutOrStdout(), "The upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.") - if !a.flags.yes { - ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?") - if err != nil { - return state.Infrastructure{}, fmt.Errorf("asking for confirmation: %w", err) - } - if !ok { - cmd.Println("Aborting upgrade.") - // User doesn't expect to see any changes in his workspace after aborting an "upgrade apply", - // therefore, roll back to the backed up state. - if err := a.clusterUpgrader.RestoreClusterWorkspace(); err != nil { - return state.Infrastructure{}, fmt.Errorf( - "restoring Terraform workspace: %w, restore the Terraform workspace manually from %s ", - err, - filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir), - ) - } - return state.Infrastructure{}, fmt.Errorf("cluster upgrade aborted by user") - } - } - a.log.Debugf("Applying Terraform migrations") - - a.spinner.Start("Migrating Terraform resources", false) - infraState, err := a.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider()) - a.spinner.Stop() - if err != nil { - return state.Infrastructure{}, fmt.Errorf("applying terraform migrations: %w", err) - } - - cmd.Printf("Infrastructure migrations applied successfully and output written to: %s\n"+ - "A backup of the pre-upgrade state has been written to: %s\n", - a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename), - a.flags.pathPrefixer.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)), - ) - return infraState, nil -} - -// runInit runs the init RPC to set up the Kubernetes cluster. -// This function only needs to be run once per cluster. -// On success, it writes the Kubernetes admin config file to disk. -// Therefore it is skipped if the Kubernetes admin config file already exists. -func (a *applyCmd) runInit(cmd *cobra.Command, conf *config.Config, stateFile *state.State) (*bytes.Buffer, error) { - a.log.Debugf("Running init RPC") - a.log.Debugf("Creating aTLS Validator for %s", conf.GetAttestationConfig().GetVariant()) - validator, err := cloudcmd.NewValidator(cmd, conf.GetAttestationConfig(), a.log) - if err != nil { - return nil, fmt.Errorf("creating new validator: %w", err) - } - - a.log.Debugf("Generating master secret") - masterSecret, err := a.generateMasterSecret(cmd.OutOrStdout()) - if err != nil { - return nil, fmt.Errorf("generating master secret: %w", err) - } - a.log.Debugf("Generated master secret key and salt values") - - a.log.Debugf("Generating measurement salt") - measurementSalt, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) - if err != nil { - return nil, fmt.Errorf("generating measurement salt: %w", err) - } - - a.spinner.Start("Connecting ", false) - req := &initproto.InitRequest{ - KmsUri: masterSecret.EncodeToURI(), - StorageUri: uri.NoStoreURI, - MeasurementSalt: measurementSalt, - KubernetesVersion: versions.VersionConfigs[conf.KubernetesVersion].ClusterVersion, - KubernetesComponents: versions.VersionConfigs[conf.KubernetesVersion].KubernetesComponents.ToInitProto(), - ConformanceMode: a.flags.conformance, - InitSecret: stateFile.Infrastructure.InitSecret, - ClusterName: stateFile.Infrastructure.Name, - ApiserverCertSans: stateFile.Infrastructure.APIServerCertSANs, - } - a.log.Debugf("Sending initialization request") - resp, err := a.initCall(cmd.Context(), a.newDialer(validator), stateFile.Infrastructure.ClusterEndpoint, req) - a.spinner.Stop() - a.log.Debugf("Initialization request finished") - - if err != nil { - var nonRetriable *nonRetriableError - if errors.As(err, &nonRetriable) { - cmd.PrintErrln("Cluster initialization failed. This error is not recoverable.") - cmd.PrintErrln("Terminate your cluster and try again.") - if nonRetriable.logCollectionErr != nil { - cmd.PrintErrf("Failed to collect logs from bootstrapper: %s\n", nonRetriable.logCollectionErr) - } else { - cmd.PrintErrf("Fetched bootstrapper logs are stored in %q\n", a.flags.pathPrefixer.PrefixPrintablePath(constants.ErrorLog)) - } - } - return nil, err - } - a.log.Debugf("Initialization request successful") - - a.log.Debugf("Buffering init success message") - bufferedOutput := &bytes.Buffer{} - if err := a.writeOutput(stateFile, resp, a.flags.mergeConfigs, bufferedOutput, measurementSalt); err != nil { - return nil, err - } - - return bufferedOutput, nil -} - -// initCall performs the gRPC call to the bootstrapper to initialize the cluster. -func (a *applyCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) { - doer := &initDoer{ - dialer: dialer, - endpoint: net.JoinHostPort(ip, strconv.Itoa(constants.BootstrapperPort)), - req: req, - log: a.log, - spinner: a.spinner, - fh: file.NewHandler(afero.NewOsFs()), - } - - // Create a wrapper function that allows logging any returned error from the retrier before checking if it's the expected retriable one. - serviceIsUnavailable := func(err error) bool { - isServiceUnavailable := grpcRetry.ServiceIsUnavailable(err) - a.log.Debugf("Encountered error (retriable: %t): %s", isServiceUnavailable, err) - return isServiceUnavailable - } - - a.log.Debugf("Making initialization call, doer is %+v", doer) - retrier := retry.NewIntervalRetrier(doer, 30*time.Second, serviceIsUnavailable) - if err := retrier.Do(ctx); err != nil { - return nil, err - } - return doer.resp, nil -} - -// generateMasterSecret reads a base64 encoded master secret from file or generates a new 32 byte secret. -func (a *applyCmd) generateMasterSecret(outWriter io.Writer) (uri.MasterSecret, error) { - // No file given, generate a new secret, and save it to disk - key, err := crypto.GenerateRandomBytes(crypto.MasterSecretLengthDefault) - if err != nil { - return uri.MasterSecret{}, err - } - salt, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) - if err != nil { - return uri.MasterSecret{}, err - } - secret := uri.MasterSecret{ - Key: key, - Salt: salt, - } - if err := a.fileHandler.WriteJSON(constants.MasterSecretFilename, secret, file.OptNone); err != nil { - return uri.MasterSecret{}, err - } - fmt.Fprintf(outWriter, "Your Constellation master secret was successfully written to %q\n", a.flags.pathPrefixer.PrefixPrintablePath(constants.MasterSecretFilename)) - return secret, nil -} - -// writeOutput writes the output of a cluster initialization to the -// state- / kubeconfig-file and saves it to disk. -func (a *applyCmd) writeOutput( - stateFile *state.State, initResp *initproto.InitSuccessResponse, - mergeConfig bool, wr io.Writer, measurementSalt []byte, -) error { - fmt.Fprint(wr, "Your Constellation cluster was successfully initialized.\n\n") - - ownerID := hex.EncodeToString(initResp.GetOwnerId()) - clusterID := hex.EncodeToString(initResp.GetClusterId()) - - stateFile.SetClusterValues(state.ClusterValues{ - MeasurementSalt: measurementSalt, - OwnerID: ownerID, - ClusterID: clusterID, - }) - - tw := tabwriter.NewWriter(wr, 0, 0, 2, ' ', 0) - writeRow(tw, "Constellation cluster identifier", clusterID) - writeRow(tw, "Kubernetes configuration", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) - tw.Flush() - fmt.Fprintln(wr) - - a.log.Debugf("Rewriting cluster server address in kubeconfig to %s", stateFile.Infrastructure.ClusterEndpoint) - kubeconfig, err := clientcmd.Load(initResp.GetKubeconfig()) - if err != nil { - return fmt.Errorf("loading kubeconfig: %w", err) - } - if len(kubeconfig.Clusters) != 1 { - return fmt.Errorf("expected exactly one cluster in kubeconfig, got %d", len(kubeconfig.Clusters)) - } - for _, cluster := range kubeconfig.Clusters { - kubeEndpoint, err := url.Parse(cluster.Server) - if err != nil { - return fmt.Errorf("parsing kubeconfig server URL: %w", err) - } - kubeEndpoint.Host = net.JoinHostPort(stateFile.Infrastructure.ClusterEndpoint, kubeEndpoint.Port()) - cluster.Server = kubeEndpoint.String() - } - kubeconfigBytes, err := clientcmd.Write(*kubeconfig) - if err != nil { - return fmt.Errorf("marshaling kubeconfig: %w", err) - } - - if err := a.fileHandler.Write(constants.AdminConfFilename, kubeconfigBytes, file.OptNone); err != nil { - return fmt.Errorf("writing kubeconfig: %w", err) - } - a.log.Debugf("Kubeconfig written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) - - if mergeConfig { - if err := a.merger.mergeConfigs(constants.AdminConfFilename, a.fileHandler); err != nil { - writeRow(tw, "Failed to automatically merge kubeconfig", err.Error()) - mergeConfig = false // Set to false so we don't print the wrong message below. - } else { - writeRow(tw, "Kubernetes configuration merged with default config", "") - } - } - - if err := stateFile.WriteToFile(a.fileHandler, constants.StateFilename); err != nil { - return fmt.Errorf("writing Constellation state file: %w", err) - } - - a.log.Debugf("Constellation state file written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) - - if !mergeConfig { - fmt.Fprintln(wr, "You can now connect to your cluster by executing:") - - exportPath, err := filepath.Abs(constants.AdminConfFilename) - if err != nil { - return fmt.Errorf("getting absolute path to kubeconfig: %w", err) - } - - fmt.Fprintf(wr, "\texport KUBECONFIG=%q\n", exportPath) - } else { - fmt.Fprintln(wr, "Constellation kubeconfig merged with default config.") - - if a.merger.kubeconfigEnvVar() != "" { - fmt.Fprintln(wr, "Warning: KUBECONFIG environment variable is set.") - fmt.Fprintln(wr, "You may need to unset it to use the default config and connect to your cluster.") - } else { - fmt.Fprintln(wr, "You can now connect to your cluster.") - } - } - fmt.Fprintln(wr) // add final newline - return nil -} - // applyJoincConfig creates or updates the cluster's join config. // If the config already exists, and is different from the new config, the user is asked to confirm the upgrade. func (a *applyCmd) applyJoinConfig( @@ -759,107 +451,6 @@ func (a *applyCmd) applyJoinConfig( return nil } -// runHelmApply handles installing or upgrading helm charts for the cluster. -func (a *applyCmd) runHelmApply( - cmd *cobra.Command, conf *config.Config, stateFile *state.State, - kubeUpgrader kubernetesUpgrader, upgradeDir string, initRequired bool, -) error { - a.log.Debugf("Installing or upgrading Helm charts") - var masterSecret uri.MasterSecret - if err := a.fileHandler.ReadJSON(constants.MasterSecretFilename, &masterSecret); err != nil { - return fmt.Errorf("reading master secret: %w", err) - } - - options := helm.Options{ - Force: a.flags.force, - Conformance: a.flags.conformance, - HelmWaitMode: a.flags.helmWaitMode, - AllowDestructive: helm.DenyDestructive, - } - helmApplier, err := a.newHelmClient(constants.AdminConfFilename, a.log) - if err != nil { - return fmt.Errorf("creating Helm client: %w", err) - } - - a.log.Debugf("Getting service account URI") - serviceAccURI, err := cloudcmd.GetMarshaledServiceAccountURI(conf, a.fileHandler) - if err != nil { - return err - } - - a.log.Debugf("Preparing Helm charts") - executor, includesUpgrades, err := helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret) - if errors.Is(err, helm.ErrConfirmationMissing) { - if !a.flags.yes { - cmd.PrintErrln("WARNING: Upgrading cert-manager will destroy all custom resources you have manually created that are based on the current version of cert-manager.") - ok, askErr := askToConfirm(cmd, "Do you want to upgrade cert-manager anyway?") - if askErr != nil { - return fmt.Errorf("asking for confirmation: %w", err) - } - if !ok { - cmd.Println("Skipping upgrade.") - return nil - } - } - options.AllowDestructive = helm.AllowDestructive - executor, includesUpgrades, err = helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret) - } - var upgradeErr *compatibility.InvalidUpgradeError - if err != nil { - if !errors.As(err, &upgradeErr) { - return fmt.Errorf("preparing Helm charts: %w", err) - } - cmd.PrintErrln(err) - } - - a.log.Debugf("Backing up Helm charts") - if err := a.backupHelmCharts(cmd.Context(), kubeUpgrader, executor, includesUpgrades, upgradeDir); err != nil { - return err - } - - a.log.Debugf("Applying Helm charts") - if initRequired { - a.spinner.Start("Installing Kubernetes components ", false) - } else { - a.spinner.Start("Upgrading Kubernetes components ", false) - } - if err := executor.Apply(cmd.Context()); err != nil { - return fmt.Errorf("applying Helm charts: %w", err) - } - a.spinner.Stop() - - if !initRequired { - cmd.Println("Successfully upgraded Constellation services.") - } - - return nil -} - -// backupHelmCharts saves the Helm charts for the upgrade to disk and creates a backup of existing CRDs and CRs. -func (a *applyCmd) backupHelmCharts( - ctx context.Context, kubeUpgrader kubernetesUpgrader, executor helm.Applier, includesUpgrades bool, upgradeDir string, -) error { - // Save the Helm charts for the upgrade to disk - chartDir := filepath.Join(upgradeDir, "helm-charts") - if err := executor.SaveCharts(chartDir, a.fileHandler); err != nil { - return fmt.Errorf("saving Helm charts to disk: %w", err) - } - a.log.Debugf("Helm charts saved to %s", a.flags.pathPrefixer.PrefixPrintablePath(chartDir)) - - if includesUpgrades { - a.log.Debugf("Creating backup of CRDs and CRs") - crds, err := kubeUpgrader.BackupCRDs(ctx, upgradeDir) - if err != nil { - return fmt.Errorf("creating CRD backup: %w", err) - } - if err := kubeUpgrader.BackupCRs(ctx, crds, upgradeDir); err != nil { - return fmt.Errorf("creating CR backup: %w", err) - } - } - - return nil -} - // runK8sUpgrade upgrades image and Kubernetes version of the Constellation cluster. func (a *applyCmd) runK8sUpgrade(cmd *cobra.Command, conf *config.Config, kubeUpgrader kubernetesUpgrader, ) error { diff --git a/cli/internal/cmd/applyhelm.go b/cli/internal/cmd/applyhelm.go new file mode 100644 index 0000000000..e6dc8bc1b2 --- /dev/null +++ b/cli/internal/cmd/applyhelm.go @@ -0,0 +1,124 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" + "github.com/edgelesssys/constellation/v2/cli/internal/helm" + "github.com/edgelesssys/constellation/v2/cli/internal/state" + "github.com/edgelesssys/constellation/v2/internal/compatibility" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" + "github.com/spf13/cobra" +) + +// runHelmApply handles installing or upgrading helm charts for the cluster. +func (a *applyCmd) runHelmApply( + cmd *cobra.Command, conf *config.Config, stateFile *state.State, + kubeUpgrader kubernetesUpgrader, upgradeDir string, initRequired bool, +) error { + a.log.Debugf("Installing or upgrading Helm charts") + var masterSecret uri.MasterSecret + if err := a.fileHandler.ReadJSON(constants.MasterSecretFilename, &masterSecret); err != nil { + return fmt.Errorf("reading master secret: %w", err) + } + + options := helm.Options{ + Force: a.flags.force, + Conformance: a.flags.conformance, + HelmWaitMode: a.flags.helmWaitMode, + AllowDestructive: helm.DenyDestructive, + } + helmApplier, err := a.newHelmClient(constants.AdminConfFilename, a.log) + if err != nil { + return fmt.Errorf("creating Helm client: %w", err) + } + + a.log.Debugf("Getting service account URI") + serviceAccURI, err := cloudcmd.GetMarshaledServiceAccountURI(conf, a.fileHandler) + if err != nil { + return err + } + + a.log.Debugf("Preparing Helm charts") + executor, includesUpgrades, err := helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret) + if errors.Is(err, helm.ErrConfirmationMissing) { + if !a.flags.yes { + cmd.PrintErrln("WARNING: Upgrading cert-manager will destroy all custom resources you have manually created that are based on the current version of cert-manager.") + ok, askErr := askToConfirm(cmd, "Do you want to upgrade cert-manager anyway?") + if askErr != nil { + return fmt.Errorf("asking for confirmation: %w", err) + } + if !ok { + cmd.Println("Skipping upgrade.") + return nil + } + } + options.AllowDestructive = helm.AllowDestructive + executor, includesUpgrades, err = helmApplier.PrepareApply(conf, stateFile, options, serviceAccURI, masterSecret) + } + var upgradeErr *compatibility.InvalidUpgradeError + if err != nil { + if !errors.As(err, &upgradeErr) { + return fmt.Errorf("preparing Helm charts: %w", err) + } + cmd.PrintErrln(err) + } + + a.log.Debugf("Backing up Helm charts") + if err := a.backupHelmCharts(cmd.Context(), kubeUpgrader, executor, includesUpgrades, upgradeDir); err != nil { + return err + } + + a.log.Debugf("Applying Helm charts") + if initRequired { + a.spinner.Start("Installing Kubernetes components ", false) + } else { + a.spinner.Start("Upgrading Kubernetes components ", false) + } + if err := executor.Apply(cmd.Context()); err != nil { + return fmt.Errorf("applying Helm charts: %w", err) + } + a.spinner.Stop() + + if !initRequired { + cmd.Println("Successfully upgraded Constellation services.") + } + + return nil +} + +// backupHelmCharts saves the Helm charts for the upgrade to disk and creates a backup of existing CRDs and CRs. +func (a *applyCmd) backupHelmCharts( + ctx context.Context, kubeUpgrader kubernetesUpgrader, executor helm.Applier, includesUpgrades bool, upgradeDir string, +) error { + // Save the Helm charts for the upgrade to disk + chartDir := filepath.Join(upgradeDir, "helm-charts") + if err := executor.SaveCharts(chartDir, a.fileHandler); err != nil { + return fmt.Errorf("saving Helm charts to disk: %w", err) + } + a.log.Debugf("Helm charts saved to %s", a.flags.pathPrefixer.PrefixPrintablePath(chartDir)) + + if includesUpgrades { + a.log.Debugf("Creating backup of CRDs and CRs") + crds, err := kubeUpgrader.BackupCRDs(ctx, upgradeDir) + if err != nil { + return fmt.Errorf("creating CRD backup: %w", err) + } + if err := kubeUpgrader.BackupCRs(ctx, crds, upgradeDir); err != nil { + return fmt.Errorf("creating CR backup: %w", err) + } + } + + return nil +} diff --git a/cli/internal/cmd/applyinit.go b/cli/internal/cmd/applyinit.go new file mode 100644 index 0000000000..e8891f7a5b --- /dev/null +++ b/cli/internal/cmd/applyinit.go @@ -0,0 +1,238 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/url" + "path/filepath" + "strconv" + "text/tabwriter" + "time" + + "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" + "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" + "github.com/edgelesssys/constellation/v2/cli/internal/state" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/crypto" + "github.com/edgelesssys/constellation/v2/internal/file" + grpcRetry "github.com/edgelesssys/constellation/v2/internal/grpc/retry" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" + "github.com/edgelesssys/constellation/v2/internal/retry" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/spf13/afero" + "github.com/spf13/cobra" + "k8s.io/client-go/tools/clientcmd" +) + +// runInit runs the init RPC to set up the Kubernetes cluster. +// This function only needs to be run once per cluster. +// On success, it writes the Kubernetes admin config file to disk. +// Therefore it is skipped if the Kubernetes admin config file already exists. +func (a *applyCmd) runInit(cmd *cobra.Command, conf *config.Config, stateFile *state.State) (*bytes.Buffer, error) { + a.log.Debugf("Running init RPC") + a.log.Debugf("Creating aTLS Validator for %s", conf.GetAttestationConfig().GetVariant()) + validator, err := cloudcmd.NewValidator(cmd, conf.GetAttestationConfig(), a.log) + if err != nil { + return nil, fmt.Errorf("creating new validator: %w", err) + } + + a.log.Debugf("Generating master secret") + masterSecret, err := a.generateMasterSecret(cmd.OutOrStdout()) + if err != nil { + return nil, fmt.Errorf("generating master secret: %w", err) + } + a.log.Debugf("Generated master secret key and salt values") + + a.log.Debugf("Generating measurement salt") + measurementSalt, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) + if err != nil { + return nil, fmt.Errorf("generating measurement salt: %w", err) + } + + a.spinner.Start("Connecting ", false) + req := &initproto.InitRequest{ + KmsUri: masterSecret.EncodeToURI(), + StorageUri: uri.NoStoreURI, + MeasurementSalt: measurementSalt, + KubernetesVersion: versions.VersionConfigs[conf.KubernetesVersion].ClusterVersion, + KubernetesComponents: versions.VersionConfigs[conf.KubernetesVersion].KubernetesComponents.ToInitProto(), + ConformanceMode: a.flags.conformance, + InitSecret: stateFile.Infrastructure.InitSecret, + ClusterName: stateFile.Infrastructure.Name, + ApiserverCertSans: stateFile.Infrastructure.APIServerCertSANs, + } + a.log.Debugf("Sending initialization request") + resp, err := a.initCall(cmd.Context(), a.newDialer(validator), stateFile.Infrastructure.ClusterEndpoint, req) + a.spinner.Stop() + a.log.Debugf("Initialization request finished") + + if err != nil { + var nonRetriable *nonRetriableError + if errors.As(err, &nonRetriable) { + cmd.PrintErrln("Cluster initialization failed. This error is not recoverable.") + cmd.PrintErrln("Terminate your cluster and try again.") + if nonRetriable.logCollectionErr != nil { + cmd.PrintErrf("Failed to collect logs from bootstrapper: %s\n", nonRetriable.logCollectionErr) + } else { + cmd.PrintErrf("Fetched bootstrapper logs are stored in %q\n", a.flags.pathPrefixer.PrefixPrintablePath(constants.ErrorLog)) + } + } + return nil, err + } + a.log.Debugf("Initialization request successful") + + a.log.Debugf("Buffering init success message") + bufferedOutput := &bytes.Buffer{} + if err := a.writeOutput(stateFile, resp, a.flags.mergeConfigs, bufferedOutput, measurementSalt); err != nil { + return nil, err + } + + return bufferedOutput, nil +} + +// initCall performs the gRPC call to the bootstrapper to initialize the cluster. +func (a *applyCmd) initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitSuccessResponse, error) { + doer := &initDoer{ + dialer: dialer, + endpoint: net.JoinHostPort(ip, strconv.Itoa(constants.BootstrapperPort)), + req: req, + log: a.log, + spinner: a.spinner, + fh: file.NewHandler(afero.NewOsFs()), + } + + // Create a wrapper function that allows logging any returned error from the retrier before checking if it's the expected retriable one. + serviceIsUnavailable := func(err error) bool { + isServiceUnavailable := grpcRetry.ServiceIsUnavailable(err) + a.log.Debugf("Encountered error (retriable: %t): %s", isServiceUnavailable, err) + return isServiceUnavailable + } + + a.log.Debugf("Making initialization call, doer is %+v", doer) + retrier := retry.NewIntervalRetrier(doer, 30*time.Second, serviceIsUnavailable) + if err := retrier.Do(ctx); err != nil { + return nil, err + } + return doer.resp, nil +} + +// generateMasterSecret reads a base64 encoded master secret from file or generates a new 32 byte secret. +func (a *applyCmd) generateMasterSecret(outWriter io.Writer) (uri.MasterSecret, error) { + // No file given, generate a new secret, and save it to disk + key, err := crypto.GenerateRandomBytes(crypto.MasterSecretLengthDefault) + if err != nil { + return uri.MasterSecret{}, err + } + salt, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) + if err != nil { + return uri.MasterSecret{}, err + } + secret := uri.MasterSecret{ + Key: key, + Salt: salt, + } + if err := a.fileHandler.WriteJSON(constants.MasterSecretFilename, secret, file.OptNone); err != nil { + return uri.MasterSecret{}, err + } + fmt.Fprintf(outWriter, "Your Constellation master secret was successfully written to %q\n", a.flags.pathPrefixer.PrefixPrintablePath(constants.MasterSecretFilename)) + return secret, nil +} + +// writeOutput writes the output of a cluster initialization to the +// state- / kubeconfig-file and saves it to disk. +func (a *applyCmd) writeOutput( + stateFile *state.State, initResp *initproto.InitSuccessResponse, + mergeConfig bool, wr io.Writer, measurementSalt []byte, +) error { + fmt.Fprint(wr, "Your Constellation cluster was successfully initialized.\n\n") + + ownerID := hex.EncodeToString(initResp.GetOwnerId()) + clusterID := hex.EncodeToString(initResp.GetClusterId()) + + stateFile.SetClusterValues(state.ClusterValues{ + MeasurementSalt: measurementSalt, + OwnerID: ownerID, + ClusterID: clusterID, + }) + + tw := tabwriter.NewWriter(wr, 0, 0, 2, ' ', 0) + writeRow(tw, "Constellation cluster identifier", clusterID) + writeRow(tw, "Kubernetes configuration", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) + tw.Flush() + fmt.Fprintln(wr) + + a.log.Debugf("Rewriting cluster server address in kubeconfig to %s", stateFile.Infrastructure.ClusterEndpoint) + kubeconfig, err := clientcmd.Load(initResp.GetKubeconfig()) + if err != nil { + return fmt.Errorf("loading kubeconfig: %w", err) + } + if len(kubeconfig.Clusters) != 1 { + return fmt.Errorf("expected exactly one cluster in kubeconfig, got %d", len(kubeconfig.Clusters)) + } + for _, cluster := range kubeconfig.Clusters { + kubeEndpoint, err := url.Parse(cluster.Server) + if err != nil { + return fmt.Errorf("parsing kubeconfig server URL: %w", err) + } + kubeEndpoint.Host = net.JoinHostPort(stateFile.Infrastructure.ClusterEndpoint, kubeEndpoint.Port()) + cluster.Server = kubeEndpoint.String() + } + kubeconfigBytes, err := clientcmd.Write(*kubeconfig) + if err != nil { + return fmt.Errorf("marshaling kubeconfig: %w", err) + } + + if err := a.fileHandler.Write(constants.AdminConfFilename, kubeconfigBytes, file.OptNone); err != nil { + return fmt.Errorf("writing kubeconfig: %w", err) + } + a.log.Debugf("Kubeconfig written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) + + if mergeConfig { + if err := a.merger.mergeConfigs(constants.AdminConfFilename, a.fileHandler); err != nil { + writeRow(tw, "Failed to automatically merge kubeconfig", err.Error()) + mergeConfig = false // Set to false so we don't print the wrong message below. + } else { + writeRow(tw, "Kubernetes configuration merged with default config", "") + } + } + + if err := stateFile.WriteToFile(a.fileHandler, constants.StateFilename); err != nil { + return fmt.Errorf("writing Constellation state file: %w", err) + } + + a.log.Debugf("Constellation state file written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) + + if !mergeConfig { + fmt.Fprintln(wr, "You can now connect to your cluster by executing:") + + exportPath, err := filepath.Abs(constants.AdminConfFilename) + if err != nil { + return fmt.Errorf("getting absolute path to kubeconfig: %w", err) + } + + fmt.Fprintf(wr, "\texport KUBECONFIG=%q\n", exportPath) + } else { + fmt.Fprintln(wr, "Constellation kubeconfig merged with default config.") + + if a.merger.kubeconfigEnvVar() != "" { + fmt.Fprintln(wr, "Warning: KUBECONFIG environment variable is set.") + fmt.Fprintln(wr, "You may need to unset it to use the default config and connect to your cluster.") + } else { + fmt.Fprintln(wr, "You can now connect to your cluster.") + } + } + fmt.Fprintln(wr) // add final newline + return nil +} diff --git a/cli/internal/cmd/applyterraform.go b/cli/internal/cmd/applyterraform.go new file mode 100644 index 0000000000..68959d7f0b --- /dev/null +++ b/cli/internal/cmd/applyterraform.go @@ -0,0 +1,115 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" + "github.com/edgelesssys/constellation/v2/cli/internal/state" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/spf13/cobra" +) + +// runTerraformApply checks if changes to Terraform are required and applies them. +func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string) error { + a.log.Debugf("Checking if Terraform migrations are required") + migrationRequired, err := a.planTerraformMigration(cmd, conf) + if err != nil { + return fmt.Errorf("planning Terraform migrations: %w", err) + } + + if !migrationRequired { + a.log.Debugf("No changes to infrastructure required, skipping Terraform migrations") + return nil + } + + a.log.Debugf("Migrating terraform resources for infrastructure changes") + postMigrationInfraState, err := a.migrateTerraform(cmd, conf, upgradeDir) + if err != nil { + return fmt.Errorf("performing Terraform migrations: %w", err) + } + + // Merge the pre-upgrade state with the post-migration infrastructure values + a.log.Debugf("Updating state file with new infrastructure state") + if _, err := stateFile.Merge( + // temporary state with post-migration infrastructure values + state.New().SetInfrastructure(postMigrationInfraState), + ); err != nil { + return fmt.Errorf("merging pre-upgrade state with post-migration infrastructure values: %w", err) + } + + // Write the post-migration state to disk + if err := stateFile.WriteToFile(a.fileHandler, constants.StateFilename); err != nil { + return fmt.Errorf("writing state file: %w", err) + } + return nil +} + +// planTerraformMigration checks if the Constellation version the cluster is being upgraded to requires a migration. +func (a *applyCmd) planTerraformMigration(cmd *cobra.Command, conf *config.Config) (bool, error) { + a.log.Debugf("Planning Terraform migrations") + vars, err := cloudcmd.TerraformUpgradeVars(conf) + if err != nil { + return false, fmt.Errorf("parsing upgrade variables: %w", err) + } + a.log.Debugf("Using Terraform variables:\n%+v", vars) + + // Check if there are any Terraform migrations to apply + + // Add manual migrations here if required + // + // var manualMigrations []terraform.StateMigration + // for _, migration := range manualMigrations { + // u.log.Debugf("Adding manual Terraform migration: %s", migration.DisplayName) + // u.upgrader.AddManualStateMigration(migration) + // } + + return a.clusterUpgrader.PlanClusterUpgrade(cmd.Context(), cmd.OutOrStdout(), vars, conf.GetProvider()) +} + +// migrateTerraform migrates an existing Terraform state and the post-migration infrastructure state is returned. +func (a *applyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string) (state.Infrastructure, error) { + // Ask for confirmation first + fmt.Fprintln(cmd.OutOrStdout(), "The upgrade requires a migration of Constellation cloud resources by applying an updated Terraform template. Please manually review the suggested changes below.") + if !a.flags.yes { + ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?") + if err != nil { + return state.Infrastructure{}, fmt.Errorf("asking for confirmation: %w", err) + } + if !ok { + cmd.Println("Aborting upgrade.") + // User doesn't expect to see any changes in his workspace after aborting an "upgrade apply", + // therefore, roll back to the backed up state. + if err := a.clusterUpgrader.RestoreClusterWorkspace(); err != nil { + return state.Infrastructure{}, fmt.Errorf( + "restoring Terraform workspace: %w, restore the Terraform workspace manually from %s ", + err, + filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir), + ) + } + return state.Infrastructure{}, fmt.Errorf("cluster upgrade aborted by user") + } + } + a.log.Debugf("Applying Terraform migrations") + + a.spinner.Start("Migrating Terraform resources", false) + infraState, err := a.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider()) + a.spinner.Stop() + if err != nil { + return state.Infrastructure{}, fmt.Errorf("applying terraform migrations: %w", err) + } + + cmd.Printf("Infrastructure migrations applied successfully and output written to: %s\n"+ + "A backup of the pre-upgrade state has been written to: %s\n", + a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename), + a.flags.pathPrefixer.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)), + ) + return infraState, nil +}