Skip to content

Commit

Permalink
use state file in CLI
Browse files Browse the repository at this point in the history 
Signed-off-by: Moritz Sanft <[email protected]>

take clusterConfig from IDFile for compat

Signed-off-by: Moritz Sanft <[email protected]>

various fixes

Signed-off-by: Moritz Sanft <[email protected]>

wip

Signed-off-by: Moritz Sanft <[email protected]>
  • Loading branch information
@msanft
msanft committed Sep 29, 2023
1 parent 361c60b commit 5fa9468
Show file tree
Hide file tree
Showing 10 changed files with 267 additions and 181 deletions.
4 changes: 2 additions & 2 deletions cli/internal/cmd/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,13 +172,13 @@ func (c *createCmd) create(cmd *cobra.Command, creator cloudCreator, fileHandler
}
c.log.Debugf("Successfully created the cloud resources for the cluster")

// TODO(msanft): Remove IDFile as per AB#3354
// TODO(msanft): Remove IDFile as per AB#3425
idFile := convertToIDFile(infraState, provider)
if err := fileHandler.WriteJSON(constants.ClusterIDsFilename, idFile, file.OptNone); err != nil {
return err
}

state := state.NewState().SetInfrastructure(infraState)
state := state.New().SetInfrastructure(infraState)
if err := state.WriteToFile(fileHandler, constants.StateFilename); err != nil {
return fmt.Errorf("writing state file: %w", err)
}
Expand Down
7 changes: 3 additions & 4 deletions cli/internal/cmd/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package cmd
import (
"bytes"
"context"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
Expand Down Expand Up @@ -157,7 +156,7 @@ func (i *initCmd) initialize(
cmd.PrintErrln("WARNING: Attestation temporarily relies on AWS nitroTPM. See https://docs.edgeless.systems/constellation/workflows/config#choosing-a-vm-type for more information.")
}

// TODO(msanft): Remove IDFile as per AB#3354
// TODO(msanft): Remove IDFile as per AB#3425
i.log.Debugf("Checking cluster ID file")
var idFile clusterid.File
if err := i.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
Expand Down Expand Up @@ -208,7 +207,7 @@ func (i *initCmd) initialize(
idFile.MeasurementSalt = measurementSalt

stateFile.SetClusterValues(state.ClusterValues{
MeasurementSalt: base64.StdEncoding.EncodeToString(measurementSalt),
MeasurementSalt: measurementSalt,
})

clusterName := clusterid.GetClusterName(conf, idFile)
Expand Down Expand Up @@ -246,7 +245,7 @@ func (i *initCmd) initialize(
}
i.log.Debugf("Initialization request succeeded")

// TODO(msanft): Remove IDFile as per AB#3354
// TODO(msanft): Remove IDFile as per AB#3425
i.log.Debugf("Writing Constellation ID file")
idFile.CloudProvider = provider

Expand Down
29 changes: 14 additions & 15 deletions cli/internal/cmd/init_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ func TestInitialize(t *testing.T) {
tc.configMutator(config)
}
require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, config, file.OptNone))
stateFile := state.NewState()
stateFile := state.New()
require.NoError(stateFile.WriteToFile(fileHandler, constants.StateFilename))
if tc.idFile != nil {
tc.idFile.CloudProvider = tc.provider
Expand Down Expand Up @@ -415,8 +415,12 @@ func TestWriteOutput(t *testing.T) {
}

expectedStateFile := &state.State{
Version: state.Version1,
ClusterValues: state.ClusterValues{ClusterID: clusterID, OwnerID: ownerID},
Version: state.Version1,
ClusterValues: state.ClusterValues{
ClusterID: clusterID,
OwnerID: ownerID,
MeasurementSalt: []byte{},
},
Infrastructure: state.Infrastructure{APIServerCertSANs: []string{}},
}

Expand All @@ -428,7 +432,7 @@ func TestWriteOutput(t *testing.T) {
UID: "test-uid",
IP: clusterEndpoint,
}
stateFile := state.NewState()
stateFile := state.New()

i := newInitCmd(fileHandler, &nopSpinner{}, &stubMerger{}, logger.NewTest(t))
err = i.writeOutput(idFile, stateFile, resp.GetInitSuccess(), false, &out)
Expand Down Expand Up @@ -793,23 +797,18 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
return res, err
}

type stubShowInfrastructure struct{}
type stubShowInfrastructure struct {
showInfraErr error
}

func (s *stubShowInfrastructure) ShowInfrastructure(_ context.Context, csp cloudprovider.Provider) (state.Infrastructure, error) {
res := state.Infrastructure{}
switch csp {
case cloudprovider.Azure:
res.Azure = &state.Azure{}
case cloudprovider.GCP:
res.GCP = &state.GCP{}
}
return res, nil
func (s *stubShowInfrastructure) ShowInfrastructure(context.Context, cloudprovider.Provider) (state.Infrastructure, error) {
return state.Infrastructure{}, s.showInfraErr
}

type stubAttestationApplier struct {
applyErr error
}

func (a *stubAttestationApplier) ApplyJoinConfig(_ context.Context, _ config.AttestationCfg, _ []byte) error {
func (a *stubAttestationApplier) ApplyJoinConfig(context.Context, config.AttestationCfg, []byte) error {
return a.applyErr
}
143 changes: 56 additions & 87 deletions cli/internal/cmd/upgradeapply.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,11 +176,18 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
return err
}

// TODO(msanft): Remove reading from idFile once v2.12.0 is released and read from state file directly.
// For now, this is only here to ensure upgradability from an id-file to a state file version.
var idFile clusterid.File
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
return fmt.Errorf("reading cluster ID file: %w", err)
}
conf.UpdateMAAURL(idFile.AttestationURL)

// Convert id-file to state file
stateFile := state.NewFromIDFile(idFile)
if stateFile.Infrastructure.Azure != nil {
conf.UpdateMAAURL(stateFile.Infrastructure.Azure.AttestationURL)
}

// Apply migrations necessary for the upgrade
if err := migrateFrom2_10(cmd.Context(), u.kubeUpgrader); err != nil {
Expand All @@ -190,32 +197,51 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
return fmt.Errorf("applying migration for upgrading from v2.11: %w", err)
}

if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), idFile.MeasurementSalt, flags); err != nil {
if err := u.confirmAndUpgradeAttestationConfig(cmd, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt, flags); err != nil {
return fmt.Errorf("upgrading measurements: %w", err)
}

// Perform Terraform migrations if infrastructure upgrade phase is not skipped.
// If it is skipped, we expect the new infrastructure to be in the state file
// already.
if !flags.skipPhases.contains(skipInfrastructurePhase) {
if err := u.migrateTerraform(cmd, conf, upgradeDir, flags); err != nil {
// If infrastructure phase is skipped, we expect the new infrastructure
// to be in the Terraform configuration already. Otherwise, perform
// the Terraform migrations.
var postMigrationInfraState state.Infrastructure
if flags.skipPhases.contains(skipInfrastructurePhase) {
postMigrationInfraState, err = u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("getting Terraform state: %w", err)
}
} else {
postMigrationInfraState, err = u.migrateTerraform(cmd, conf, upgradeDir, flags)
if err != nil {
return fmt.Errorf("performing Terraform migrations: %w", err)
}
}
// reload idFile after terraform migration
// it might have been updated by the migration
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
return fmt.Errorf("reading updated cluster ID file: %w", err)

// Merge the pre-upgrade state with the post-migration infrastructure values
if _, err := stateFile.Merge(
// temporary state with post-migration infrastructure values
state.New().SetInfrastructure(postMigrationInfraState),
); err != nil {
return fmt.Errorf("merging pre-upgrade state with post-migration infrastructure values: %w", err)
}
stateFile, err := state.ReadFromFile(u.fileHandler, constants.StateFilename)
if err != nil {
return fmt.Errorf("reading state file: %w", err)

// Write the post-migration state to disk
if err := stateFile.WriteToFile(u.fileHandler, constants.StateFilename); err != nil {
return fmt.Errorf("writing state file: %w", err)
}

cmd.Printf("Infrastructure migrations applied successfully and output written to: %s and %s\n"+
"A backup of the pre-upgrade state has been written to: %s\n",
flags.pf.PrefixPrintablePath(constants.StateFilename),
flags.pf.PrefixPrintablePath(constants.ClusterIDsFilename),
flags.pf.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)),
)

// extend the clusterConfig cert SANs with any of the supported endpoints:
// - (legacy) public IP
// - fallback endpoint
// - custom (user-provided) endpoint
// At this point, state file and id-file should have been merged, so we can use the state file.
sans := append([]string{stateFile.Infrastructure.ClusterEndpoint, conf.CustomEndpoint}, stateFile.Infrastructure.APIServerCertSANs...)
if err := u.kubeUpgrader.ExtendClusterConfigCertSANs(cmd.Context(), sans); err != nil {
return fmt.Errorf("extending cert SANs: %w", err)
Expand Down Expand Up @@ -269,14 +295,15 @@ func diffAttestationCfg(currentAttestationCfg config.AttestationCfg, newAttestat
}

// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration
// of cloud resources with Terraform. If so, the migration is performed and the output is written to the state / ID file.
func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
) error {
// of cloud resources with Terraform. If so, the migration is performed and the post-migration infrastructure state is returned.
func (u *upgradeApplyCmd) migrateTerraform(
cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
) (state.Infrastructure, error) {
u.log.Debugf("Planning Terraform migrations")

vars, err := cloudcmd.TerraformUpgradeVars(conf)
if err != nil {
return fmt.Errorf("parsing upgrade variables: %w", err)
return state.Infrastructure{}, fmt.Errorf("parsing upgrade variables: %w", err)
}
u.log.Debugf("Using Terraform variables:\n%v", vars)

Expand All @@ -292,7 +319,7 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf

hasDiff, err := u.clusterUpgrader.PlanClusterUpgrade(cmd.Context(), cmd.OutOrStdout(), vars, conf.GetProvider())
if err != nil {
return fmt.Errorf("planning terraform migrations: %w", err)
return state.Infrastructure{}, fmt.Errorf("planning terraform migrations: %w", err)
}

if hasDiff {
Expand All @@ -301,45 +328,35 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
if !flags.yes {
ok, err := askToConfirm(cmd, "Do you want to apply the Terraform migrations?")
if err != nil {
return fmt.Errorf("asking for confirmation: %w", err)
return state.Infrastructure{}, fmt.Errorf("asking for confirmation: %w", err)
}
if !ok {
cmd.Println("Aborting upgrade.")
// User doesn't expect to see any changes in his workspace after aborting an "upgrade apply",
// therefore, roll back to the backed up state.
if err := u.clusterUpgrader.RestoreClusterWorkspace(); err != nil {
return fmt.Errorf(
return state.Infrastructure{}, fmt.Errorf(
"restoring Terraform workspace: %w, restore the Terraform workspace manually from %s ",
err,
filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir),
)
}
return fmt.Errorf("cluster upgrade aborted by user")
return state.Infrastructure{}, fmt.Errorf("cluster upgrade aborted by user")
}
}
u.log.Debugf("Applying Terraform migrations")

infraState, err := u.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("applying terraform migrations: %w", err)
return state.Infrastructure{}, fmt.Errorf("applying terraform migrations: %w", err)
}

// Apply possible updates to cluster ID file
if err := updateClusterIDFile(infraState, u.fileHandler); err != nil {
return fmt.Errorf("merging cluster ID files: %w", err)
}

cmd.Printf("Terraform migrations applied successfully and output written to: %s and %s\n"+
"A backup of the pre-upgrade state has been written to: %s\n",
flags.pf.PrefixPrintablePath(constants.StateFilename),
flags.pf.PrefixPrintablePath(constants.ClusterIDsFilename),
flags.pf.PrefixPrintablePath(filepath.Join(upgradeDir, constants.TerraformUpgradeBackupDir)),
)
} else {
u.log.Debugf("No Terraform diff detected")
cmd.Printf("Terraform migrations applied successfully.")
return infraState, nil
}
u.log.Debugf("No Terraform diff detected")

return nil
u.log.Debugf("No Terraform diff detected")
return state.Infrastructure{}, nil
}

// validK8sVersion checks if the Kubernetes patch version is supported and asks for confirmation if not.
Expand Down Expand Up @@ -401,7 +418,7 @@ func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
if err := u.kubeUpgrader.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt); err != nil {
return fmt.Errorf("updating attestation config: %w", err)
}
cmd.Println("Successfully update the cluster's attestation config")
cmd.Println("Successfully updated the cluster's attestation config")
return nil
}

Expand Down Expand Up @@ -579,54 +596,6 @@ func parseUpgradeApplyFlags(cmd *cobra.Command) (upgradeApplyFlags, error) {
}, nil
}

func updateClusterIDFile(infraState state.Infrastructure, fileHandler file.Handler) error {
newIDFile := clusterid.File{
InitSecret: []byte(infraState.InitSecret),
IP: infraState.ClusterEndpoint,
APIServerCertSANs: infraState.APIServerCertSANs,
UID: infraState.UID,
}
if infraState.Azure != nil {
newIDFile.AttestationURL = infraState.Azure.AttestationURL
}
newStateFile := state.NewState()
newStateFile.SetInfrastructure(state.Infrastructure{
InitSecret: infraState.InitSecret,
ClusterEndpoint: infraState.ClusterEndpoint,
APIServerCertSANs: infraState.APIServerCertSANs,
UID: infraState.UID,
})
if infraState.Azure != nil {
newStateFile.Infrastructure.Azure = &state.Azure{
AttestationURL: infraState.Azure.AttestationURL,
}
}

idFile := &clusterid.File{}
if err := fileHandler.ReadJSON(constants.ClusterIDsFilename, idFile); err != nil {
return fmt.Errorf("reading %s: %w", constants.ClusterIDsFilename, err)
}

stateFile, err := state.ReadFromFile(fileHandler, constants.StateFilename)
if err != nil {
return fmt.Errorf("reading %s: %w", constants.StateFilename, err)
}

if err := fileHandler.WriteJSON(constants.ClusterIDsFilename, idFile.Merge(newIDFile), file.OptOverwrite); err != nil {
return fmt.Errorf("writing %s: %w", constants.ClusterIDsFilename, err)
}

_, err = stateFile.Merge(newStateFile)
if err != nil {
return fmt.Errorf("merging state: %w", err)
}
if err := stateFile.WriteToFile(fileHandler, constants.StateFilename); err != nil {
return fmt.Errorf("writing %s: %w", constants.StateFilename, err)
}

return nil
}

type upgradeApplyFlags struct {
pf pathprefix.PathPrefixer
yes bool
Expand Down
Loading

0 comments on commit 5fa9468

Please sign in to comment.