Skip to content

Commit

Permalink
use infra state in upgrade
Browse files Browse the repository at this point in the history
  • Loading branch information
elchead committed Sep 8, 2023
1 parent 9a3eb7a commit 61747c3
Show file tree
Hide file tree
Showing 17 changed files with 143 additions and 98 deletions.
1 change: 1 addition & 0 deletions cli/internal/cloudcmd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ go_test(
],
embed = [":cloudcmd"],
deps = [
"//cli/internal/state",
"//cli/internal/terraform",
"//internal/attestation/measurements",
"//internal/attestation/variant",
Expand Down
3 changes: 2 additions & 1 deletion cli/internal/cloudcmd/clients.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"context"
"io"

"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
Expand All @@ -33,7 +34,7 @@ type tfCommonClient interface {
type tfResourceClient interface {
tfCommonClient
ApplyCluster(ctx context.Context, provider cloudprovider.Provider, logLevel terraform.LogLevel) (terraform.ApplyOutput, error)
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error)
}

type tfIAMClient interface {
Expand Down
7 changes: 4 additions & 3 deletions cli/internal/cloudcmd/clients_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"io"
"testing"

"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
Expand All @@ -31,7 +32,7 @@ type stubTerraformClient struct {
iamOutput terraform.IAMOutput
uid string
attestationURL string
applyOutput terraform.ApplyOutput
infraState state.Infrastructure
cleanUpWorkspaceCalled bool
removeInstallerCalled bool
destroyCalled bool
Expand Down Expand Up @@ -77,9 +78,9 @@ func (c *stubTerraformClient) RemoveInstaller() {
c.removeInstallerCalled = true
}

func (c *stubTerraformClient) ShowCluster(_ context.Context, _ cloudprovider.Provider) (terraform.ApplyOutput, error) {
func (c *stubTerraformClient) ShowInfrastructure(_ context.Context, _ cloudprovider.Provider) (state.Infrastructure, error) {
c.showCalled = true
return c.applyOutput, c.showErr
return c.infraState, c.showErr
}

func (c *stubTerraformClient) ShowIAM(_ context.Context, _ cloudprovider.Provider) (terraform.IAMOutput, error) {
Expand Down
26 changes: 1 addition & 25 deletions cli/internal/cloudcmd/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,31 +112,7 @@ func (c *Creator) Create(ctx context.Context, opts CreateOptions) (state.Infrast
if err != nil {
return state.Infrastructure{}, fmt.Errorf("creating cluster: %w", err)
}
return convertToInfrastructure(tfOutput), nil
}

func convertToInfrastructure(applyOutput terraform.ApplyOutput) state.Infrastructure {
var infra state.Infrastructure
infra.UID = applyOutput.UID
infra.PublicIP = applyOutput.IP
infra.InitSecret = applyOutput.Secret
infra.APIServerCertSANs = applyOutput.APIServerCertSANs

if applyOutput.Azure != nil {
infra.Azure.ResourceGroup = applyOutput.Azure.ResourceGroup
infra.Azure.SubscriptionID = applyOutput.Azure.SubscriptionID
infra.Azure.NetworkSecurityGroupName = applyOutput.Azure.NetworkSecurityGroupName
infra.Azure.LoadBalancerName = applyOutput.Azure.LoadBalancerName
infra.Azure.UserAssignedIdentity = applyOutput.Azure.UserAssignedIdentity
infra.Azure.AttestationURL = applyOutput.Azure.AttestationURL
}

if applyOutput.GCP != nil {
infra.GCP.ProjectID = applyOutput.GCP.ProjectID
infra.GCP.IPCidrNode = applyOutput.GCP.IPCidrNode
infra.GCP.IPCidrPod = applyOutput.GCP.IPCidrPod
}
return infra
return terraform.ConvertToInfrastructure(tfOutput), nil
}

func (c *Creator) createAWS(ctx context.Context, cl tfResourceClient, opts CreateOptions) (tfOutput terraform.ApplyOutput, retErr error) {
Expand Down
19 changes: 11 additions & 8 deletions cli/internal/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/config"
Expand Down Expand Up @@ -76,12 +77,12 @@ type initCmd struct {
merger configMerger
spinner spinnerInterf
fileHandler file.Handler
clusterShower clusterShower
clusterShower infrastructureShower
pf pathprefix.PathPrefixer
}

func newInitCmd(
clusterShower clusterShower, fileHandler file.Handler,
clusterShower infrastructureShower, fileHandler file.Handler,
spinner spinnerInterf, merger configMerger, log debugLog,
) *initCmd {
return &initCmd{
Expand Down Expand Up @@ -259,9 +260,9 @@ func (i *initCmd) initialize(
return fmt.Errorf("applying attestation config: %w", err)
}

output, err := i.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
infraState, err := i.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return fmt.Errorf("getting Terraform output: %w", err)
return fmt.Errorf("getting infrastructure state: %w", err)
}

i.spinner.Start("Installing Kubernetes components ", false)
Expand All @@ -275,7 +276,7 @@ func (i *initCmd) initialize(
if err != nil {
return fmt.Errorf("creating Helm client: %w", err)
}
executor, includesUpgrades, err := helmApplier.PrepareApply(conf, k8sVersion, idFile, options, output,
executor, includesUpgrades, err := helmApplier.PrepareApply(conf, k8sVersion, idFile, options, infraState,
serviceAccURI, masterSecret)
if err != nil {
return fmt.Errorf("getting Helm chart executor: %w", err)
Expand Down Expand Up @@ -629,9 +630,11 @@ type attestationConfigApplier interface {
}

type helmApplier interface {
PrepareApply(conf *config.Config, validK8sversion versions.ValidK8sVersion, idFile clusterid.File, flags helm.Options, tfOutput terraform.ApplyOutput, serviceAccURI string, masterSecret uri.MasterSecret) (helm.Applier, bool, error)
PrepareApply(conf *config.Config, validK8sversion versions.ValidK8sVersion, idFile clusterid.File,
flags helm.Options, infra state.Infrastructure, serviceAccURI string, masterSecret uri.MasterSecret) (
helm.Applier, bool, error)
}

type clusterShower interface {
ShowCluster(ctx context.Context, provider cloudprovider.Provider) (terraform.ApplyOutput, error)
type infrastructureShower interface {
ShowInfrastructure(ctx context.Context, provider cloudprovider.Provider) (state.Infrastructure, error)
}
16 changes: 8 additions & 8 deletions cli/internal/cmd/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/atls"
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
Expand Down Expand Up @@ -186,7 +186,7 @@ func TestInitialize(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
i := newInitCmd(&stubShowCluster{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
i := newInitCmd(&stubShowInfrastructure{}, fileHandler, &nopSpinner{}, nil, logger.NewTest(t))
err := i.initialize(cmd, newDialer, &stubLicenseClient{}, stubAttestationFetcher{},
func(io.Writer, string, debugLog) (attestationConfigApplier, error) {
return &stubAttestationApplier{}, nil
Expand Down Expand Up @@ -222,7 +222,7 @@ type stubApplier struct {
err error
}

func (s stubApplier) PrepareApply(_ *config.Config, _ versions.ValidK8sVersion, _ clusterid.File, _ helm.Options, _ terraform.ApplyOutput, _ string, _ uri.MasterSecret) (helm.Applier, bool, error) {
func (s stubApplier) PrepareApply(_ *config.Config, _ versions.ValidK8sVersion, _ clusterid.File, _ helm.Options, _ state.Infrastructure, _ string, _ uri.MasterSecret) (helm.Applier, bool, error) {
return stubRunner{}, false, s.err
}

Expand Down Expand Up @@ -672,15 +672,15 @@ func (c stubInitClient) Recv() (*initproto.InitResponse, error) {
return res, err
}

type stubShowCluster struct{}
type stubShowInfrastructure struct{}

func (s *stubShowCluster) ShowCluster(_ context.Context, csp cloudprovider.Provider) (terraform.ApplyOutput, error) {
res := terraform.ApplyOutput{}
func (s *stubShowInfrastructure) ShowInfrastructure(_ context.Context, csp cloudprovider.Provider) (state.Infrastructure, error) {
res := state.Infrastructure{}
switch csp {
case cloudprovider.Azure:
res.Azure = &terraform.AzureApplyOutput{}
res.Azure = &state.Azure{}
case cloudprovider.GCP:
res.GCP = &terraform.GCPApplyOutput{}
res.GCP = &state.GCP{}
}
return res, nil
}
Expand Down
27 changes: 18 additions & 9 deletions cli/internal/cmd/upgradeapply.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/edgelesssys/constellation/v2/cli/internal/cmd/pathprefix"
"github.com/edgelesssys/constellation/v2/cli/internal/helm"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
Expand Down Expand Up @@ -122,7 +123,7 @@ type upgradeApplyCmd struct {
kubeUpgrader kubernetesUpgrader
clusterUpgrader clusterUpgrader
configFetcher attestationconfigapi.Fetcher
clusterShower clusterShower
clusterShower infrastructureShower
fileHandler file.Handler
log debugLog
}
Expand Down Expand Up @@ -172,10 +173,17 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
return fmt.Errorf("upgrading measurements: %w", err)
}

tfOutput, err := u.migrateTerraform(cmd, conf, upgradeDir, flags)
infraState, err := u.migrateTerraform(cmd, conf, upgradeDir, flags)
if err != nil {
return fmt.Errorf("performing Terraform migrations: %w", err)
}
state := state.State{
Version: "v1",
Infrastructure: infraState,
}
if err := u.fileHandler.WriteYAML(constants.StateFilename, state, file.OptOverwrite); err != nil {
return fmt.Errorf("writing state file: %w", err)
}
// reload idFile after terraform migration
// it might have been updated by the migration
if err := u.fileHandler.ReadJSON(constants.ClusterIDsFilename, &idFile); err != nil {
Expand All @@ -197,7 +205,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, upgradeDir string, fl
}

var upgradeErr *compatibility.InvalidUpgradeError
err = u.handleServiceUpgrade(cmd, conf, idFile, tfOutput, validK8sVersion, upgradeDir, flags)
err = u.handleServiceUpgrade(cmd, conf, idFile, infraState, validK8sVersion, upgradeDir, flags)
switch {
case errors.As(err, &upgradeErr):
cmd.PrintErrln(err)
Expand Down Expand Up @@ -237,7 +245,7 @@ func diffAttestationCfg(currentAttestationCfg config.AttestationCfg, newAttestat
// migrateTerraform checks if the Constellation version the cluster is being upgraded to requires a migration
// of cloud resources with Terraform. If so, the migration is performed.
func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Config, upgradeDir string, flags upgradeApplyFlags,
) (res terraform.ApplyOutput, err error) {
) (res state.Infrastructure, err error) {
u.log.Debugf("Planning Terraform migrations")

vars, err := cloudcmd.TerraformUpgradeVars(conf)
Expand Down Expand Up @@ -280,13 +288,14 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
}
u.log.Debugf("Applying Terraform migrations")
tfOutput, err := u.clusterUpgrader.ApplyClusterUpgrade(cmd.Context(), conf.GetProvider())
res = terraform.ConvertToInfrastructure(tfOutput)
if err != nil {
return tfOutput, fmt.Errorf("applying terraform migrations: %w", err)
return res, fmt.Errorf("applying terraform migrations: %w", err)
}

// Apply possible updates to cluster ID file
if err := updateClusterIDFile(tfOutput, u.fileHandler); err != nil {
return tfOutput, fmt.Errorf("merging cluster ID files: %w", err)
return res, fmt.Errorf("merging cluster ID files: %w", err)
}

cmd.Printf("Terraform migrations applied successfully and output written to: %s\n"+
Expand All @@ -298,7 +307,7 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, conf *config.Conf
u.log.Debugf("No Terraform diff detected")
}
u.log.Debugf("No Terraform diff detected")
tfOutput, err := u.clusterShower.ShowCluster(cmd.Context(), conf.GetProvider())
tfOutput, err := u.clusterShower.ShowInfrastructure(cmd.Context(), conf.GetProvider())
if err != nil {
return tfOutput, fmt.Errorf("getting Terraform output: %w", err)
}
Expand Down Expand Up @@ -369,7 +378,7 @@ func (u *upgradeApplyCmd) confirmAndUpgradeAttestationConfig(
}

func (u *upgradeApplyCmd) handleServiceUpgrade(
cmd *cobra.Command, conf *config.Config, idFile clusterid.File, tfOutput terraform.ApplyOutput,
cmd *cobra.Command, conf *config.Config, idFile clusterid.File, infra state.Infrastructure,
validK8sVersion versions.ValidK8sVersion, upgradeDir string, flags upgradeApplyFlags,
) error {
var secret uri.MasterSecret
Expand All @@ -389,7 +398,7 @@ func (u *upgradeApplyCmd) handleServiceUpgrade(
prepareApply := func(allowDestructive bool) (helm.Applier, bool, error) {
options.AllowDestructive = allowDestructive
executor, includesUpgrades, err := u.helmApplier.PrepareApply(conf, validK8sVersion, idFile, options,
tfOutput, serviceAccURI, secret)
infra, serviceAccURI, secret)
var upgradeErr *compatibility.InvalidUpgradeError
switch {
case errors.As(err, &upgradeErr):
Expand Down
12 changes: 11 additions & 1 deletion cli/internal/cmd/upgradeapply_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/kubecmd"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
Expand Down Expand Up @@ -124,7 +125,7 @@ func TestUpgradeApply(t *testing.T) {
clusterUpgrader: tc.terraformUpgrader,
log: logger.NewTest(t),
configFetcher: stubAttestationFetcher{},
clusterShower: &stubShowCluster{},
clusterShower: &stubShowInfrastructure{},
fileHandler: handler,
}

Expand All @@ -134,6 +135,15 @@ func TestUpgradeApply(t *testing.T) {
return
}
assert.NoError(err)

var gotState state.State
expectedState := state.Infrastructure{
APIServerCertSANs: []string{},
Azure: &state.Azure{},
}
require.NoError(handler.ReadYAML(constants.StateFilename, &gotState))
assert.Equal("v1", gotState.Version)
assert.Equal(expectedState, gotState.Infrastructure)
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions cli/internal/helm/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ go_library(
deps = [
"//cli/internal/clusterid",
"//cli/internal/helm/imageversion",
"//cli/internal/terraform",
"//cli/internal/state",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
"//internal/cloud/gcpshared",
Expand Down Expand Up @@ -459,7 +459,7 @@ go_test(
embed = [":helm"],
deps = [
"//cli/internal/clusterid",
"//cli/internal/terraform",
"//cli/internal/state",
"//internal/attestation/measurements",
"//internal/cloud/azureshared",
"//internal/cloud/cloudprovider",
Expand Down
14 changes: 9 additions & 5 deletions cli/internal/helm/helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (
"fmt"

"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
"github.com/edgelesssys/constellation/v2/cli/internal/state"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kms/uri"
Expand Down Expand Up @@ -87,8 +87,10 @@ type Options struct {

// PrepareApply loads the charts and returns the executor to apply them.
// TODO(elchead): remove validK8sVersion by putting ValidK8sVersion into config.Config, see AB#3374.
func (h Client) PrepareApply(conf *config.Config, validK8sversion versions.ValidK8sVersion, idFile clusterid.File, flags Options, tfOutput terraform.ApplyOutput, serviceAccURI string, masterSecret uri.MasterSecret) (Applier, bool, error) {
releases, err := h.loadReleases(conf, masterSecret, validK8sversion, idFile, flags, tfOutput, serviceAccURI)
func (h Client) PrepareApply(conf *config.Config, validK8sversion versions.ValidK8sVersion, idFile clusterid.File,
flags Options, infra state.Infrastructure, serviceAccURI string, masterSecret uri.MasterSecret,
) (Applier, bool, error) {
releases, err := h.loadReleases(conf, masterSecret, validK8sversion, idFile, flags, infra, serviceAccURI)
if err != nil {
return nil, false, fmt.Errorf("loading Helm releases: %w", err)
}
Expand All @@ -97,11 +99,13 @@ func (h Client) PrepareApply(conf *config.Config, validK8sversion versions.Valid
return &ChartApplyExecutor{actions: actions, log: h.log}, includesUpgrades, err
}

func (h Client) loadReleases(conf *config.Config, secret uri.MasterSecret, validK8sVersion versions.ValidK8sVersion, idFile clusterid.File, flags Options, tfOutput terraform.ApplyOutput, serviceAccURI string) ([]Release, error) {
func (h Client) loadReleases(conf *config.Config, secret uri.MasterSecret, validK8sVersion versions.ValidK8sVersion,
idFile clusterid.File, flags Options, infra state.Infrastructure, serviceAccURI string,
) ([]Release, error) {
helmLoader := newLoader(conf, idFile, validK8sVersion, h.cliVersion)
h.log.Debugf("Created new Helm loader")
return helmLoader.loadReleases(flags.Conformance, flags.HelmWaitMode, secret,
serviceAccURI, tfOutput)
serviceAccURI, infra)
}

// Applier runs the Helm actions.
Expand Down
Loading

0 comments on commit 61747c3

Please sign in to comment.