diff --git a/examples/vcluster/vcluster_with_config/main_test.go b/examples/vcluster/vcluster_with_config/main_test.go new file mode 100644 index 00000000..7bef50a0 --- /dev/null +++ b/examples/vcluster/vcluster_with_config/main_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vcluster + +import ( + "context" + "log" + "os" + "testing" + + "sigs.k8s.io/e2e-framework/klient/conf" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/support" + "sigs.k8s.io/e2e-framework/support/kind" + "sigs.k8s.io/e2e-framework/third_party/vcluster" +) + +var testenv env.Environment + +func TestMain(m *testing.M) { + opts := []support.ClusterOpts{} + // vcluster requires a "host" cluster to install into, so we should resolve one + if os.Getenv("REAL_CLUSTER") == "true" { + cfg := conf.ResolveKubeConfigFile() + opts = append(opts, vcluster.WithHostKubeConfig(cfg)) + } else { + // create a kind cluster to use as the vcluster "host" + cfg, err := kind.NewProvider().WithName("kind-vc-host").Create(context.Background()) + if err != nil { + log.Fatal(err) + } + opts = append(opts, vcluster.WithHostKubeConfig(cfg)) + } + + testenv, _ = env.NewFromFlags() + vclusterName := envconf.RandomName("vcluster-with-config", 16) + namespace := envconf.RandomName("vcluster-ns", 16) + opts = append(opts, vcluster.WithNamespace(namespace)) + testenv.Setup( + envfuncs.CreateNamespace(namespace), + envfuncs.CreateClusterWithConfig(vcluster.NewProvider(), vclusterName, "values.yaml", opts...), + ) + + testenv.Finish( + envfuncs.DestroyCluster(vclusterName), + envfuncs.DeleteNamespace(namespace), + ) + + if os.Getenv("REAL_CLUSTER") != "true" { + // cleanup the vcluster "host"-kind-cluster + testenv.Finish( + func(ctx context.Context, c *envconf.Config) (context.Context, error) { + if err := kind.NewProvider().WithName("kind-vc-host").Destroy(ctx); err != nil { + return ctx, err + } + return ctx, nil + }, + ) + } + + os.Exit(testenv.Run(m)) +} diff --git a/examples/vcluster/vcluster_with_config/values.yaml b/examples/vcluster/vcluster_with_config/values.yaml new file mode 100644 index 00000000..28afa64a --- /dev/null +++ b/examples/vcluster/vcluster_with_config/values.yaml @@ -0,0 +1 @@ +# TODO: add vcluster configs diff --git a/go.mod b/go.mod index 904d9f5d..3fcd1e93 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( k8s.io/component-base v0.31.2 k8s.io/klog/v2 v2.130.1 sigs.k8s.io/controller-runtime v0.19.1 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -63,5 +64,4 @@ require ( k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/pkg/env/env.go b/pkg/env/env.go index df5c4523..50250eed 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -226,7 +226,7 @@ func (e *testEnv) processTestFeature(ctx context.Context, t *testing.T, featureN t.Helper() skipped, message := e.requireFeatureProcessing(feature) if skipped { - t.Skipf(message) + t.Skip(message) } // execute beforeEachFeature actions ctx = e.processFeatureActions(ctx, t, feature, e.getBeforeFeatureActions()) @@ -510,7 +510,7 @@ func (e *testEnv) execFeature(ctx context.Context, t *testing.T, featName string internalT.Helper() skipped, message := e.requireAssessmentProcessing(assess, i+1) if skipped { - internalT.Skipf(message) + internalT.Skip(message) } // Set shouldFailNow to true before actually running the assessment, because if the assessment // calls t.FailNow(), the function will be abruptly stopped in the middle of `e.executeSteps()`. diff --git a/third_party/flux/flux_setup.go b/third_party/flux/flux_setup.go index b8b460b9..6d5bd629 100644 --- a/third_party/flux/flux_setup.go +++ b/third_party/flux/flux_setup.go @@ -18,6 +18,7 @@ package flux import ( "context" + "errors" "fmt" "sigs.k8s.io/e2e-framework/pkg/env" @@ -44,7 +45,7 @@ func InstallFlux(opts ...Option) env.Func { func CreateGitRepo(gitRepoName, gitRepoURL string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.createSource(Git, gitRepoName, gitRepoURL, opts...) if err != nil { @@ -58,7 +59,7 @@ func CreateGitRepo(gitRepoName, gitRepoURL string, opts ...Option) env.Func { func CreateHelmRepository(name, url string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.createSource(Helm, name, url, opts...) if err != nil { @@ -72,7 +73,7 @@ func CreateHelmRepository(name, url string, opts ...Option) env.Func { func CreateKustomization(kustomizationName, sourceRef string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.createKustomization(kustomizationName, sourceRef, opts...) if err != nil { @@ -87,7 +88,7 @@ func CreateKustomization(kustomizationName, sourceRef string, opts ...Option) en func CreateHelmRelease(name, source, chart string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.createHelmRelease(name, source, chart, opts...) if err != nil { @@ -101,7 +102,7 @@ func CreateHelmRelease(name, source, chart string, opts ...Option) env.Func { func UninstallFlux(opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.uninstallFlux(opts...) if err != nil { @@ -115,7 +116,7 @@ func UninstallFlux(opts ...Option) env.Func { func DeleteKustomization(kustomizationName string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.deleteKustomization(kustomizationName, opts...) if err != nil { @@ -129,7 +130,7 @@ func DeleteKustomization(kustomizationName string, opts ...Option) env.Func { func DeleteHelmRelease(name string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.deleteHelmRelease(name, opts...) if err != nil { @@ -143,7 +144,7 @@ func DeleteHelmRelease(name string, opts ...Option) env.Func { func DeleteGitRepo(gitRepoName string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.deleteSource(Git, gitRepoName, opts...) if err != nil { @@ -157,7 +158,7 @@ func DeleteGitRepo(gitRepoName string, opts ...Option) env.Func { func DeleteHelmRepo(name string, opts ...Option) env.Func { return func(ctx context.Context, c *envconf.Config) (context.Context, error) { if manager == nil { - return ctx, fmt.Errorf(NoFluxInstallationFoundMsg) + return ctx, errors.New(NoFluxInstallationFoundMsg) } err := manager.deleteSource(Helm, name, opts...) if err != nil { diff --git a/third_party/helm/helm.go b/third_party/helm/helm.go index 93af15e9..4851b758 100644 --- a/third_party/helm/helm.go +++ b/third_party/helm/helm.go @@ -18,6 +18,7 @@ package helm import ( "bytes" + "errors" "fmt" "strings" @@ -237,7 +238,7 @@ func (m *Manager) run(opts *Opts) (err error) { } log.V(4).InfoS("Determining if helm binary is available or not", "executable", m.path) if m.e.Prog().Avail(m.path) == "" { - err = fmt.Errorf(missingHelm) + err = errors.New(missingHelm) return } command, err := m.getCommand(opts) diff --git a/third_party/ko/ko.go b/third_party/ko/ko.go index 469b8cfe..b6841186 100644 --- a/third_party/ko/ko.go +++ b/third_party/ko/ko.go @@ -19,6 +19,7 @@ package ko import ( "bytes" "context" + "errors" "fmt" "strings" @@ -159,7 +160,7 @@ func (m *Manager) GetLocalImage(ctx context.Context, packagePath string) (string func (m *Manager) run(opts *Opts) (out string, err error) { log.V(4).InfoS("Determining if ko binary is available or not", "executable", m.path) if m.e.Prog().Avail(m.path) == "" { - return "", fmt.Errorf(missingKo) + return "", errors.New(missingKo) } envs := m.getEnvs(opts) diff --git a/third_party/kubetest2/pkg/tester/e2e-tester_test.go b/third_party/kubetest2/pkg/tester/e2e-tester_test.go index 3c93f03d..f6661b05 100644 --- a/third_party/kubetest2/pkg/tester/e2e-tester_test.go +++ b/third_party/kubetest2/pkg/tester/e2e-tester_test.go @@ -17,8 +17,9 @@ limitations under the License. package tester import ( - "github.com/octago/sflags/gen/gpflag" "testing" + + "github.com/octago/sflags/gen/gpflag" ) func TestBuildFlags(t *testing.T) { diff --git a/third_party/vcluster/vcluster.go b/third_party/vcluster/vcluster.go new file mode 100644 index 00000000..993021e8 --- /dev/null +++ b/third_party/vcluster/vcluster.go @@ -0,0 +1,338 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vcluster + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/vladimirvivien/gexe" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + log "k8s.io/klog/v2" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/klient/conf" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/support" + "sigs.k8s.io/e2e-framework/support/utils" + "sigs.k8s.io/yaml" +) + +const ( + vclusterVersion = "v0.20.0" + vclusterPath = "vclusterctl" +) + +type Cluster struct { + path string + name string + kubecfgFile string // kubeconfig file for the vcluster + version string + namespace string // namespace to create the vcluster in + hostKubeCfg string // kubeconfig file for the host cluster + hostKubeContext string // kubeconfig context for the host cluster + rc *rest.Config +} + +// Enforce Type check always to avoid future breaks +var _ support.E2EClusterProvider = &Cluster{} + +func NewCluster(name string) *Cluster { + return &Cluster{name: name} +} + +func NewProvider() support.E2EClusterProvider { + return &Cluster{} +} + +func WithPath(path string) support.ClusterOpts { + return func(c support.E2EClusterProvider) { + v, ok := c.(*Cluster) + if ok { + v.WithPath(path) + } + } +} + +func WithNamespace(ns string) support.ClusterOpts { + return func(c support.E2EClusterProvider) { + v, ok := c.(*Cluster) + if ok { + v.namespace = ns + } + } +} + +func WithHostKubeConfig(kubeconfig string) support.ClusterOpts { + return func(c support.E2EClusterProvider) { + v, ok := c.(*Cluster) + if ok { + v.hostKubeCfg = kubeconfig + } + } +} + +func WithHostKubeContext(kubeContext string) support.ClusterOpts { + return func(c support.E2EClusterProvider) { + v, ok := c.(*Cluster) + if ok { + v.hostKubeContext = kubeContext + } + } +} + +func (c *Cluster) WithName(name string) support.E2EClusterProvider { + c.name = name + return c +} + +func (c *Cluster) WithVersion(version string) support.E2EClusterProvider { + c.version = version + return c +} + +func (c *Cluster) WithPath(path string) support.E2EClusterProvider { + c.path = path + return c +} + +func (c *Cluster) WithOpts(opts ...support.ClusterOpts) support.E2EClusterProvider { + for _, opt := range opts { + opt(c) + } + return c +} + +func (c *Cluster) SetDefaults() support.E2EClusterProvider { + if c.path == "" { + c.path = vclusterPath + } + return c +} + +func (c *Cluster) Create(ctx context.Context, args ...string) (string, error) { + log.V(4).Info("Creating vcluster ", c.name) + if err := c.findOrInstallVcluster(); err != nil { + return "", err + } + + if _, exists := c.clusterExists(c.name); exists { + log.V(4).Info("Skipping vcluster Cluster.Create: cluster already created: ", c.name) + kConfig, err := c.getKubeconfig() + if err != nil { + return "", err + } + return kConfig, c.initKubernetesAccessClients() + } + + if c.namespace != "" { + args = append(args, "--namespace", c.namespace) + } + + if c.hostKubeContext != "" { + args = append(args, "--context", c.hostKubeContext) + } + + command := fmt.Sprintf("%s create %s --connect=false --update-current=false", c.path, c.name) + if len(args) > 0 { + command = fmt.Sprintf("%s %s", command, strings.Join(args, " ")) + } + log.V(4).Info("Launching:", command) + echo := gexe.New() + if c.hostKubeCfg != "" { + echo.SetEnv("KUBECONFIG", c.hostKubeCfg) + } + + p := echo.RunProc(command) + if p.Err() != nil { + outBytes, err := io.ReadAll(p.Out()) + if err != nil { + log.ErrorS(err, "failed to read data from the vcluster create process output due to an error") + } + return "", fmt.Errorf("vcluster: failed to create cluster %q: %s: %s: %s", c.name, p.Err(), p.Result(), string(outBytes)) + } + clusters, ok := c.clusterExists(c.name) + if !ok { + return "", fmt.Errorf("vcluster Cluster.Create: cluster %v still not in 'cluster list' after creation: %v", c.name, clusters) + } + log.V(4).Info("vcluster clusters available: ", clusters) + + kConfig, err := c.getKubeconfig() + if err != nil { + return "", err + } + + return kConfig, nil +} + +func (c *Cluster) CreateWithConfig(ctx context.Context, configFile string) (string, error) { + var args []string + if configFile != "" { + args = append(args, "--values", configFile) + } + return c.Create(ctx, args...) +} + +func (c *Cluster) GetKubeconfig() string { + return c.kubecfgFile +} + +type kubeconfig struct { + CurrentContext string `json:"current-context"` +} + +func (c *Cluster) GetKubectlContext() string { + kc := &kubeconfig{} + raw, err := os.ReadFile(c.kubecfgFile) + if err != nil { + return "" + } + if err := yaml.Unmarshal(raw, kc); err != nil { + return "" + } + return kc.CurrentContext +} + +func (c *Cluster) ExportLogs(ctx context.Context, dest string) error { + // Not implemented + return nil +} + +func (c *Cluster) Destroy(ctx context.Context) error { + log.V(4).Info("Destroying vcluster ", c.name) + if err := c.findOrInstallVcluster(); err != nil { + return err + } + + command := fmt.Sprintf("%s delete %s", c.path, c.name) + p := utils.RunCommand(command) + if p.Err() != nil { + outBytes, err := io.ReadAll(p.Out()) + if err != nil { + log.ErrorS(err, "failed to read data from the vcluster delete process output due to an error") + } + return fmt.Errorf("vcluster: failed to delete cluster %q: %s: %s: %s", c.name, p.Err(), p.Result(), string(outBytes)) + } + + log.V(4).Info("Removing kubeconfig file ", c.kubecfgFile) + if err := os.Remove(c.kubecfgFile); err != nil { + return fmt.Errorf("vcluster: failed to remove kubeconfig file %q: %w", c.kubecfgFile, err) + } + return nil +} + +func (c *Cluster) WaitForControlPlane(ctx context.Context, client klient.Client) error { + r, err := resources.New(client.RESTConfig()) + if err != nil { + return err + } + for _, sl := range []metav1.LabelSelectorRequirement{ + {Key: "k8s-app", Operator: metav1.LabelSelectorOpIn, Values: []string{"kube-dns"}}, + } { + selector, err := metav1.LabelSelectorAsSelector( + &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + sl, + }, + }, + ) + if err != nil { + return err + } + err = wait.For(conditions.New(r).ResourceListN(&v1.PodList{}, len(sl.Values), resources.WithLabelSelector(selector.String()))) + if err != nil { + return err + } + } + return nil +} + +func (c *Cluster) KubernetesRestConfig() *rest.Config { + return c.rc +} + +// helpers to implement support.E2EClusterProvider +func (c *Cluster) findOrInstallVcluster() error { + version := c.version + if c.version == "" { + version = vclusterVersion + } + path, err := utils.FindOrInstallGoBasedProvider(c.path, vclusterPath, "github.com/loft-sh/vcluster/cmd/vclusterctl", version) + if path != "" { + c.path = path + } + return err +} + +type clusterItem struct { + Name string `json:"Name"` +} + +func (c *Cluster) clusterExists(name string) (string, bool) { + raw := utils.FetchCommandOutput(fmt.Sprintf("%s list --output json", c.path)) + clusters := []clusterItem{} + if err := json.Unmarshal([]byte(raw), &clusters); err != nil { + return raw, false + } + for _, c := range clusters { + if c.Name == name { + return raw, true + } + } + return raw, false +} + +func (c *Cluster) getKubeconfig() (string, error) { + kubecfg := fmt.Sprintf("%s-kubecfg", c.name) + + var stdout, stderr bytes.Buffer + err := utils.RunCommandWithSeperatedOutput(fmt.Sprintf(`%s connect %s --print`, c.path, c.name), &stdout, &stderr) + if err != nil { + return "", fmt.Errorf("vcluster connect: stderr: %s: %w", stderr.String(), err) + } + log.V(4).Info("vcluster connect stderr \n", stderr.String()) + + file, err := os.CreateTemp("", fmt.Sprintf("vcluster-cluster-%s", kubecfg)) + if err != nil { + return "", fmt.Errorf("vcluster kubeconfig file: %w", err) + } + defer file.Close() + + c.kubecfgFile = file.Name() + + if n, err := io.WriteString(file, stdout.String()); n == 0 || err != nil { + return "", fmt.Errorf("vcluster kubecfg file: bytes copied: %d: %w]", n, err) + } + return file.Name(), nil +} + +func (c *Cluster) initKubernetesAccessClients() error { + cfg, err := conf.New(c.kubecfgFile) + if err != nil { + return err + } + c.rc = cfg + return nil +} diff --git a/third_party/vcluster/vcluster_test.go b/third_party/vcluster/vcluster_test.go new file mode 100644 index 00000000..ba27133e --- /dev/null +++ b/third_party/vcluster/vcluster_test.go @@ -0,0 +1,11 @@ +package vcluster + +import "testing" + +func TestCluster_VclusterInstall(t *testing.T) { + c := Cluster{} + err := c.findOrInstallVcluster() + if err != nil { + t.Fatalf("Error installing vcluster: %v", err) + } +}