Merge pull request #84 from ibuildthecloud/master

Remove gitrepo directory structure restrictions

charts/fleet/templates/configmap.yaml

@@ -11,7 +11,7 @@ data:
       "apiServerCA": "{{b64enc .Values.apiServerCA}}",
       "ignoreClusterRegistrationLabels": {{.Values.ignoreClusterRegistrationLabels}},
       "bootstrap": {
-        "dirs": "{{.Values.bootstrap.dirs}}",
+        "paths": "{{.Values.bootstrap.paths}}",
         "repo": "{{.Values.bootstrap.repo}}",
         "secret": "{{.Values.bootstrap.secret}}",
         "branch":  "{{.Values.bootstrap.branch}}",

charts/fleet/values.yaml

@@ -36,4 +36,4 @@ bootstrap:
   repo: ""
   secret: ""
   branch: master
-  dirs: ""
+  paths: ""

go.mod

@@ -14,12 +14,13 @@ require (
 	github.com/hashicorp/go-getter v1.4.1
 	github.com/pkg/errors v0.9.1
 	github.com/rancher/fleet/pkg/apis v0.0.0
-	github.com/rancher/gitjob v0.1.0
+	github.com/rancher/gitjob v0.1.1
 	github.com/rancher/lasso v0.0.0-20200820172840-0e4cc0ef5cb0
-	github.com/rancher/wrangler v0.6.2-0.20200912225020-2e02d61f54bc
+	github.com/rancher/wrangler v0.6.2-0.20200916052259-24781350894c
 	github.com/rancher/wrangler-cli v0.0.0-20200815040857-81c48cf8ab43
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
+	go.mozilla.org/sops/v3 v3.6.1
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	helm.sh/helm/v3 v3.0.0
 	k8s.io/api v0.18.8

modules/agent/pkg/controllers/controllers.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/rancher/wrangler/pkg/ticker"
+
 	"github.com/rancher/fleet/modules/agent/pkg/controllers/bundledeployment"
 	"github.com/rancher/fleet/modules/agent/pkg/controllers/cluster"
 	"github.com/rancher/fleet/modules/agent/pkg/controllers/secret"
@@ -76,7 +78,7 @@ func (a *appContext) start(ctx context.Context) error {
 }
 
 func Register(ctx context.Context, leaderElect bool, fleetNamespace, agentNamespace, defaultNamespace, clusterNamespace, clusterName string, fleetConfig *rest.Config, clientConfig clientcmd.ClientConfig) error {
-	appCtx, err := newContext(fleetNamespace, agentNamespace, clusterNamespace, clusterName, fleetConfig, clientConfig)
+	appCtx, err := newContext(ctx, fleetNamespace, agentNamespace, clusterNamespace, clusterName, fleetConfig, clientConfig)
 	if err != nil {
 		return err
 	}
@@ -126,7 +128,7 @@ func Register(ctx context.Context, leaderElect bool, fleetNamespace, agentNamesp
 	return nil
 }
 
-func newContext(fleetNamespace, agentNamespace, clusterNamespace, clusterName string, fleetConfig *rest.Config, clientConfig clientcmd.ClientConfig) (*appContext, error) {
+func newContext(ctx context.Context, fleetNamespace, agentNamespace, clusterNamespace, clusterName string, fleetConfig *rest.Config, clientConfig clientcmd.ClientConfig) (*appContext, error) {
 	client, err := clientConfig.ClientConfig()
 	if err != nil {
 		return nil, err
@@ -184,6 +186,13 @@ func newContext(fleetNamespace, agentNamespace, clusterNamespace, clusterName st
 	}
 
 	cache := memory.NewMemCacheClient(k8s.Discovery())
+	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cache)
+	go func() {
+		for range ticker.Context(ctx, 30*time.Second) {
+			cache.Invalidate()
+			restMapper.Reset()
+		}
+	}()
 
 	return &appContext{
 		Dynamic:          dynamic,
@@ -201,7 +210,7 @@ func newContext(fleetNamespace, agentNamespace, clusterNamespace, clusterName st
 		clientConfig:             clientConfig,
 		restConfig:               client,
 		cachedDiscoveryInterface: cache,
-		restMapper:               restmapper.NewDeferredDiscoveryRESTMapper(cache),
+		restMapper:               restMapper,
 		starters: []start.Starter{
 			core,
 			coreNSed,

modules/agent/pkg/deployer/deployer.go

@@ -12,9 +12,14 @@ type Resources struct {
 	Objects          []runtime.Object `json:"objects,omitempty"`
 }
 
+type DeployedBundle struct {
+	BundleID    string
+	ReleaseName string
+}
+
 type Deployer interface {
 	Deploy(bundleID string, manifest *manifest.Manifest, options fleet.BundleDeploymentOptions) (*Resources, error)
-	ListDeployments() ([]string, error)
+	ListDeployments() ([]DeployedBundle, error)
 	Resources(bundleID, resourcesID string) (*Resources, error)
-	Delete(bundleID string) error
+	Delete(bundleID, releaseName string) error
 }

modules/agent/pkg/deployer/manager.go

@@ -38,20 +38,38 @@ func NewManager(fleetNamespace string,
 	}
 }
 
+func releaseName(bd *fleet.BundleDeployment) string {
+	ns := "default"
+	if bd.Spec.Options.DefaultNamespace != "" {
+		ns = bd.Spec.Options.DefaultNamespace
+	}
+	if bd.Spec.Options.Helm == nil || bd.Spec.Options.Helm.ReleaseName == "" {
+		return ns + "/" + bd.Name
+	}
+	return ns + "/" + bd.Spec.Options.Helm.ReleaseName
+}
+
 func (m *Manager) Cleanup() error {
-	ids, err := m.deployer.ListDeployments()
+	deployed, err := m.deployer.ListDeployments()
 	if err != nil {
 		return err
 	}
 
-	for _, id := range ids {
-		_, err := m.bundleDeploymentCache.Get(m.fleetNamespace, id)
+	for _, deployed := range deployed {
+		bundleDeployment, err := m.bundleDeploymentCache.Get(m.fleetNamespace, deployed.BundleID)
 		if apierror.IsNotFound(err) {
-			if err := m.deployer.Delete(id); err != nil {
+			if err := m.deployer.Delete(deployed.BundleID, ""); err != nil {
 				return err
 			}
 		} else if err != nil {
 			return err
+		} else {
+			releaseName := releaseName(bundleDeployment)
+			if releaseName != deployed.ReleaseName {
+				if err := m.deployer.Delete(deployed.BundleID, deployed.ReleaseName); err != nil {
+					return err
+				}
+			}
 		}
 	}
 
@@ -60,7 +78,7 @@ func (m *Manager) Cleanup() error {
 
 func (m *Manager) Delete(bundleDeploymentKey string) error {
 	_, name := kv.RSplit(bundleDeploymentKey, "/")
-	return m.deployer.Delete(name)
+	return m.deployer.Delete(name, "")
 }
 
 func (m *Manager) Resources(bd *fleet.BundleDeployment) (*Resources, error) {
@@ -69,8 +87,7 @@ func (m *Manager) Resources(bd *fleet.BundleDeployment) (*Resources, error) {
 		return nil, nil
 	}
 
-	plan, err := m.getApply(bd, resources.DefaultNamespace).
-		DryRun(resources.Objects...)
+	plan, err := m.plan(bd, resources.DefaultNamespace, resources.Objects...)
 	if err != nil {
 		return nil, err
 	}

modules/agent/pkg/deployer/monitor.go

@@ -1,13 +1,16 @@
 package deployer
 
 import (
+	"encoding/json"
 	"sort"
 
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/wrangler/pkg/apply"
 	"github.com/rancher/wrangler/pkg/name"
 	"github.com/rancher/wrangler/pkg/summary"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 type DeploymentStatus struct {
@@ -17,26 +20,49 @@ type DeploymentStatus struct {
 	ModifiedStatus []fleet.ModifiedStatus `json:"modifiedStatus,omitempty"`
 }
 
-func (m *Manager) getApply(bd *fleet.BundleDeployment, ns string) apply.Apply {
+func (m *Manager) plan(bd *fleet.BundleDeployment, ns string, objs ...runtime.Object) (apply.Plan, error) {
+	a, err := m.getApply(bd, ns)
+	if err != nil {
+		return apply.Plan{}, err
+	}
+	return a.DryRun(objs...)
+}
+
+func (m *Manager) getApply(bd *fleet.BundleDeployment, ns string) (apply.Apply, error) {
+	apply := m.apply
 	if ns == "" {
 		ns = m.defaultNamespace
 	}
-	return m.apply.
+
+	if bd.Spec.Options.Diff != nil {
+		for _, compare := range bd.Spec.Options.Diff.ComparePatches {
+			for _, op := range compare.Operations {
+				// compile each operation by itself so that one failing operation doesn't block the others
+				patch, err := json.Marshal([]interface{}{op})
+				if err != nil {
+					return nil, err
+				}
+				gvk := schema.FromAPIVersionAndKind(compare.APIVersion, compare.Kind)
+				apply = apply.WithDiffPatch(gvk, compare.Namespace, compare.Name, patch)
+			}
+		}
+	}
+
+	return apply.
 		WithIgnorePreviousApplied().
 		WithSetID(name.SafeConcatName(m.labelPrefix, bd.Name)).
-		WithDefaultNamespace(ns)
+		WithDefaultNamespace(ns), nil
 }
 
 func (m *Manager) MonitorBundle(bd *fleet.BundleDeployment) (DeploymentStatus, error) {
 	var status DeploymentStatus
 
 	resources, err := m.deployer.Resources(bd.Name, bd.Status.Release)
 	if err != nil {
-		return status, nil
+		return status, err
 	}
 
-	plan, err := m.getApply(bd, resources.DefaultNamespace).
-		DryRun(resources.Objects...)
+	plan, err := m.plan(bd, resources.DefaultNamespace, resources.Objects...)
 	if err != nil {
 		return status, err
 	}

modules/agent/pkg/register/register.go

@@ -93,6 +93,8 @@ func tryRegister(ctx context.Context, namespace, clusterID string, config *rest.
 		return nil, err
 	}
 
+	// delete the bootstrap cred
+	_ = k8s.Core().V1().Secret().Delete(namespace, BootstrapCredName, nil)
 	return &AgentInfo{
 		ClusterNamespace: string(secret.Data[ClusterNamespace]),
 		ClusterName:      string(secret.Data[ClusterName]),

modules/cli/apply/apply.go

@@ -71,13 +71,28 @@ func Apply(ctx context.Context, client *client.Getter, name string, baseDirs []s
 					return err
 				}
 			}
-			if err := Dir(ctx, client, name, baseDir, opts); err == ErrNoResources {
-				logrus.Warnf("%s: %v", baseDir, err)
-				continue
-			} else if err != nil {
+			err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
+				// always consider the root valid
+				if baseDir != path {
+					if !info.IsDir() {
+						return nil
+					}
+					if _, err := os.Stat(filepath.Join(path, "fleet.yaml")); err != nil {
+						return nil
+					}
+				}
+				if err := Dir(ctx, client, name, path, opts); err == ErrNoResources {
+					logrus.Warnf("%s: %v", path, err)
+					return nil
+				} else if err != nil {
+					return err
+				}
+				foundBundle = true
+				return nil
+			})
+			if err != nil {
 				return err
 			}
-			foundBundle = true
 		}
 	}
 

modules/cli/cmds/apply.go

@@ -22,7 +22,6 @@ type Apply struct {
 	BundleInputArgs
 	OutputArgsNoDefault
 	Label          map[string]string `usage:"Labels to apply to created bundles" short:"l"`
-	File           string            `usage:"Read full bundle contents from file" short:"f"`
 	TargetsFile    string            `usage:"Addition source of targets and restrictions to be append"`
 	Compress       bool              `usage:"Force all resources to be compress" short:"c"`
 	ServiceAccount string            `usage:"Service account to assign to bundle created" short:"a"`

modules/cli/cmds/root.go

@@ -48,7 +48,8 @@ func (r *Fleet) PersistentPre(cmd *cobra.Command, args []string) error {
 }
 
 type BundleInputArgs struct {
-	BundleFile string `usage:"Location of the bundle.yaml" short:"b"`
+	File       string `usage:"Location of the fleet.yaml" short:"f"`
+	BundleFile string `usage:"Location of the raw Bundle resource yaml" short:"b"`
 }
 
 type OutputArgs struct {

modules/cli/cmds/test.go

@@ -42,6 +42,7 @@ func (m *Test) Run(cmd *cobra.Command, args []string) error {
 	opts := &match.Options{
 		Output:             os.Stdout,
 		BaseDir:            baseDir,
+		BundleSpec:         m.File,
 		BundleFile:         m.BundleFile,
 		ClusterGroup:       m.Group,
 		ClusterLabels:      m.Label,

modules/cli/match/match.go

@@ -6,17 +6,20 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 
+	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
+	"github.com/rancher/fleet/pkg/bundle"
 	"github.com/rancher/fleet/pkg/helmdeployer"
+	"github.com/rancher/fleet/pkg/options"
 	"github.com/rancher/wrangler/pkg/yaml"
-
-	"github.com/rancher/fleet/pkg/bundle"
 )
 
 type Options struct {
 	Output             io.Writer
 	BaseDir            string
+	BundleSpec         string
 	BundleFile         string
 	ClusterGroup       string
 	ClusterLabels      map[string]string
@@ -29,22 +32,44 @@ func Match(ctx context.Context, opts *Options) error {
 		opts = &Options{}
 	}
 
-	bundle, err := bundle.Open(ctx, "test", opts.BaseDir, opts.BundleFile, nil)
-	if err != nil {
-		return err
+	var (
+		b   *bundle.Bundle
+		err error
+	)
+
+	if opts.BundleFile == "" {
+		b, err = bundle.Open(ctx, "test", opts.BaseDir, opts.BundleSpec, nil)
+		if err != nil {
+			return err
+		}
+	} else {
+		data, err := ioutil.ReadFile(opts.BundleFile)
+		if err != nil {
+			return err
+		}
+
+		bundleConfig := &fleet.Bundle{}
+		if err := yaml.Unmarshal(data, bundleConfig); err != nil {
+			return err
+		}
+
+		b, err = bundle.New(bundleConfig)
+		if err != nil {
+			return err
+		}
 	}
 
 	if opts.Target == "" {
-		m := bundle.Match(map[string]map[string]string{
+		m := b.Match(map[string]map[string]string{
 			opts.ClusterGroup: opts.ClusterGroupLabels,
 		}, opts.ClusterLabels)
-		return printMatch(m, opts.Output)
+		return printMatch(b, m, opts.Output)
 	}
 
-	return printMatch(bundle.MatchForTarget(opts.Target), opts.Output)
+	return printMatch(b, b.MatchForTarget(opts.Target), opts.Output)
 }
 
-func printMatch(m *bundle.Match, output io.Writer) error {
+func printMatch(bundle *bundle.Bundle, m *bundle.Match, output io.Writer) error {
 	if m == nil {
 		return errors.New("no match found")
 	}
@@ -53,12 +78,17 @@ func printMatch(m *bundle.Match, output io.Writer) error {
 		return nil
 	}
 
+	opts, err := options.Calculate(&bundle.Definition.Spec, m.Target)
+	if err != nil {
+		return err
+	}
+
 	manifest, err := m.Manifest()
 	if err != nil {
 		return err
 	}
 
-	objs, err := helmdeployer.Template(m.Bundle.Definition.Name, manifest, m.Target.BundleDeploymentOptions)
+	objs, err := helmdeployer.Template(m.Bundle.Definition.Name, manifest, opts)
 	if err != nil {
 		return err
 	}