110 feature sync

bob/aggregate.go

@@ -186,7 +186,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	aggregate.Dependencies = make([]string, 0)
 	aggregate.Dependencies = append(aggregate.Dependencies, allDeps...)
 
-	// Initialize remote store in case of a valid remote url / project name
+	// Initialize remote store for artifacts and sync in case of a valid remote url / project name
 	if aggregate.Project != "" {
 		projectName, err := project.Parse(aggregate.Project)
 		if err != nil {
@@ -213,6 +213,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 			} else {
 				boblog.Log.V(1).Info(fmt.Sprintf("Using remote store: %s", url.String()))
 				aggregate.SetRemotestore(bobfile.NewRemotestore(url, b.allowInsecure, authCtx.Token))
+				aggregate.SetVersionedSyncStore(bobfile.NewVersionedSyncStore(url, b.allowInsecure, authCtx.Token))
 			}
 		}
 	} else {
@@ -245,6 +246,11 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 		}
 	}
 
+	for _, sync := range aggregate.SyncCollections {
+		err = sync.Validate(aggregate.Dir())
+		errz.Fatal(err)
+	}
+
 	err = aggregate.Verify()
 	errz.Fatal(err)
 
@@ -255,7 +261,7 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	err = aggregate.BTasks.FilterInputs()
 	errz.Fatal(err)
 
-	return aggregate, nil
+	return aggregate, aggregate.Verify()
 }
 
 func addTaskPrefix(prefix, taskname string) string {

bob/bobfile/bobfile.go

@@ -8,6 +8,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/benchkram/bob/pkg/versionedsync/remotesyncstore"
+
+	"github.com/benchkram/bob/bobsync"
 	"github.com/benchkram/bob/pkg/nix"
 	storeclient "github.com/benchkram/bob/pkg/store-client"
 
@@ -42,6 +45,8 @@ var (
 
 	ErrInvalidRunType = fmt.Errorf("Invalid run type")
 
+	ErrDuplicateSyncPath = fmt.Errorf("found duplicate sync path in bob.yaml")
+
 	ProjectNameFormatHint = "project name should be in the form 'project' or 'registry.com/user/project'"
 )
 
@@ -70,6 +75,9 @@ type Bobfile struct {
 	// Nixpkgs specifies an optional nixpkgs source.
 	Nixpkgs string `yaml:"nixpkgs"`
 
+	// SyncCollections are folder synchronisations through bob-server
+	SyncCollections bobsync.SyncList `yaml:"syncCollections"`
+
 	// Parent directory of the Bobfile.
 	// Populated through BobfileRead().
 	dir string
@@ -78,6 +86,8 @@ type Bobfile struct {
 
 	RemoteStoreHost string
 	remotestore     store.Store
+
+	versionedSyncStore *remotesyncstore.S
 }
 
 func NewBobfile() *Bobfile {
@@ -105,6 +115,14 @@ func (b *Bobfile) Remotestore() store.Store {
 	return b.remotestore
 }
 
+func (b *Bobfile) SetVersionedSyncStore(syncStore *remotesyncstore.S) {
+	b.versionedSyncStore = syncStore
+}
+
+func (b *Bobfile) VersionedSyncStore() *remotesyncstore.S {
+	return b.versionedSyncStore
+}
+
 // bobfileRead reads a bobfile and initializes private fields.
 func bobfileRead(dir string) (_ *Bobfile, err error) {
 	defer errz.Recover(&err)
@@ -236,6 +254,30 @@ func NewRemotestore(endpoint *url.URL, allowInsecure bool, token string) (s stor
 	return s
 }
 
+func NewVersionedSyncStore(endpoint *url.URL, allowInsecure bool, token string) (s *remotesyncstore.S) {
+	const sep = "/"
+
+	parts := strings.Split(strings.TrimLeft(endpoint.Path, sep), sep)
+
+	username := parts[0]
+	proj := strings.Join(parts[1:], sep)
+
+	protocol := "https://"
+	if allowInsecure {
+		protocol = "http://"
+	}
+
+	s = remotesyncstore.New(
+		username,
+		proj,
+
+		remotesyncstore.WithClient(
+			storeclient.New(protocol+endpoint.Host, token),
+		),
+	)
+	return s
+}
+
 // BobfileRead read from a bobfile.
 // Calls sanitize on the result.
 func BobfileRead(dir string) (_ *Bobfile, err error) {
@@ -326,6 +368,17 @@ func (b *Bobfile) Validate() (err error) {
 		}
 	}
 
+	// no duplicate paths allowed
+	existingPaths := make(map[string]int)
+	for _, sync := range b.SyncCollections {
+		err = sync.Validate(b.Dir())
+		errz.Fatal(err)
+		existingPaths[sync.Path]++
+		if existingPaths[sync.Path] > 1 {
+			return usererror.Wrapm(ErrDuplicateSyncPath, fmt.Sprintf("invalid collection path %s", sync.Path))
+		}
+	}
+
 	return nil
 }
 

bob/sync.go

@@ -0,0 +1,170 @@
+package bob
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/benchkram/bob/bobsync"
+	"github.com/benchkram/bob/pkg/boblog"
+	"github.com/benchkram/bob/pkg/usererror"
+	"github.com/benchkram/bob/pkg/versionedsync/localsyncstore"
+	"github.com/benchkram/bob/pkg/versionedsync/remotesyncstore"
+	"github.com/logrusorgru/aurora"
+	"os"
+	"sort"
+
+	"github.com/benchkram/bob/bob/bobfile"
+	"github.com/benchkram/errz"
+)
+
+func (b *B) SyncCreatePush(ctx context.Context, collectionName, version, path string, dry bool) (err error) {
+	defer errz.Recover(&err)
+
+	wd, _ := os.Getwd()
+	_, err = bobfile.BobfileRead(wd)
+	errz.Fatal(err)
+	aggregate, err := b.Aggregate()
+	errz.Fatal(err)
+
+	remoteStore := aggregate.VersionedSyncStore()
+	localStore := localsyncstore.New()
+
+	if remoteStore == nil {
+		fmt.Println(aurora.Red("No remote project configured can not push"))
+	} else {
+		// create only sync object and check if exists and is inside bobdir
+		// TODO: check if it conflicts with any git tracked file
+		sync, err := bobsync.NewSync(collectionName, version, path, aggregate.Dir())
+		errz.Fatal(err)
+		// check if path is the same as in another sync in bob.yaml
+		err = bobsync.CheckForConflicts(aggregate.SyncCollections, *sync)
+		if errors.Is(err, bobsync.ErrSyncPathTaken) {
+			errz.Fatal(usererror.Wrap(err))
+		} else {
+			errz.Fatal(err)
+		}
+		// create collection on remote, if name-version exists on remote => fail
+		err = sync.CreateOnRemote(ctx, *remoteStore, dry)
+		errz.Fatal(err)
+		// TODO: automatically add sync to bob file
+		// add it to the bobfile and check for conflicts
+		// 		if path exists => replace sync entry in bobfile
+		// err = aggregate.AddSync(sync)
+		// errz.Fatal(err)
+		fmt.Printf("Add that to your bob file so that others can use it:\n"+
+			"syncCollections:\n  - name: %s\n    version: %s\n    path: %s\n", sync.Name, sync.Version, sync.Path)
+
+		err = sync.Push(ctx, *remoteStore, *localStore, aggregate.Dir(), dry)
+		if err != nil {
+			boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from local to remote [collection: %s@%s]", sync.Name, sync.Version))
+
+		}
+	}
+
+	return nil
+}
+
+func (b *B) SyncPull(ctx context.Context, force bool) (err error) {
+	defer errz.Recover(&err)
+
+	wd, _ := os.Getwd()
+	_, err = bobfile.BobfileRead(wd)
+	errz.Fatal(err)
+	aggregate, err := b.Aggregate()
+	errz.Fatal(err)
+
+	remoteStore := aggregate.VersionedSyncStore()
+	localStore := localsyncstore.New()
+
+	if remoteStore == nil {
+		fmt.Println(aurora.Red("no remote project configured can not pull"))
+	} else {
+		for _, sync := range aggregate.SyncCollections {
+			err = sync.Pull(ctx, *remoteStore, *localStore, aggregate.Dir(), force)
+			if err != nil {
+				boblog.Log.V(1).Error(err, fmt.Sprintf("failed to sync from remote to local [collection: %s@%s]", sync.Name, sync.Version))
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *B) SyncListLocal(_ context.Context) (err error) {
+	defer errz.Recover(&err)
+
+	wd, _ := os.Getwd()
+	aggregate, err := bobfile.BobfileRead(wd)
+	errz.Fatal(err)
+
+	// call validate separate since aggregate is not called
+	for _, sync := range aggregate.SyncCollections {
+		err = sync.Validate(aggregate.Dir())
+		if err != nil {
+			errz.Fatal(usererror.Wrapm(err, "can not validate defined sync"))
+		}
+	}
+
+	for _, sync := range aggregate.SyncCollections {
+		err = sync.ListLocal(aggregate.Dir())
+		if err != nil {
+			boblog.Log.V(1).Error(err, fmt.Sprintf("failed list local [collection: %s@%s]", sync.Name, sync.Version))
+		}
+	}
+
+	return nil
+}
+
+func (b *B) SyncListRemote(ctx context.Context) (err error) {
+	defer errz.Recover(&err)
+
+	wd, _ := os.Getwd()
+	_, err = bobfile.BobfileRead(wd)
+	errz.Fatal(err)
+	aggregate, err := b.Aggregate()
+	errz.Fatal(err)
+
+	remoteStore := aggregate.VersionedSyncStore()
+
+	if remoteStore == nil {
+		fmt.Println(aurora.Red("No remote project configured can not list remote"))
+	} else {
+		for i := range aggregate.SyncCollections {
+			err = aggregate.SyncCollections[i].FillRemoteId(ctx, *remoteStore)
+			if err != nil {
+				if errors.Is(err, remotesyncstore.ErrCollectionNotFound) {
+					boblog.Log.V(1).Error(err, fmt.Sprintf("sync collection %s@%s does not exist on the server",
+						aggregate.SyncCollections[i].Name, aggregate.SyncCollections[i].Version))
+				} else {
+					errz.Fatal(err)
+				}
+			}
+		}
+
+		collections, err := (*remoteStore).Collections(ctx)
+		errz.Fatal(err)
+		for _, c := range collections {
+			sort.Sort(c)
+			fmt.Printf("%s@%s", aurora.Bold(c.Name), aurora.Italic(c.Version))
+			referenced, path := func() (bool, string) {
+				for _, sync := range aggregate.SyncCollections {
+					if c.ID == sync.GetRemoteId() {
+						return true, sync.Path
+					}
+				}
+				return false, ""
+			}()
+			if referenced {
+				fmt.Printf(" [referenced in bob.yaml] (./%s)", path)
+			}
+			fmt.Println()
+			for _, f := range c.Files {
+				fmt.Printf("\t%s\n", f.LocalPath)
+			}
+
+		}
+
+	}
+
+	return nil
+}