Implement remove command

baton/baton_test.go

@@ -0,0 +1,243 @@
+/*******************************************************************************
+ * Copyright (c) 2025 Genome Research Ltd.
+ *
+ * Author: Rosie Kern <[email protected]>
+ * Author: Iaroslav Popov <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ ******************************************************************************/
+
+package baton
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+	"github.com/wtsi-hgi/ibackup/internal"
+)
+
+func TestBaton(t *testing.T) {
+	h, errgbh := GetBatonHandler()
+	if errgbh != nil {
+		t.Logf("GetBatonHandler error: %s", errgbh)
+		SkipConvey("Skipping baton tests since couldn't find baton", t, func() {})
+
+		return
+	}
+
+	remotePath := os.Getenv("IBACKUP_TEST_COLLECTION")
+	if remotePath == "" {
+		SkipConvey("Skipping baton tests since IBACKUP_TEST_COLLECTION is not defined", t, func() {})
+
+		return
+	}
+
+	resetIRODS(remotePath)
+
+	localPath := t.TempDir()
+
+	Convey("Given clients on a baton handler", t, func() {
+		err := h.InitClients()
+		So(err, ShouldBeNil)
+
+		So(h.removeClient.IsRunning(), ShouldBeTrue)
+		So(h.metaClient.IsRunning(), ShouldBeTrue)
+		So(h.putClient.IsRunning(), ShouldBeTrue)
+
+		meta := map[string]string{
+			"ibackup:test:a": "1",
+			"ibackup:test:b": "2",
+		}
+
+		Convey("And a local file", func() {
+			file1local := filepath.Join(localPath, "file1")
+			file1remote := filepath.Join(remotePath, "file1")
+
+			internal.CreateTestFileOfLength(t, file1local, 1)
+
+			Convey("You can stat a file not in iRODS", func() {
+				exists, _, errs := h.Stat(file1remote)
+				So(errs, ShouldBeNil)
+				So(exists, ShouldBeFalse)
+			})
+
+			Convey("You can put the file in iRODS", func() {
+				err = h.Put(file1local, file1remote)
+				So(err, ShouldBeNil)
+
+				So(isObjectInIRODS(remotePath, "file1"), ShouldBeTrue)
+
+				Convey("And putting a new file in the same location will overwrite existing object", func() {
+					file2local := filepath.Join(localPath, "file2")
+					sizeOfFile2 := 10
+
+					internal.CreateTestFileOfLength(t, file2local, sizeOfFile2)
+
+					err = h.Put(file2local, file1remote)
+					So(err, ShouldBeNil)
+
+					So(getSizeOfObject(file1remote), ShouldEqual, sizeOfFile2)
+				})
+
+				Convey("And then you can add metadata to it", func() {
+					errm := h.AddMeta(file1remote, meta)
+					So(errm, ShouldBeNil)
+
+					So(getRemoteMeta(file1remote), ShouldContainSubstring, "ibackup:test:a")
+					So(getRemoteMeta(file1remote), ShouldContainSubstring, "ibackup:test:b")
+				})
+
+				Convey("And given metadata on the file in iRODS", func() {
+					for k, v := range meta {
+						addRemoteMeta(file1remote, k, v)
+					}
+
+					Convey("You can get the metadata from a file in iRODS", func() {
+						fileMeta, errm := h.GetMeta(file1remote)
+						So(errm, ShouldBeNil)
+
+						So(fileMeta, ShouldResemble, meta)
+					})
+
+					Convey("You can stat a file and get its metadata", func() {
+						exists, fileMeta, errm := h.Stat(file1remote)
+						So(errm, ShouldBeNil)
+						So(exists, ShouldBeTrue)
+						So(fileMeta, ShouldResemble, meta)
+					})
+
+					Convey("You can query if files contain specific metadata", func() {
+						files, errq := h.QueryMeta(remotePath, map[string]string{"ibackup:test:a": "1"})
+						So(errq, ShouldBeNil)
+
+						So(len(files), ShouldEqual, 1)
+						So(files, ShouldContain, file1remote)
+
+						files, errq = h.QueryMeta(remotePath, map[string]string{"ibackup:test:a": "2"})
+						So(errq, ShouldBeNil)
+
+						So(len(files), ShouldEqual, 0)
+					})
+
+					Convey("You can remove specific metadata from a file in iRODS", func() {
+						errm := h.RemoveMeta(file1remote, map[string]string{"ibackup:test:a": "1"})
+						So(errm, ShouldBeNil)
+
+						fileMeta, errm := h.GetMeta(file1remote)
+						So(errm, ShouldBeNil)
+
+						So(fileMeta, ShouldResemble, map[string]string{"ibackup:test:b": "2"})
+					})
+				})
+
+				Convey("And then you can remove it from iRODS", func() {
+					err = h.RemoveFile(file1remote)
+					So(err, ShouldBeNil)
+
+					So(isObjectInIRODS(remotePath, "file1"), ShouldBeFalse)
+				})
+			})
+		})
+
+		Convey("You can open collection clients and put an empty dir in iRODS", func() {
+			dir1remote := filepath.Join(remotePath, "dir1")
+			err = h.EnsureCollection(dir1remote)
+			So(err, ShouldBeNil)
+
+			So(h.collPool.IsOpen(), ShouldBeTrue)
+
+			for _, client := range h.collClients {
+				So(client.IsRunning(), ShouldBeTrue)
+			}
+
+			So(isObjectInIRODS(remotePath, "dir1"), ShouldBeTrue)
+
+			Convey("Then you can close the collection clients", func() {
+				err = h.CollectionsDone()
+				So(err, ShouldBeNil)
+
+				So(h.collPool.IsOpen(), ShouldBeFalse)
+				So(len(h.collClients), ShouldEqual, 0)
+
+				Convey("And then you can remove the dir from iRODS", func() {
+					err = h.RemoveDir(dir1remote)
+					So(err, ShouldBeNil)
+
+					So(isObjectInIRODS(remotePath, "dir1"), ShouldBeFalse)
+				})
+			})
+		})
+
+		Convey("You can close those clients", func() {
+			h.CloseClients()
+
+			So(h.AllClientsStopped(), ShouldBeTrue)
+		})
+	})
+}
+
+func resetIRODS(remotePath string) {
+	if remotePath == "" {
+		return
+	}
+
+	exec.Command("irm", "-r", remotePath).Run() //nolint:errcheck
+
+	exec.Command("imkdir", remotePath).Run() //nolint:errcheck
+}
+
+func isObjectInIRODS(remotePath, name string) bool {
+	output, err := exec.Command("ils", remotePath).CombinedOutput()
+	So(err, ShouldBeNil)
+
+	return strings.Contains(string(output), name)
+}
+
+func getRemoteMeta(path string) string {
+	output, err := exec.Command("imeta", "ls", "-d", path).CombinedOutput()
+	So(err, ShouldBeNil)
+
+	return string(output)
+}
+
+func addRemoteMeta(path, key, val string) {
+	output, err := exec.Command("imeta", "add", "-d", path, key, val).CombinedOutput()
+	if strings.Contains(string(output), "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME") {
+		return
+	}
+
+	So(err, ShouldBeNil)
+}
+
+func getSizeOfObject(path string) int {
+	output, err := exec.Command("ils", "-l", path).CombinedOutput()
+	So(err, ShouldBeNil)
+
+	cols := strings.Fields(string(output))
+	size, err := strconv.Atoi(cols[3])
+	So(err, ShouldBeNil)
+
+	return size
+}

cmd/filestatus.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/dustin/go-humanize" //nolint:misspell
 	"github.com/spf13/cobra"
+	"github.com/wtsi-hgi/ibackup/baton"
 	"github.com/wtsi-hgi/ibackup/put"
 	"github.com/wtsi-hgi/ibackup/set"
 	"github.com/wtsi-npg/extendo/v2"
@@ -107,7 +108,7 @@ func newFSG(db *set.DBRO, filePath string, useIRods bool) *fileStatusGetter {
 		return fsg
 	}
 
-	put.GetBatonHandler() //nolint:errcheck
+	baton.GetBatonHandler() //nolint:errcheck
 
 	if client, err := extendo.FindAndStart("--unbuffered", "--no-error"); err != nil {
 		warn("error occurred invoking baton; disabling irods mode: %s", err)

cmd/put.go

@@ -37,6 +37,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/wtsi-hgi/ibackup/baton"
 	"github.com/wtsi-hgi/ibackup/put"
 	"github.com/wtsi-hgi/ibackup/server"
 )
@@ -247,7 +248,7 @@ func handlePut(client *server.Client, requests []*put.Request) (chan *put.Reques
 }
 
 func getPutter(requests []*put.Request) (*put.Putter, func()) {
-	handler, err := put.GetBatonHandler()
+	handler, err := baton.GetBatonHandler()
 	if err != nil {
 		die("%s", err)
 	}
@@ -301,7 +302,7 @@ func createCollection(p *put.Putter) {
 func handleManualMode() {
 	requests, err := getRequestsFromFile(putFile, putMeta, putBase64)
 	if err != nil {
-		die(err.Error())
+		die("%s", err.Error())
 	}
 
 	_, uploadResults, skipResults, dfunc := handlePut(nil, requests)