From 200ae79c3a28c1edc39f7e02a0487d00c76cd442 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 15 Jan 2025 16:31:04 +0000 Subject: [PATCH 01/35] Add implentation to remove files and dirs from a set --- cmd/remove.go | 125 +++++++++++++++++++++++++++++++++++++++++++++++ main_test.go | 97 ++++++++++++++++++++++++++++++++++++ server/client.go | 8 +++ server/setdb.go | 43 ++++++++++++++++ set/db.go | 35 +++++++++++++ set/set_test.go | 17 +++++++ 6 files changed, 325 insertions(+) create mode 100644 cmd/remove.go diff --git a/cmd/remove.go b/cmd/remove.go new file mode 100644 index 0000000..d68830e --- /dev/null +++ b/cmd/remove.go @@ -0,0 +1,125 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Authors: + * - Rosie Kern + * - Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package cmd + +import ( + "path/filepath" + + "github.com/spf13/cobra" + "github.com/wtsi-hgi/ibackup/server" +) + +// options for this cmd. +var removeUser string +var removeName string +var removeItems string +var removePath string + +// removeCmd represents the add command. +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove files from backed up set", + Long: `Remove files from backed up set. + + Remove files from a backed up set by providing the files or directories to be + removed. This will remove files from the set and from iRODS if it is not found + in any other sets. + + `, + Run: func(cmd *cobra.Command, args []string) { + ensureURLandCert() + + if (removeItems == "") == (removePath == "") { + die("only one of --items or --path can be provided") + } + + var files []string + var dirs []string + + if removeItems != "" { + filesAndDirs := readPaths(removeItems, fofnLineSplitter(setNull)) + files, dirs = categorisePaths(filesAndDirs, files, dirs) + } + + if removePath != "" { + removePath, err := filepath.Abs(removePath) + if err != nil { + die("%s", err.Error()) + } + + if pathIsDir(removePath) { + dirs = append(dirs, removePath) + } else { + files = append(files, removePath) + } + } + + client, err := newServerClient(serverURL, serverCert) + if err != nil { + die("%s", err.Error()) + } + + handleRemove(client, removeUser, removeName, files, dirs) + }, +} + +func init() { + RootCmd.AddCommand(removeCmd) + + // flags specific to this sub-command + removeCmd.Flags().StringVar(&removeUser, "user", currentUsername(), + "pretend to be this user (only works if you started the server)") + removeCmd.Flags().StringVarP(&removeName, "name", "n", "", "remove files from the set with this name") + removeCmd.Flags().StringVarP(&removeItems, "items", "i", "", + "path to file with one absolute local directory or file path per line") + removeCmd.Flags().StringVarP(&removePath, "path", "p", "", + "path to a single file or directory you wish to remove") + + if err := removeCmd.MarkFlagRequired("name"); err != nil { + die("%s", err.Error()) + } +} + +func handleRemove(client *server.Client, user, name string, files, dirs []string) { + sets := getSetByName(client, user, name) + if len(sets) == 0 { + warn("No backup sets found with name %s", name) + + return + } + + err := client.RemoveFiles(sets[0].ID(), files) + if err != nil { + die("%s", err.Error()) + } + + err = client.RemoveDirs(sets[0].ID(), dirs) + if err != nil { + die("%s", err.Error()) + } +} diff --git a/main_test.go b/main_test.go index b19f993..07e2aba 100644 --- a/main_test.go +++ b/main_test.go @@ -275,6 +275,16 @@ func (s *TestServer) confirmOutputContains(t *testing.T, args []string, expected So(actual, ShouldContainSubstring, expected) } +func (s *TestServer) confirmOutputDoesNotContain(t *testing.T, args []string, expectedCode int, + expected string) { + t.Helper() + + exitCode, actual := s.runBinary(t, args...) + + So(exitCode, ShouldEqual, expectedCode) + So(actual, ShouldNotContainSubstring, expected) +} + var ErrStatusNotFound = errors.New("status not found") func (s *TestServer) addSetForTesting(t *testing.T, name, transformer, path string) { @@ -1872,3 +1882,90 @@ func confirmFileContents(file, expectedContents string) { So(string(data), ShouldEqual, expectedContents) } + +func TestRemove(t *testing.T) { + Convey("Given a server", t, func() { + remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") + if remotePath == "" { + SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) + + return + } + + schedulerDeployment := os.Getenv("IBACKUP_TEST_SCHEDULER") + if schedulerDeployment == "" { + SkipConvey("skipping iRODS backup test since IBACKUP_TEST_SCHEDULER not set", func() {}) + + return + } + + dir := t.TempDir() + s := new(TestServer) + s.prepareFilePaths(dir) + s.prepareConfig() + + s.schedulerDeployment = schedulerDeployment + s.backupFile = filepath.Join(dir, "db.bak") + + s.startServer() + + path := t.TempDir() + transformer := "prefix=" + path + ":" + remotePath + + Convey("And an added set with files and folders", func() { + dir1 := filepath.Join(path, "path/to/some/dir/") + dir2 := filepath.Join(path, "path/to/other/dir/") + + tempTestFileOfPaths, err := os.CreateTemp(dir, "testFileSet") + So(err, ShouldBeNil) + + err = os.MkdirAll(dir1, 0755) + So(err, ShouldBeNil) + + err = os.MkdirAll(dir2, 0755) + So(err, ShouldBeNil) + + file1 := filepath.Join(path, "file1") + file2 := filepath.Join(path, "file2") + + internal.CreateTestFile(t, file1, "some data1") + internal.CreateTestFile(t, file2, "some data2") + + _, err = io.WriteString(tempTestFileOfPaths, + fmt.Sprintf("%s\n%s\n%s\n%s", file1, file2, dir1, dir2)) + So(err, ShouldBeNil) + + setName := "testRemoveFiles" + + exitCode, _ := s.runBinary(t, "add", "--items", tempTestFileOfPaths.Name(), + "--name", setName, "--transformer", transformer) + So(exitCode, ShouldEqual, 0) + + Convey("Remove removes the file from the set", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) + + So(exitCode, ShouldEqual, 0) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, file2) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, file1) + }) + + Convey("Remove removes the dir from the set", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", dir1) + + So(exitCode, ShouldEqual, 0) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, dir2) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1) + }) + + }) + //TODO add tests for failed files + }) +} diff --git a/server/client.go b/server/client.go index 7216bf9..6b038df 100644 --- a/server/client.go +++ b/server/client.go @@ -614,3 +614,11 @@ func (c *Client) RetryFailedSetUploads(id string) (int, error) { return retried, err } + +func (c *Client) RemoveFiles(setID string, files []string) error { + return c.putThing(EndPointAuthRemoveFiles+"/"+setID, stringsToBytes(files)) +} + +func (c *Client) RemoveDirs(setID string, dirs []string) error { + return c.putThing(EndPointAuthRemoveDirs+"/"+setID, stringsToBytes(dirs)) +} diff --git a/server/setdb.go b/server/setdb.go index 7dad838..33aee13 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -54,6 +54,8 @@ const ( workingPath = "/working" fileStatusPath = "/file_status" fileRetryPath = "/retry" + removeFilesPath = "/remove_files" + removeDirsPath = "/remove_dirs" // EndPointAuthSet is the endpoint for getting and setting sets. EndPointAuthSet = gas.EndPointAuth + setPath @@ -90,6 +92,12 @@ const ( // EndPointAuthRetryEntries is the endpoint for retrying file uploads. EndPointAuthRetryEntries = gas.EndPointAuth + fileRetryPath + // + EndPointAuthRemoveFiles = gas.EndPointAuth + removeFilesPath + + // + EndPointAuthRemoveDirs = gas.EndPointAuth + removeDirsPath + ErrNoAuth = gas.Error("auth must be enabled") ErrNoSetDBDirFound = gas.Error("set database directory not found") ErrNoRequester = gas.Error("requester not supplied") @@ -268,6 +276,9 @@ func (s *Server) addDBEndpoints(authGroup *gin.RouterGroup) { authGroup.PUT(fileStatusPath, s.putFileStatus) authGroup.GET(fileRetryPath+idParam, s.retryFailedEntries) + + authGroup.PUT(removeFilesPath+idParam, s.removeFiles) + authGroup.PUT(removeDirsPath+idParam, s.removeDirs) } // putSet interprets the body as a JSON encoding of a set.Set and stores it in @@ -368,6 +379,38 @@ func (s *Server) putFiles(c *gin.Context) { c.Status(http.StatusOK) } +func (s *Server) removeFiles(c *gin.Context) { + sid, paths, ok := s.bindPathsAndValidateSet(c) + if !ok { + return + } + + err := s.db.RemoveFileEntries(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + c.Status(http.StatusOK) +} + +func (s *Server) removeDirs(c *gin.Context) { + sid, paths, ok := s.bindPathsAndValidateSet(c) + if !ok { + return + } + + err := s.db.RemoveDirEntries(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + c.Status(http.StatusOK) +} + // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id // from the URL parameter if Requester matches logged-in username. func (s *Server) bindPathsAndValidateSet(c *gin.Context) (string, []string, bool) { diff --git a/set/db.go b/set/db.go index e7fb799..e58b967 100644 --- a/set/db.go +++ b/set/db.go @@ -253,6 +253,41 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } +// RemoveFileEntries removes the provided files from a given set. +func (d *DB) RemoveFileEntries(setID string, paths []string) error { + err := d.removeEntries(setID, paths, fileBucket) + if err != nil { + return err + } + + return d.removeEntries(setID, paths, discoveredBucket) +} + +// removeEntries removes the entries with the provided entry keys from a given +// bucket of a given set. +func (d *DB) removeEntries(setID string, entryKeys []string, bucketName string) error { + return d.db.Update(func(tx *bolt.Tx) error { + subBucketName := []byte(bucketName + separator + setID) + setsBucket := tx.Bucket([]byte(setsBucket)) + + entriesBucket := setsBucket.Bucket(subBucketName) + + for _, v := range entryKeys { + err := entriesBucket.Delete([]byte(v)) + if err != nil { + return err + } + } + + return nil + }) +} + +// RemoveDirEntries removes the provided directories from a given set. +func (d *DB) RemoveDirEntries(setID string, paths []string) error { + return d.removeEntries(setID, paths, dirBucket) +} + // SetFileEntries sets the file paths for the given backup set. Only supply // absolute paths to files. func (d *DB) SetFileEntries(setID string, paths []string) error { diff --git a/set/set_test.go b/set/set_test.go index f105897..89c0134 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -357,6 +357,23 @@ func TestSetDB(t *testing.T) { err = db.SetFileEntries(set2.ID(), []string{"/a/b.txt", "/c/k.txt"}) So(err, ShouldBeNil) + Convey("Then remove files and dirs from the sets", func() { + db.RemoveFileEntries(set.ID(), []string{"/a/b.txt"}) + + fEntries, errg := db.GetFileEntries(set.ID()) + So(errg, ShouldBeNil) + So(len(fEntries), ShouldEqual, 2) + So(fEntries[0], ShouldResemble, &Entry{Path: "/c/d.txt"}) + So(fEntries[1], ShouldResemble, &Entry{Path: "/e/f.txt"}) + + db.RemoveDirEntries(set.ID(), []string{"/g/h"}) + + dEntries, errg := db.GetDirEntries(set.ID()) + So(errg, ShouldBeNil) + So(len(dEntries), ShouldEqual, 1) + So(dEntries[0], ShouldResemble, &Entry{Path: "/g/i"}) + }) + Convey("Then get a particular Set", func() { retrieved := db.GetByID(set.ID()) So(retrieved, ShouldNotBeNil) From 2d73f4d9333f1fd15deb9a3ede81aad25f680573 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 17 Jan 2025 15:56:00 +0000 Subject: [PATCH 02/35] Add implementation to remove metadata and files from iRODS --- cmd/remove.go | 1 + main_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++--- put/baton.go | 84 ++++++++++++++++++++++++++++++++++++ server/server.go | 7 +++ server/setdb.go | 36 ++++++++++++++++ 5 files changed, 232 insertions(+), 6 deletions(-) diff --git a/cmd/remove.go b/cmd/remove.go index d68830e..0e9af5e 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -122,4 +122,5 @@ func handleRemove(client *server.Client, user, name string, files, dirs []string if err != nil { die("%s", err.Error()) } + } diff --git a/main_test.go b/main_test.go index 07e2aba..eb1b7e2 100644 --- a/main_test.go +++ b/main_test.go @@ -297,6 +297,16 @@ func (s *TestServer) addSetForTesting(t *testing.T, name, transformer, path stri s.waitForStatus(name, "\nDiscovery: completed", 5*time.Second) } +func (s *TestServer) addSetForTestingWithItems(t *testing.T, name, transformer, path string) { + t.Helper() + + exitCode, _ := s.runBinary(t, "add", "--name", name, "--transformer", transformer, "--items", path) + + So(exitCode, ShouldEqual, 0) + + s.waitForStatus(name, "\nStatus: complete", 5*time.Second) +} + func (s *TestServer) addSetForTestingWithFlag(t *testing.T, name, transformer, path, flag, data string) { t.Helper() @@ -1913,6 +1923,10 @@ func TestRemove(t *testing.T) { transformer := "prefix=" + path + ":" + remotePath Convey("And an added set with files and folders", func() { + dir := t.TempDir() + + linkPath := filepath.Join(path, "link") + symPath := filepath.Join(path, "sym") dir1 := filepath.Join(path, "path/to/some/dir/") dir2 := filepath.Join(path, "path/to/other/dir/") @@ -1927,19 +1941,22 @@ func TestRemove(t *testing.T) { file1 := filepath.Join(path, "file1") file2 := filepath.Join(path, "file2") + //file3 := filepath.Join(dir1, "file3") internal.CreateTestFile(t, file1, "some data1") internal.CreateTestFile(t, file2, "some data2") + //internal.CreateTestFile(t, file3, "some data3") + + err = os.Link(file1, linkPath) + err = os.Symlink(file2, symPath) _, err = io.WriteString(tempTestFileOfPaths, - fmt.Sprintf("%s\n%s\n%s\n%s", file1, file2, dir1, dir2)) + fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n", file1, file2, dir1, dir2, linkPath, symPath)) So(err, ShouldBeNil) - setName := "testRemoveFiles" + setName := "testRemoveFiles1" - exitCode, _ := s.runBinary(t, "add", "--items", tempTestFileOfPaths.Name(), - "--name", setName, "--transformer", transformer) - So(exitCode, ShouldEqual, 0) + s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) Convey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) @@ -1954,7 +1971,7 @@ func TestRemove(t *testing.T) { }) Convey("Remove removes the dir from the set", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", dir1) + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) @@ -1965,6 +1982,87 @@ func TestRemove(t *testing.T) { 0, dir1) }) + Convey("Remove takes a flag --items and removes all provided files and dirs from the set", func() { + tempTestFileOfPathsToRemove, err := os.CreateTemp(dir, "testFileSet") + So(err, ShouldBeNil) + + _, err = io.WriteString(tempTestFileOfPathsToRemove, + fmt.Sprintf("%s\n%s", file1, dir1)) + So(err, ShouldBeNil) + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--items", tempTestFileOfPathsToRemove.Name()) + + So(exitCode, ShouldEqual, 0) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, file2) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, file1) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, dir2) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1) + }) + + Convey("Remove removes the provided file from iRODS", func() { + output, err := exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + So(string(output), ShouldContainSubstring, "file1") + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) + So(exitCode, ShouldEqual, 0) + + output, err = exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + So(string(output), ShouldNotContainSubstring, "file1") + }) + + SkipConvey("Remove removes the provided dir from iRODS", func() { + output, err := exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + So(string(output), ShouldContainSubstring, "dir1") + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) + So(exitCode, ShouldEqual, 0) + + output, err = exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + So(string(output), ShouldNotContainSubstring, "dir1") + }) + + Convey("And another added set with the same files and dirs", func() { + setName2 := "testRemoveFiles2" + + s.addSetForTestingWithItems(t, setName2, transformer, tempTestFileOfPaths.Name()) + + Convey("Remove removes the metadata related to the set", func() { + output := getRemoteMeta(filepath.Join(remotePath, "file1")) + So(output, ShouldContainSubstring, setName) + So(output, ShouldContainSubstring, setName2) + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) + + So(exitCode, ShouldEqual, 0) + + output = getRemoteMeta(filepath.Join(remotePath, "file1")) + So(output, ShouldNotContainSubstring, setName) + So(output, ShouldContainSubstring, setName2) + }) + + Convey("Remove does not remove the provided file from iRODS", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) + So(exitCode, ShouldEqual, 0) + + output, err := exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + So(string(output), ShouldContainSubstring, "file1") + }) + + // add something about requesters + }) }) //TODO add tests for failed files }) diff --git a/put/baton.go b/put/baton.go index 24b241f..8f43b37 100644 --- a/put/baton.go +++ b/put/baton.go @@ -30,8 +30,10 @@ package put import ( "context" "errors" + "fmt" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -85,6 +87,34 @@ func GetBatonHandler() (*Baton, error) { return &Baton{}, err } +// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact +// with iRODS and contains a meta client for interacting with metadata. If you +// don't have baton-do in your PATH, you'll get an error. +func GetBatonHandlerWithMetaClient() (*Baton, error) { + setupExtendoLogger() + + _, err := ex.FindBaton() + if err != nil { + return nil, err + } + + params := ex.DefaultClientPoolParams + params.MaxSize = 1 + pool := ex.NewClientPool(params, "") + + metaClient, err := pool.Get() + if err != nil { + return nil, fmt.Errorf("failed to get metaClient: %w", err) + } + + baton := &Baton{ + putMetaPool: pool, + metaClient: metaClient, + } + + return baton, nil +} + // setupExtendoLogger sets up a STDERR logger that the extendo library will use. // (We don't actually care about what it might log, but extendo doesn't work // without this.) @@ -473,6 +503,50 @@ func metaToAVUs(meta map[string]string) []ex.AVU { return avus } +func (b *Baton) RemoveSetFromIRODSMetadata(path, setName string, meta map[string]string) error { + sets := strings.Split(meta[MetaKeySets], ",") + + sets, err := removeElementFromSlice(sets, setName) + if err != nil { + return err + } + + if len(sets) == 0 { + return b.removeFileFromIRODS(path) + } + + err = b.RemoveMeta(path, map[string]string{MetaKeySets: meta[MetaKeySets]}) + if err != nil { + return err + } + + return b.AddMeta(path, map[string]string{MetaKeySets: strings.Join(sets, ",")}) +} + +func (b *Baton) removeFileFromIRODS(path string) error { + it := remotePathToRodsItem(path) + + err := timeoutOp(func() error { + _, errl := b.metaClient.RemObj(ex.Args{}, *it) + + return errl + }, "remove meta error: "+path) + + return err +} + +func removeElementFromSlice(slice []string, element string) ([]string, error) { + index := slices.Index(slice, element) + if index < 0 { + return nil, fmt.Errorf("Element %s not in slice", element) + } + + slice[index] = slice[len(slice)-1] + slice[len(slice)-1] = "" + + return slice[:len(slice)-1], nil +} + func (b *Baton) RemoveMeta(path string, meta map[string]string) error { it := remotePathToRodsItem(path) it.IAVUs = metaToAVUs(meta) @@ -486,6 +560,16 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { return err } +func (b *Baton) GetMeta(path string) (map[string]string, error) { + //it := remotePathToRodsItem(path) + it, err := b.metaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ + IPath: filepath.Dir(path), + IName: filepath.Base(path), + }) + + return rodsItemToMeta(it), err +} + // remotePathToRodsItem converts a path in to an extendo RodsItem. func remotePathToRodsItem(path string) *ex.RodsItem { return &ex.RodsItem{ diff --git a/server/server.go b/server/server.go index 96cef15..7fb992d 100644 --- a/server/server.go +++ b/server/server.go @@ -96,6 +96,7 @@ type Server struct { cacheMu sync.Mutex dirPool *workerpool.WorkerPool queue *queue.Queue + // removeQueue *queue.Queue sched *scheduler.Scheduler putCmd string req *jqs.Requirements @@ -128,6 +129,7 @@ func New(conf Config) (*Server, error) { numClients: 1, dirPool: workerpool.New(workerPoolSizeDir), queue: queue.New(context.Background(), "put"), + // removeQueue: queue.New(context.Background(), "remove"), creatingCollections: make(map[string]bool), slacker: conf.Slacker, stillRunningMsgFreq: conf.StillRunningMsgFreq, @@ -139,6 +141,7 @@ func New(conf Config) (*Server, error) { s.clientQueue.SetTTRCallback(s.clientTTRC) s.Server.Router().Use(gas.IncludeAbortErrorsInBody) + // s.removeQueue.SetReadyAddedCallback(s.removeCallback) s.monitor = NewMonitor(func(given *set.Set) { if err := s.discoverSet(given); err != nil { @@ -200,6 +203,10 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC return nil } +// func (s *Server) removeCallback(_ string, item []interface{}) { +// fmt.Println("removeQueue callback called") +// } + // rac is our queue's ready added callback which will get all ready put Requests // and ensure there are enough put jobs added to wr. // diff --git a/server/setdb.go b/server/setdb.go index 33aee13..5d4a4d2 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -392,9 +392,45 @@ func (s *Server) removeFiles(c *gin.Context) { return } + // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) + s.removeFilesFromIRODS(sid, paths) + c.Status(http.StatusOK) } +func (s *Server) removeFilesFromIRODS(sid string, paths []string) error { + set := s.db.GetByID(sid) + + baton, err := put.GetBatonHandlerWithMetaClient() + if err != nil { + return err + } + + tranformer, err := set.MakeTransformer() + if err != nil { + return err + } + + for _, path := range paths { + rpath, err := tranformer(path) + if err != nil { + return err + } + + remoteMeta, err := baton.GetMeta(rpath) + if err != nil { + return err + } + + err = baton.RemoveSetFromIRODSMetadata(rpath, set.Name, remoteMeta) + if err != nil { + return err + } + } + + return nil +} + func (s *Server) removeDirs(c *gin.Context) { sid, paths, ok := s.bindPathsAndValidateSet(c) if !ok { From ec3ce1033cd4ca0197758176fa8bbc0b7814b843 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 17 Jan 2025 16:45:07 +0000 Subject: [PATCH 03/35] Add test for removing hardlink files --- main_test.go | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/main_test.go b/main_test.go index eb1b7e2..65e36ca 100644 --- a/main_test.go +++ b/main_test.go @@ -1337,7 +1337,7 @@ func remoteDBBackupPath() string { } func TestPuts(t *testing.T) { - Convey("Given a server configured with a remote hardlink location", t, func() { + FocusConvey("Given a server configured with a remote hardlink location", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1380,7 +1380,7 @@ func TestPuts(t *testing.T) { s.waitForStatus(setName, "\nStatus: complete (but with failures - try a retry)", 60*time.Second) }) - Convey("Given a file containing directory and file paths", func() { + FocusConvey("Given a file containing directory and file paths", func() { dir1 := filepath.Join(path, "path/to/some/dir/") dir2 := filepath.Join(path, "path/to/other/dir/") subdir1 := filepath.Join(dir1, "subdir/") @@ -1661,7 +1661,7 @@ Local Path Status Size Attempts Date Error`+"\n"+ }) // TODO: re-enable once hardlinks metamod bug fixed - SkipConvey("Putting a set with hardlinks uploads an empty file and special inode file", func() { + FocusConvey("Putting a set with hardlinks uploads an empty file and special inode file", func() { file := filepath.Join(path, "file") link1 := filepath.Join(path, "hardlink1") link2 := filepath.Join(path, "hardlink2") @@ -1894,7 +1894,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - Convey("Given a server", t, func() { + FocusConvey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1915,6 +1915,7 @@ func TestRemove(t *testing.T) { s.prepareConfig() s.schedulerDeployment = schedulerDeployment + s.remoteHardlinkPrefix = filepath.Join(remotePath, "hardlinks") s.backupFile = filepath.Join(dir, "db.bak") s.startServer() @@ -1922,7 +1923,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - Convey("And an added set with files and folders", func() { + FocusConvey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -1948,10 +1949,13 @@ func TestRemove(t *testing.T) { //internal.CreateTestFile(t, file3, "some data3") err = os.Link(file1, linkPath) + So(err, ShouldBeNil) + err = os.Symlink(file2, symPath) + So(err, ShouldBeNil) _, err = io.WriteString(tempTestFileOfPaths, - fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n", file1, file2, dir1, dir2, linkPath, symPath)) + fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", file1, file2, dir1, dir2, linkPath, symPath)) So(err, ShouldBeNil) setName := "testRemoveFiles1" @@ -2020,6 +2024,31 @@ func TestRemove(t *testing.T) { So(string(output), ShouldNotContainSubstring, "file1") }) + FocusConvey("Remove with a hardlink removes both the hardlink file and inode file", func() { + output := getRemoteMeta(filepath.Join(remotePath, "link")) + attrFind := "attribute: ibackup:remotehardlink\nvalue: " + attrPos := strings.Index(output, attrFind) + So(attrPos, ShouldNotEqual, -1) + + remoteInode := output[attrPos+len(attrFind):] + nlPos := strings.Index(remoteInode, "\n") + So(nlPos, ShouldNotEqual, -1) + + remoteInode = remoteInode[:nlPos] + + _, err := exec.Command("ils", remoteInode).CombinedOutput() + So(err, ShouldBeNil) + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) + So(exitCode, ShouldEqual, 0) + + _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() + So(err, ShouldNotBeNil) + + _, err = exec.Command("ils", remoteInode).CombinedOutput() + So(err, ShouldNotBeNil) + }) + SkipConvey("Remove removes the provided dir from iRODS", func() { output, err := exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) From 45ab2589a4153b2bbdefdaf70e96fc401e86c7ba Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Mon, 20 Jan 2025 16:48:27 +0000 Subject: [PATCH 04/35] Add implementation for removing hardlinks and dirs from iRODS --- cmd/remove.go | 40 +++++++++++++++++++++++--------- main_test.go | 30 +++++++++++++++++------- put/baton.go | 49 +++++++++++++++++++++++++++++++-------- server/setdb.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++--- set/db.go | 17 ++++++++++++++ 5 files changed, 165 insertions(+), 32 deletions(-) diff --git a/cmd/remove.go b/cmd/remove.go index 0e9af5e..84ebaae 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -39,6 +39,7 @@ var removeUser string var removeName string var removeItems string var removePath string +var removeNull bool // removeCmd represents the add command. var removeCmd = &cobra.Command{ @@ -50,24 +51,44 @@ var removeCmd = &cobra.Command{ removed. This will remove files from the set and from iRODS if it is not found in any other sets. + You also need to supply the ibackup server's URL in the form domain:port (using + the IBACKUP_SERVER_URL environment variable, or overriding that with the --url + argument) and if necessary, the certificate (using the IBACKUP_SERVER_CERT + environment variable, or overriding that with the --cert argument). + + --name is a required flag used to describe which set you want to remove files + from. + + You must also provide at least one of: + --items: the path to a file containing the paths of files/directories you want + to remove from the set. Each path should be on its own line. Because + filenames can contain new line characters in them, it's safer to + null-terminate them instead and use the optional --null argument. + --path: if you want to remove a single file or directory, provide its absolute + path. `, Run: func(cmd *cobra.Command, args []string) { ensureURLandCert() if (removeItems == "") == (removePath == "") { - die("only one of --items or --path can be provided") + die("exactly one of --items or --path must be provided") } var files []string var dirs []string + client, err := newServerClient(serverURL, serverCert) + if err != nil { + die("%s", err.Error()) + } + if removeItems != "" { - filesAndDirs := readPaths(removeItems, fofnLineSplitter(setNull)) + filesAndDirs := readPaths(removeItems, fofnLineSplitter(removeNull)) files, dirs = categorisePaths(filesAndDirs, files, dirs) } if removePath != "" { - removePath, err := filepath.Abs(removePath) + removePath, err = filepath.Abs(removePath) if err != nil { die("%s", err.Error()) } @@ -79,12 +100,7 @@ var removeCmd = &cobra.Command{ } } - client, err := newServerClient(serverURL, serverCert) - if err != nil { - die("%s", err.Error()) - } - - handleRemove(client, removeUser, removeName, files, dirs) + remove(client, removeUser, removeName, files, dirs) }, } @@ -99,13 +115,16 @@ func init() { "path to file with one absolute local directory or file path per line") removeCmd.Flags().StringVarP(&removePath, "path", "p", "", "path to a single file or directory you wish to remove") + removeCmd.Flags().BoolVarP(&removeNull, "null", "0", false, + "input paths are terminated by a null character instead of a new line") if err := removeCmd.MarkFlagRequired("name"); err != nil { die("%s", err.Error()) } } -func handleRemove(client *server.Client, user, name string, files, dirs []string) { +// remove does the main job of sending the set, files and dirs to the server. +func remove(client *server.Client, user, name string, files, dirs []string) { sets := getSetByName(client, user, name) if len(sets) == 0 { warn("No backup sets found with name %s", name) @@ -122,5 +141,4 @@ func handleRemove(client *server.Client, user, name string, files, dirs []string if err != nil { die("%s", err.Error()) } - } diff --git a/main_test.go b/main_test.go index 65e36ca..12c63f9 100644 --- a/main_test.go +++ b/main_test.go @@ -304,6 +304,7 @@ func (s *TestServer) addSetForTestingWithItems(t *testing.T, name, transformer, So(exitCode, ShouldEqual, 0) + s.waitForStatus(name, "\nDiscovery: completed", 5*time.Second) s.waitForStatus(name, "\nStatus: complete", 5*time.Second) } @@ -1942,11 +1943,11 @@ func TestRemove(t *testing.T) { file1 := filepath.Join(path, "file1") file2 := filepath.Join(path, "file2") - //file3 := filepath.Join(dir1, "file3") + file3 := filepath.Join(dir1, "file3") internal.CreateTestFile(t, file1, "some data1") internal.CreateTestFile(t, file2, "some data2") - //internal.CreateTestFile(t, file3, "some data3") + internal.CreateTestFile(t, file3, "some data3") err = os.Link(file1, linkPath) So(err, ShouldBeNil) @@ -2024,7 +2025,7 @@ func TestRemove(t *testing.T) { So(string(output), ShouldNotContainSubstring, "file1") }) - FocusConvey("Remove with a hardlink removes both the hardlink file and inode file", func() { + Convey("Remove with a hardlink removes both the hardlink file and inode file", func() { output := getRemoteMeta(filepath.Join(remotePath, "link")) attrFind := "attribute: ibackup:remotehardlink\nvalue: " attrPos := strings.Index(output, attrFind) @@ -2049,19 +2050,23 @@ func TestRemove(t *testing.T) { So(err, ShouldNotBeNil) }) - SkipConvey("Remove removes the provided dir from iRODS", func() { - output, err := exec.Command("ils", remotePath).CombinedOutput() + FocusConvey("Remove removes the provided dir from iRODS", func() { + output, err := exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) - So(string(output), ShouldContainSubstring, "dir1") + So(string(output), ShouldContainSubstring, "path/to/some/dir") + So(string(output), ShouldContainSubstring, "file3\n") exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) - output, err = exec.Command("ils", remotePath).CombinedOutput() + output, err = exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) - So(string(output), ShouldNotContainSubstring, "dir1") + So(string(output), ShouldNotContainSubstring, "path/to/some/dir") + So(string(output), ShouldNotContainSubstring, "file3\n") }) + //TODO: add test about removing a dir with non-irods uploaded files in it + Convey("And another added set with the same files and dirs", func() { setName2 := "testRemoveFiles2" @@ -2090,6 +2095,15 @@ func TestRemove(t *testing.T) { So(string(output), ShouldContainSubstring, "file1") }) + FocusConvey("...", func() { + exitCode, _ := s.runBinary(t, "add", "--name", "different_user_set", "--transformer", + transformer, "--items", tempTestFileOfPaths.Name(), "--user", "rk18") + + So(exitCode, ShouldEqual, 0) + + s.waitForStatus("different_user_set", "\nStatus: complete", 20*time.Second) + }) + // add something about requesters }) }) diff --git a/put/baton.go b/put/baton.go index 8f43b37..2c23b4e 100644 --- a/put/baton.go +++ b/put/baton.go @@ -503,7 +503,10 @@ func metaToAVUs(meta map[string]string) []ex.AVU { return avus } -func (b *Baton) RemoveSetFromIRODSMetadata(path, setName string, meta map[string]string) error { +// RemovePathFromSetInIRODS removes the given path from iRODS if the path is not +// associated with any other sets. Otherwise it updates the iRODS metadata for +// the path to not include the given set. +func (b *Baton) RemovePathFromSetInIRODS(path, setName string, meta map[string]string) error { sets := strings.Split(meta[MetaKeySets], ",") sets, err := removeElementFromSlice(sets, setName) @@ -512,7 +515,7 @@ func (b *Baton) RemoveSetFromIRODSMetadata(path, setName string, meta map[string } if len(sets) == 0 { - return b.removeFileFromIRODS(path) + return b.handleHardlinkAndRemoveFromIRODS(path, meta) } err = b.RemoveMeta(path, map[string]string{MetaKeySets: meta[MetaKeySets]}) @@ -523,6 +526,31 @@ func (b *Baton) RemoveSetFromIRODSMetadata(path, setName string, meta map[string return b.AddMeta(path, map[string]string{MetaKeySets: strings.Join(sets, ",")}) } +func removeElementFromSlice(slice []string, element string) ([]string, error) { + index := slices.Index(slice, element) + if index < 0 { + return nil, fmt.Errorf("Element %s not in slice", element) + } + + slice[index] = slice[len(slice)-1] + slice[len(slice)-1] = "" + + return slice[:len(slice)-1], nil +} + +func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, meta map[string]string) error { + err := b.removeFileFromIRODS(path) + if err != nil { + return err + } + + if meta[MetaKeyHardlink] == "" { + return nil + } + + return b.removeFileFromIRODS(meta[MetaKeyRemoteHardlink]) +} + func (b *Baton) removeFileFromIRODS(path string) error { it := remotePathToRodsItem(path) @@ -535,16 +563,18 @@ func (b *Baton) removeFileFromIRODS(path string) error { return err } -func removeElementFromSlice(slice []string, element string) ([]string, error) { - index := slices.Index(slice, element) - if index < 0 { - return nil, fmt.Errorf("Element %s not in slice", element) +func (b *Baton) RemoveDirFromIRODS(path string) error { + it := &ex.RodsItem{ + IPath: path, } - slice[index] = slice[len(slice)-1] - slice[len(slice)-1] = "" + err := timeoutOp(func() error { + _, errl := b.metaClient.RemDir(ex.Args{}, *it) - return slice[:len(slice)-1], nil + return errl + }, "remove meta error: "+path) + + return err } func (b *Baton) RemoveMeta(path string, meta map[string]string) error { @@ -561,7 +591,6 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { } func (b *Baton) GetMeta(path string) (map[string]string, error) { - //it := remotePathToRodsItem(path) it, err := b.metaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ IPath: filepath.Dir(path), IName: filepath.Base(path), diff --git a/server/setdb.go b/server/setdb.go index 5d4a4d2..bba1ab9 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -393,7 +393,12 @@ func (s *Server) removeFiles(c *gin.Context) { } // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) - s.removeFilesFromIRODS(sid, paths) + err = s.removeFilesFromIRODS(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } c.Status(http.StatusOK) } @@ -422,7 +427,36 @@ func (s *Server) removeFilesFromIRODS(sid string, paths []string) error { return err } - err = baton.RemoveSetFromIRODSMetadata(rpath, set.Name, remoteMeta) + err = baton.RemovePathFromSetInIRODS(rpath, set.Name, remoteMeta) + if err != nil { + return err + } + } + + return nil +} + +// deduplicates pre-loop linesbased on RemoveFilesFromIRODS +func (s *Server) removeDirsFromIRODS(sid string, paths []string) error { + set := s.db.GetByID(sid) + + baton, err := put.GetBatonHandlerWithMetaClient() + if err != nil { + return err + } + + tranformer, err := set.MakeTransformer() + if err != nil { + return err + } + + for _, path := range paths { + rpath, err := tranformer(path) + if err != nil { + return err + } + + err = baton.RemoveDirFromIRODS(rpath) if err != nil { return err } @@ -437,7 +471,28 @@ func (s *Server) removeDirs(c *gin.Context) { return } - err := s.db.RemoveDirEntries(sid, paths) + var filepaths []string + + var err error + + for _, path := range paths { + filepaths, err = s.db.GetFilesInDir(sid, path, filepaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + } + } + + err = s.removeFilesFromIRODS(sid, filepaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + } + + err = s.removeDirsFromIRODS(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + } + + err = s.db.RemoveDirEntries(sid, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck diff --git a/set/db.go b/set/db.go index e58b967..21c3104 100644 --- a/set/db.go +++ b/set/db.go @@ -288,6 +288,23 @@ func (d *DB) RemoveDirEntries(setID string, paths []string) error { return d.removeEntries(setID, paths, dirBucket) } +func (d *DB) GetFilesInDir(setID string, dirpath string, filepaths []string) ([]string, error) { + entries, err := d.getEntries(setID, discoveredBucket) + if err != nil { + return nil, err + } + + for _, entry := range entries { + path := entry.Path + + if strings.HasPrefix(path, dirpath) { + filepaths = append(filepaths, path) + } + } + + return filepaths, nil +} + // SetFileEntries sets the file paths for the given backup set. Only supply // absolute paths to files. func (d *DB) SetFileEntries(setID string, paths []string) error { From f3c628356c3e9c3de55cdb87c8e76cf16ff7cb69 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 21 Jan 2025 15:59:00 +0000 Subject: [PATCH 05/35] Add validation to check if files/dirs are in the set --- main_test.go | 39 ++++++++++++++++++++++++++++++++++----- server/setdb.go | 48 ++++++++++++++++++++++++++++++++++++------------ set/db.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/main_test.go b/main_test.go index 12c63f9..0fa7eb3 100644 --- a/main_test.go +++ b/main_test.go @@ -304,8 +304,8 @@ func (s *TestServer) addSetForTestingWithItems(t *testing.T, name, transformer, So(exitCode, ShouldEqual, 0) - s.waitForStatus(name, "\nDiscovery: completed", 5*time.Second) - s.waitForStatus(name, "\nStatus: complete", 5*time.Second) + s.waitForStatus(name, "\nDiscovery: completed", 10*time.Second) + s.waitForStatus(name, "\nStatus: complete", 10*time.Second) } func (s *TestServer) addSetForTestingWithFlag(t *testing.T, name, transformer, path, flag, data string) { @@ -1929,7 +1929,8 @@ func TestRemove(t *testing.T) { linkPath := filepath.Join(path, "link") symPath := filepath.Join(path, "sym") - dir1 := filepath.Join(path, "path/to/some/dir/") + testDir := filepath.Join(path, "path/to/some/") + dir1 := filepath.Join(testDir, "dir") dir2 := filepath.Join(path, "path/to/other/dir/") tempTestFileOfPaths, err := os.CreateTemp(dir, "testFileSet") @@ -1944,10 +1945,12 @@ func TestRemove(t *testing.T) { file1 := filepath.Join(path, "file1") file2 := filepath.Join(path, "file2") file3 := filepath.Join(dir1, "file3") + file4 := filepath.Join(testDir, "dir_not_removed") internal.CreateTestFile(t, file1, "some data1") internal.CreateTestFile(t, file2, "some data2") internal.CreateTestFile(t, file3, "some data3") + internal.CreateTestFile(t, file4, "some data3") err = os.Link(file1, linkPath) So(err, ShouldBeNil) @@ -1956,7 +1959,7 @@ func TestRemove(t *testing.T) { So(err, ShouldBeNil) _, err = io.WriteString(tempTestFileOfPaths, - fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", file1, file2, dir1, dir2, linkPath, symPath)) + fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s", file1, file2, file4, dir1, dir2, linkPath, symPath)) So(err, ShouldBeNil) setName := "testRemoveFiles1" @@ -2061,11 +2064,37 @@ func TestRemove(t *testing.T) { output, err = exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) + So(string(output), ShouldContainSubstring, "dir_not_removed\n") So(string(output), ShouldNotContainSubstring, "path/to/some/dir") So(string(output), ShouldNotContainSubstring, "file3\n") }) - //TODO: add test about removing a dir with non-irods uploaded files in it + FocusConvey("Given a new file added to a directory already in the set", func() { + file5 := filepath.Join(dir1, "file5") + internal.CreateTestFile(t, file5, "some data5") + + FocusConvey("Remove returns an error if you try to remove just the file", func() { + s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", file5}, + 1, fmt.Sprintf("%s is not part of the backup set [%s]", file5, setName)) + }) + + FocusConvey("Remove ignores the file if you remove the directory", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) + So(exitCode, ShouldEqual, 0) + }) + }) + + FocusConvey("Given a new directory", func() { + dir3 := filepath.Join(path, "path/to/new/dir/") + + err = os.MkdirAll(dir3, 0755) + So(err, ShouldBeNil) + + FocusConvey("Remove returns an error if you try to remove the directory", func() { + s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", dir3}, + 1, fmt.Sprintf("%s is not part of the backup set [%s]", dir3, setName)) + }) + }) Convey("And another added set with the same files and dirs", func() { setName2 := "testRemoveFiles2" diff --git a/server/setdb.go b/server/setdb.go index bba1ab9..d166e96 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -385,7 +385,14 @@ func (s *Server) removeFiles(c *gin.Context) { return } - err := s.db.RemoveFileEntries(sid, paths) + err := s.db.ValidateFilePaths(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + err = s.db.RemoveFileEntries(sid, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -437,7 +444,7 @@ func (s *Server) removeFilesFromIRODS(sid string, paths []string) error { } // deduplicates pre-loop linesbased on RemoveFilesFromIRODS -func (s *Server) removeDirsFromIRODS(sid string, paths []string) error { +func (s *Server) removeDirsFromIRODS(sid string, dirpaths, filepaths []string) error { set := s.db.GetByID(sid) baton, err := put.GetBatonHandlerWithMetaClient() @@ -450,7 +457,24 @@ func (s *Server) removeDirsFromIRODS(sid string, paths []string) error { return err } - for _, path := range paths { + for _, path := range filepaths { + rpath, err := tranformer(path) + if err != nil { + return err + } + + remoteMeta, err := baton.GetMeta(rpath) + if err != nil { + return err + } + + err = baton.RemovePathFromSetInIRODS(rpath, set.Name, remoteMeta) + if err != nil { + return err + } + } + + for _, path := range dirpaths { rpath, err := tranformer(path) if err != nil { return err @@ -471,23 +495,23 @@ func (s *Server) removeDirs(c *gin.Context) { return } - var filepaths []string + err := s.db.ValidateDirPaths(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - var err error + return + } + + var filepaths []string for _, path := range paths { filepaths, err = s.db.GetFilesInDir(sid, path, filepaths) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck } } - err = s.removeFilesFromIRODS(sid, filepaths) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - } - - err = s.removeDirsFromIRODS(sid, paths) + err = s.removeDirsFromIRODS(sid, paths, filepaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck } diff --git a/set/db.go b/set/db.go index 21c3104..ef34896 100644 --- a/set/db.go +++ b/set/db.go @@ -253,6 +253,39 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } +func (d *DB) ValidateFilePaths(setID string, paths []string) error { + return d.validatePaths(setID, fileBucket, discoveredBucket, paths) +} + +func (d *DB) validatePaths(setID, bucket1, bucket2 string, paths []string) error { + entriesMap := make(map[string]bool) + + set := d.GetByID(setID) + + for _, bucket := range []string{bucket1, bucket2} { + entries, err := d.getEntries(setID, bucket) + if err != nil { + return err + } + + for _, entry := range entries { + entriesMap[entry.Path] = true + } + } + + for _, path := range paths { + if _, ok := entriesMap[path]; !ok { + return Error{fmt.Sprintf("%s is not part of the backup set", path), set.Name} + } + } + + return nil +} + +func (d *DB) ValidateDirPaths(setID string, paths []string) error { + return d.validatePaths(setID, dirBucket, discoveredBucket, paths) +} + // RemoveFileEntries removes the provided files from a given set. func (d *DB) RemoveFileEntries(setID string, paths []string) error { err := d.removeEntries(setID, paths, fileBucket) @@ -294,9 +327,13 @@ func (d *DB) GetFilesInDir(setID string, dirpath string, filepaths []string) ([] return nil, err } + //dirpath = filepath.Clean(dirpath) + "/" + for _, entry := range entries { path := entry.Path + //path = filepath.Clean(path) + if strings.HasPrefix(path, dirpath) { filepaths = append(filepaths, path) } From b04ee4e52c0c2ad7a2c452cb995a2f0d7560edba Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 21 Jan 2025 16:27:52 +0000 Subject: [PATCH 06/35] Refactor removeFilesFromIRODS and removeDirsFromIRODS functions --- server/setdb.go | 103 ++++++++++++++++++++++++------------------------ set/db.go | 14 +++---- 2 files changed, 57 insertions(+), 60 deletions(-) diff --git a/server/setdb.go b/server/setdb.go index d166e96..07bca1a 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -385,7 +385,9 @@ func (s *Server) removeFiles(c *gin.Context) { return } - err := s.db.ValidateFilePaths(sid, paths) + set := s.db.GetByID(sid) + + err := s.db.ValidateFilePaths(set, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -399,8 +401,15 @@ func (s *Server) removeFiles(c *gin.Context) { return } + baton, transformer, err := s.getBatonAndTransformer(set) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) - err = s.removeFilesFromIRODS(sid, paths) + err = s.removeFilesFromIRODS(set, paths, baton, transformer) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -410,21 +419,21 @@ func (s *Server) removeFiles(c *gin.Context) { c.Status(http.StatusOK) } -func (s *Server) removeFilesFromIRODS(sid string, paths []string) error { - set := s.db.GetByID(sid) - +func (s *Server) getBatonAndTransformer(set *set.Set) (*put.Baton, put.PathTransformer, error) { baton, err := put.GetBatonHandlerWithMetaClient() if err != nil { - return err + return nil, nil, err } tranformer, err := set.MakeTransformer() - if err != nil { - return err - } + return baton, tranformer, err +} + +func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, + baton *put.Baton, transformer put.PathTransformer) error { for _, path := range paths { - rpath, err := tranformer(path) + rpath, err := transformer(path) if err != nil { return err } @@ -443,39 +452,10 @@ func (s *Server) removeFilesFromIRODS(sid string, paths []string) error { return nil } -// deduplicates pre-loop linesbased on RemoveFilesFromIRODS -func (s *Server) removeDirsFromIRODS(sid string, dirpaths, filepaths []string) error { - set := s.db.GetByID(sid) - - baton, err := put.GetBatonHandlerWithMetaClient() - if err != nil { - return err - } - - tranformer, err := set.MakeTransformer() - if err != nil { - return err - } - - for _, path := range filepaths { - rpath, err := tranformer(path) - if err != nil { - return err - } - - remoteMeta, err := baton.GetMeta(rpath) - if err != nil { - return err - } - - err = baton.RemovePathFromSetInIRODS(rpath, set.Name, remoteMeta) - if err != nil { - return err - } - } - +func (s *Server) removeDirsFromIRODS(set *set.Set, dirpaths []string, + baton *put.Baton, transformer put.PathTransformer) error { for _, path := range dirpaths { - rpath, err := tranformer(path) + rpath, err := transformer(path) if err != nil { return err } @@ -495,35 +475,54 @@ func (s *Server) removeDirs(c *gin.Context) { return } - err := s.db.ValidateDirPaths(sid, paths) + set := s.db.GetByID(sid) + + err := s.db.ValidateDirPaths(set, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + err = s.handleRemovalOfDirsFromIRODS(set, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } + err = s.db.RemoveDirEntries(sid, paths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + c.Status(http.StatusOK) +} + +func (s *Server) handleRemovalOfDirsFromIRODS(set *set.Set, paths []string) error { var filepaths []string + var err error for _, path := range paths { - filepaths, err = s.db.GetFilesInDir(sid, path, filepaths) + filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + return err } } - err = s.removeDirsFromIRODS(sid, paths, filepaths) + baton, transformer, err := s.getBatonAndTransformer(set) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + return err } - err = s.db.RemoveDirEntries(sid, paths) + err = s.removeFilesFromIRODS(set, filepaths, baton, transformer) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return + return err } - c.Status(http.StatusOK) + return s.removeDirsFromIRODS(set, paths, baton, transformer) } // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id diff --git a/set/db.go b/set/db.go index ef34896..e27fe22 100644 --- a/set/db.go +++ b/set/db.go @@ -253,17 +253,15 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } -func (d *DB) ValidateFilePaths(setID string, paths []string) error { - return d.validatePaths(setID, fileBucket, discoveredBucket, paths) +func (d *DB) ValidateFilePaths(set *Set, paths []string) error { + return d.validatePaths(set, fileBucket, discoveredBucket, paths) } -func (d *DB) validatePaths(setID, bucket1, bucket2 string, paths []string) error { +func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) error { entriesMap := make(map[string]bool) - set := d.GetByID(setID) - for _, bucket := range []string{bucket1, bucket2} { - entries, err := d.getEntries(setID, bucket) + entries, err := d.getEntries(set.ID(), bucket) if err != nil { return err } @@ -282,8 +280,8 @@ func (d *DB) validatePaths(setID, bucket1, bucket2 string, paths []string) error return nil } -func (d *DB) ValidateDirPaths(setID string, paths []string) error { - return d.validatePaths(setID, dirBucket, discoveredBucket, paths) +func (d *DB) ValidateDirPaths(set *Set, paths []string) error { + return d.validatePaths(set, dirBucket, discoveredBucket, paths) } // RemoveFileEntries removes the provided files from a given set. From e8bfdf22b976fac28f29d784bdec73b8e1a13350 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 22 Jan 2025 11:59:51 +0000 Subject: [PATCH 07/35] Fix wrong implementation about deleting hardlink inodes file --- main_test.go | 86 +++++++++++++++++++++++++++++++++++-------------- put/baton.go | 38 ++++++++++++++++++++-- server/setdb.go | 34 +++++++++++-------- 3 files changed, 118 insertions(+), 40 deletions(-) diff --git a/main_test.go b/main_test.go index 0fa7eb3..de3e592 100644 --- a/main_test.go +++ b/main_test.go @@ -1338,7 +1338,7 @@ func remoteDBBackupPath() string { } func TestPuts(t *testing.T) { - FocusConvey("Given a server configured with a remote hardlink location", t, func() { + Convey("Given a server configured with a remote hardlink location", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1381,7 +1381,7 @@ func TestPuts(t *testing.T) { s.waitForStatus(setName, "\nStatus: complete (but with failures - try a retry)", 60*time.Second) }) - FocusConvey("Given a file containing directory and file paths", func() { + Convey("Given a file containing directory and file paths", func() { dir1 := filepath.Join(path, "path/to/some/dir/") dir2 := filepath.Join(path, "path/to/other/dir/") subdir1 := filepath.Join(dir1, "subdir/") @@ -1662,7 +1662,7 @@ Local Path Status Size Attempts Date Error`+"\n"+ }) // TODO: re-enable once hardlinks metamod bug fixed - FocusConvey("Putting a set with hardlinks uploads an empty file and special inode file", func() { + Convey("Putting a set with hardlinks uploads an empty file and special inode file", func() { file := filepath.Join(path, "file") link1 := filepath.Join(path, "hardlink1") link2 := filepath.Join(path, "hardlink2") @@ -1895,7 +1895,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - FocusConvey("Given a server", t, func() { + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1924,7 +1924,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - FocusConvey("And an added set with files and folders", func() { + Convey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -1987,7 +1987,10 @@ func TestRemove(t *testing.T) { 0, dir2) s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, - 0, dir1) + 0, dir1+"/") + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1+" => ") }) Convey("Remove takes a flag --items and removes all provided files and dirs from the set", func() { @@ -2012,7 +2015,10 @@ func TestRemove(t *testing.T) { 0, dir2) s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, - 0, dir1) + 0, dir1+"/") + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1+" => ") }) Convey("Remove removes the provided file from iRODS", func() { @@ -2029,16 +2035,7 @@ func TestRemove(t *testing.T) { }) Convey("Remove with a hardlink removes both the hardlink file and inode file", func() { - output := getRemoteMeta(filepath.Join(remotePath, "link")) - attrFind := "attribute: ibackup:remotehardlink\nvalue: " - attrPos := strings.Index(output, attrFind) - So(attrPos, ShouldNotEqual, -1) - - remoteInode := output[attrPos+len(attrFind):] - nlPos := strings.Index(remoteInode, "\n") - So(nlPos, ShouldNotEqual, -1) - - remoteInode = remoteInode[:nlPos] + remoteInode := getRemoteInodePath(filepath.Join(remotePath, "link")) _, err := exec.Command("ils", remoteInode).CombinedOutput() So(err, ShouldBeNil) @@ -2053,7 +2050,34 @@ func TestRemove(t *testing.T) { So(err, ShouldNotBeNil) }) - FocusConvey("Remove removes the provided dir from iRODS", func() { + Convey("Given another set with a hardlink to the same file", func() { + linkPath2 := filepath.Join(path, "link2") + + err = os.Link(file1, linkPath2) + So(err, ShouldBeNil) + + s.addSetForTesting(t, "testHardlinks", transformer, linkPath2) + + s.waitForStatus("testHardlinks", "\nStatus: complete", 10*time.Second) + + Convey("Removing a hardlink does not remove the inode file", func() { + remoteInode := getRemoteInodePath(filepath.Join(remotePath, "link2")) + + _, err := exec.Command("ils", remoteInode).CombinedOutput() + So(err, ShouldBeNil) + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) + So(exitCode, ShouldEqual, 0) + + _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() + So(err, ShouldNotBeNil) + + _, err = exec.Command("ils", remoteInode).CombinedOutput() + So(err, ShouldBeNil) + }) + }) + + Convey("Remove removes the provided dir from iRODS", func() { output, err := exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) So(string(output), ShouldContainSubstring, "path/to/some/dir") @@ -2069,28 +2093,28 @@ func TestRemove(t *testing.T) { So(string(output), ShouldNotContainSubstring, "file3\n") }) - FocusConvey("Given a new file added to a directory already in the set", func() { + Convey("Given a new file added to a directory already in the set", func() { file5 := filepath.Join(dir1, "file5") internal.CreateTestFile(t, file5, "some data5") - FocusConvey("Remove returns an error if you try to remove just the file", func() { + Convey("Remove returns an error if you try to remove just the file", func() { s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", file5}, 1, fmt.Sprintf("%s is not part of the backup set [%s]", file5, setName)) }) - FocusConvey("Remove ignores the file if you remove the directory", func() { + Convey("Remove ignores the file if you remove the directory", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) }) }) - FocusConvey("Given a new directory", func() { + Convey("Given a new directory", func() { dir3 := filepath.Join(path, "path/to/new/dir/") err = os.MkdirAll(dir3, 0755) So(err, ShouldBeNil) - FocusConvey("Remove returns an error if you try to remove the directory", func() { + Convey("Remove returns an error if you try to remove the directory", func() { s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", dir3}, 1, fmt.Sprintf("%s is not part of the backup set [%s]", dir3, setName)) }) @@ -2124,7 +2148,7 @@ func TestRemove(t *testing.T) { So(string(output), ShouldContainSubstring, "file1") }) - FocusConvey("...", func() { + Convey("...", func() { exitCode, _ := s.runBinary(t, "add", "--name", "different_user_set", "--transformer", transformer, "--items", tempTestFileOfPaths.Name(), "--user", "rk18") @@ -2139,3 +2163,17 @@ func TestRemove(t *testing.T) { //TODO add tests for failed files }) } + +func getRemoteInodePath(linkPath string) string { + output := getRemoteMeta(linkPath) + + attrFind := "attribute: ibackup:remotehardlink\nvalue: " + attrPos := strings.Index(output, attrFind) + So(attrPos, ShouldNotEqual, -1) + + remoteInode := output[attrPos+len(attrFind):] + nlPos := strings.Index(remoteInode, "\n") + So(nlPos, ShouldNotEqual, -1) + + return remoteInode[:nlPos] +} diff --git a/put/baton.go b/put/baton.go index 2c23b4e..98bc467 100644 --- a/put/baton.go +++ b/put/baton.go @@ -506,7 +506,7 @@ func metaToAVUs(meta map[string]string) []ex.AVU { // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not // associated with any other sets. Otherwise it updates the iRODS metadata for // the path to not include the given set. -func (b *Baton) RemovePathFromSetInIRODS(path, setName string, meta map[string]string) error { +func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path, setName string, meta map[string]string) error { sets := strings.Split(meta[MetaKeySets], ",") sets, err := removeElementFromSlice(sets, setName) @@ -515,7 +515,7 @@ func (b *Baton) RemovePathFromSetInIRODS(path, setName string, meta map[string]s } if len(sets) == 0 { - return b.handleHardlinkAndRemoveFromIRODS(path, meta) + return b.handleHardlinkAndRemoveFromIRODS(path, transformer, meta) } err = b.RemoveMeta(path, map[string]string{MetaKeySets: meta[MetaKeySets]}) @@ -538,7 +538,8 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { return slice[:len(slice)-1], nil } -func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, meta map[string]string) error { +func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, transformer PathTransformer, + meta map[string]string) error { err := b.removeFileFromIRODS(path) if err != nil { return err @@ -548,6 +549,15 @@ func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, meta map[string]st return nil } + items, err := b.queryIRODSMeta(transformer, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) + if err != nil { + return err + } + + if len(items) != 0 { + return nil + } + return b.removeFileFromIRODS(meta[MetaKeyRemoteHardlink]) } @@ -563,6 +573,28 @@ func (b *Baton) removeFileFromIRODS(path string) error { return err } +func (b *Baton) queryIRODSMeta(transformer PathTransformer, meta map[string]string) ([]ex.RodsItem, error) { + path, err := transformer("/") + if err != nil { + return nil, err + } + + it := &ex.RodsItem{ + IPath: path, + IAVUs: metaToAVUs(meta), + } + + var items []ex.RodsItem + + err = timeoutOp(func() error { + items, err = b.metaClient.MetaQuery(ex.Args{Object: true}, *it) + + return err + }, "remove meta error: "+path) + + return items, err +} + func (b *Baton) RemoveDirFromIRODS(path string) error { it := &ex.RodsItem{ IPath: path, diff --git a/server/setdb.go b/server/setdb.go index 07bca1a..c8d4908 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -443,7 +443,7 @@ func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, return err } - err = baton.RemovePathFromSetInIRODS(rpath, set.Name, remoteMeta) + err = baton.RemovePathFromSetInIRODS(transformer, rpath, set.Name, remoteMeta) if err != nil { return err } @@ -484,7 +484,18 @@ func (s *Server) removeDirs(c *gin.Context) { return } - err = s.handleRemovalOfDirsFromIRODS(set, paths) + var filepaths []string + + for _, path := range paths { + filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + } + + err = s.handleRemovalOfDirsFromIRODS(set, paths, filepaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -498,20 +509,17 @@ func (s *Server) removeDirs(c *gin.Context) { return } - c.Status(http.StatusOK) -} - -func (s *Server) handleRemovalOfDirsFromIRODS(set *set.Set, paths []string) error { - var filepaths []string - var err error + err = s.db.RemoveFileEntries(sid, filepaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - for _, path := range paths { - filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) - if err != nil { - return err - } + return } + c.Status(http.StatusOK) +} + +func (s *Server) handleRemovalOfDirsFromIRODS(set *set.Set, paths, filepaths []string) error { baton, transformer, err := s.getBatonAndTransformer(set) if err != nil { return err From b8792944f0a0394e7cef457453edb1ddc26fbcd7 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 22 Jan 2025 15:49:42 +0000 Subject: [PATCH 08/35] Add implementation to remove requester from iRODS metadata --- main_test.go | 92 +++++++++++++++++++++++++++++++++++----------- put/baton.go | 37 ++++++++----------- server/setdb.go | 97 ++++++++++++++++++++++++++++++++++--------------- set/db.go | 4 -- 4 files changed, 153 insertions(+), 77 deletions(-) diff --git a/main_test.go b/main_test.go index de3e592..790c5b0 100644 --- a/main_test.go +++ b/main_test.go @@ -349,6 +349,35 @@ func (s *TestServer) waitForStatus(name, statusToFind string, timeout time.Durat So(status.Err, ShouldBeNil) } +func (s *TestServer) waitForStatusWithUser(name, statusToFind, user string, timeout time.Duration) { + ctx, cancelFn := context.WithTimeout(context.Background(), timeout) + defer cancelFn() + + cmd := []string{"status", "--name", name, "--url", s.url, "--cert", s.cert, "--user", user} + + status := retry.Do(ctx, func() error { + clientCmd := exec.Command("./"+app, cmd...) + clientCmd.Env = s.env + + output, err := clientCmd.CombinedOutput() + if err != nil { + return err + } + + if strings.Contains(string(output), statusToFind) { + return nil + } + + return ErrStatusNotFound + }, &retry.UntilNoError{}, btime.SecondsRangeBackoff(), "waiting for matching status") + + if status.Err != nil { + fmt.Printf("\nfailed to see set %s get status: %s\n", name, statusToFind) //nolint:forbidigo + } + + So(status.Err, ShouldBeNil) +} + func (s *TestServer) Shutdown() error { if s.stopped { return nil @@ -2035,7 +2064,7 @@ func TestRemove(t *testing.T) { }) Convey("Remove with a hardlink removes both the hardlink file and inode file", func() { - remoteInode := getRemoteInodePath(filepath.Join(remotePath, "link")) + remoteInode := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "link")), "ibackup:remotehardlink") _, err := exec.Command("ils", remoteInode).CombinedOutput() So(err, ShouldBeNil) @@ -2050,7 +2079,7 @@ func TestRemove(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("Given another set with a hardlink to the same file", func() { + Convey("And another set with a hardlink to the same file", func() { linkPath2 := filepath.Join(path, "link2") err = os.Link(file1, linkPath2) @@ -2061,7 +2090,7 @@ func TestRemove(t *testing.T) { s.waitForStatus("testHardlinks", "\nStatus: complete", 10*time.Second) Convey("Removing a hardlink does not remove the inode file", func() { - remoteInode := getRemoteInodePath(filepath.Join(remotePath, "link2")) + remoteInode := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "link2")), "ibackup:remotehardlink") _, err := exec.Command("ils", remoteInode).CombinedOutput() So(err, ShouldBeNil) @@ -2093,7 +2122,7 @@ func TestRemove(t *testing.T) { So(string(output), ShouldNotContainSubstring, "file3\n") }) - Convey("Given a new file added to a directory already in the set", func() { + Convey("And a new file added to a directory already in the set", func() { file5 := filepath.Join(dir1, "file5") internal.CreateTestFile(t, file5, "some data5") @@ -2108,7 +2137,7 @@ func TestRemove(t *testing.T) { }) }) - Convey("Given a new directory", func() { + Convey("And a new directory", func() { dir3 := filepath.Join(path, "path/to/new/dir/") err = os.MkdirAll(dir3, 0755) @@ -2120,10 +2149,18 @@ func TestRemove(t *testing.T) { }) }) - Convey("And another added set with the same files and dirs", func() { - setName2 := "testRemoveFiles2" + Convey("And a set with the same files added by a different user", func() { + user, err := user.Current() + So(err, ShouldBeNil) + + setName2 := "different_user_set" + + exitCode, _ := s.runBinary(t, "add", "--name", setName2, "--transformer", + transformer, "--items", tempTestFileOfPaths.Name(), "--user", "testUser") - s.addSetForTestingWithItems(t, setName2, transformer, tempTestFileOfPaths.Name()) + So(exitCode, ShouldEqual, 0) + + s.waitForStatusWithUser(setName2, "\nStatus: complete", "testUser", 20*time.Second) Convey("Remove removes the metadata related to the set", func() { output := getRemoteMeta(filepath.Join(remotePath, "file1")) @@ -2148,32 +2185,43 @@ func TestRemove(t *testing.T) { So(string(output), ShouldContainSubstring, "file1") }) - Convey("...", func() { - exitCode, _ := s.runBinary(t, "add", "--name", "different_user_set", "--transformer", - transformer, "--items", tempTestFileOfPaths.Name(), "--user", "rk18") - + Convey("Remove on a file removes the user as a requester", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - s.waitForStatus("different_user_set", "\nStatus: complete", 20*time.Second) + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") + + So(requesters, ShouldNotContainSubstring, user.Username) }) - // add something about requesters + Convey("And a second set with the same files added by the same user", func() { + setName3 := "same_user_set" + + s.addSetForTestingWithItems(t, setName3, transformer, tempTestFileOfPaths.Name()) + + Convey("Remove keeps the user as a requester", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) + So(exitCode, ShouldEqual, 0) + + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") + + So(requesters, ShouldContainSubstring, user.Username) + }) + }) }) }) //TODO add tests for failed files }) } -func getRemoteInodePath(linkPath string) string { - output := getRemoteMeta(linkPath) - - attrFind := "attribute: ibackup:remotehardlink\nvalue: " - attrPos := strings.Index(output, attrFind) +func getMetaValue(meta, key string) string { + attrFind := "attribute: " + key + "\nvalue: " + attrPos := strings.Index(meta, attrFind) So(attrPos, ShouldNotEqual, -1) - remoteInode := output[attrPos+len(attrFind):] - nlPos := strings.Index(remoteInode, "\n") + value := meta[attrPos+len(attrFind):] + nlPos := strings.Index(value, "\n") So(nlPos, ShouldNotEqual, -1) - return remoteInode[:nlPos] + return value[:nlPos] } diff --git a/put/baton.go b/put/baton.go index 98bc467..36e71b6 100644 --- a/put/baton.go +++ b/put/baton.go @@ -33,7 +33,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "sync" "time" @@ -506,38 +505,32 @@ func metaToAVUs(meta map[string]string) []ex.AVU { // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not // associated with any other sets. Otherwise it updates the iRODS metadata for // the path to not include the given set. -func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path, setName string, meta map[string]string) error { - sets := strings.Split(meta[MetaKeySets], ",") - - sets, err := removeElementFromSlice(sets, setName) - if err != nil { - return err - } - +func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path string, sets, requesters []string, meta map[string]string) error { if len(sets) == 0 { return b.handleHardlinkAndRemoveFromIRODS(path, transformer, meta) } - err = b.RemoveMeta(path, map[string]string{MetaKeySets: meta[MetaKeySets]}) - if err != nil { - return err + metaToRemove := map[string]string{ + MetaKeySets: meta[MetaKeySets], + MetaKeyRequester: meta[MetaKeyRequester], } - return b.AddMeta(path, map[string]string{MetaKeySets: strings.Join(sets, ",")}) -} - -func removeElementFromSlice(slice []string, element string) ([]string, error) { - index := slices.Index(slice, element) - if index < 0 { - return nil, fmt.Errorf("Element %s not in slice", element) + newMeta := map[string]string{ + MetaKeySets: strings.Join(sets, ","), + MetaKeyRequester: strings.Join(requesters, ","), } - slice[index] = slice[len(slice)-1] - slice[len(slice)-1] = "" + err := b.RemoveMeta(path, metaToRemove) + if err != nil { + return err + } - return slice[:len(slice)-1], nil + return b.AddMeta(path, newMeta) } +// handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the +// path is found to be a hardlink, it checks if there are other hardlinks to the +// same file, if not, it removes the file. func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, transformer PathTransformer, meta map[string]string) error { err := b.removeFileFromIRODS(path) diff --git a/server/setdb.go b/server/setdb.go index c8d4908..6819a24 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -28,9 +28,12 @@ package server import ( "context" "errors" + "fmt" "math" "net/http" "os" + "slices" + "strings" "time" "github.com/VertebrateResequencing/wr/queue" @@ -394,14 +397,7 @@ func (s *Server) removeFiles(c *gin.Context) { return } - err = s.db.RemoveFileEntries(sid, paths) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return - } - - baton, transformer, err := s.getBatonAndTransformer(set) + err = s.removeFromIRODSandDB(set, []string{}, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -409,16 +405,12 @@ func (s *Server) removeFiles(c *gin.Context) { } // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) - err = s.removeFilesFromIRODS(set, paths, baton, transformer) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return - } c.Status(http.StatusOK) } +// getBatonAndTransformer returns a baton with a meta client and a transformer +// for the given set. func (s *Server) getBatonAndTransformer(set *set.Set) (*put.Baton, put.PathTransformer, error) { baton, err := put.GetBatonHandlerWithMetaClient() if err != nil { @@ -443,7 +435,12 @@ func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, return err } - err = baton.RemovePathFromSetInIRODS(transformer, rpath, set.Name, remoteMeta) + sets, requesters, err := s.handleSetsAndRequesters(set, remoteMeta) + if err != nil { + return err + } + + err = baton.RemovePathFromSetInIRODS(transformer, rpath, sets, requesters, remoteMeta) if err != nil { return err } @@ -452,6 +449,44 @@ func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, return nil } +func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { + sets := strings.Split(meta[put.MetaKeySets], ",") + + sets, err := removeElementFromSlice(sets, set.Name) + if err != nil { + return nil, nil, err + } + + userSets, err := s.db.GetByRequester(set.Requester) + if err != nil { + return nil, nil, err + } + + requesters := strings.Split(meta[put.MetaKeyRequester], ",") + + for _, userSet := range userSets { + if slices.Contains(sets, userSet.Name) { + return sets, requesters, nil + } + } + + requesters, err = removeElementFromSlice(requesters, set.Requester) + + return sets, requesters, err +} + +func removeElementFromSlice(slice []string, element string) ([]string, error) { + index := slices.Index(slice, element) + if index < 0 { + return nil, fmt.Errorf("Element %s not in slice", element) + } + + slice[index] = slice[len(slice)-1] + slice[len(slice)-1] = "" + + return slice[:len(slice)-1], nil +} + func (s *Server) removeDirsFromIRODS(set *set.Set, dirpaths []string, baton *put.Baton, transformer put.PathTransformer) error { for _, path := range dirpaths { @@ -495,31 +530,26 @@ func (s *Server) removeDirs(c *gin.Context) { } } - err = s.handleRemovalOfDirsFromIRODS(set, paths, filepaths) + err = s.removeFromIRODSandDB(set, paths, filepaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } - err = s.db.RemoveDirEntries(sid, paths) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return - } + c.Status(http.StatusOK) +} - err = s.db.RemoveFileEntries(sid, filepaths) +func (s *Server) removeFromIRODSandDB(set *set.Set, dirpaths, filepaths []string) error { + err := s.removePathsFromIRODS(set, dirpaths, filepaths) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return + return err } - c.Status(http.StatusOK) + return s.removePathsFromDB(set, dirpaths, filepaths) } -func (s *Server) handleRemovalOfDirsFromIRODS(set *set.Set, paths, filepaths []string) error { +func (s *Server) removePathsFromIRODS(set *set.Set, dirpaths, filepaths []string) error { baton, transformer, err := s.getBatonAndTransformer(set) if err != nil { return err @@ -530,7 +560,16 @@ func (s *Server) handleRemovalOfDirsFromIRODS(set *set.Set, paths, filepaths []s return err } - return s.removeDirsFromIRODS(set, paths, baton, transformer) + return s.removeDirsFromIRODS(set, dirpaths, baton, transformer) +} + +func (s *Server) removePathsFromDB(set *set.Set, dirpaths, filepaths []string) error { + err := s.db.RemoveDirEntries(set.ID(), dirpaths) + if err != nil { + return err + } + + return s.db.RemoveFileEntries(set.ID(), filepaths) } // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id diff --git a/set/db.go b/set/db.go index e27fe22..4af490b 100644 --- a/set/db.go +++ b/set/db.go @@ -325,13 +325,9 @@ func (d *DB) GetFilesInDir(setID string, dirpath string, filepaths []string) ([] return nil, err } - //dirpath = filepath.Clean(dirpath) + "/" - for _, entry := range entries { path := entry.Path - //path = filepath.Clean(path) - if strings.HasPrefix(path, dirpath) { filepaths = append(filepaths, path) } From fc55102633e0b4f09fbc4539c9332893893676af Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 22 Jan 2025 16:33:04 +0000 Subject: [PATCH 09/35] Add extra validation for different user`s sets with the same name WIP --- main_test.go | 38 ++++++++++++++++++++++++++++++++++++++ put/baton.go | 3 ++- server/setdb.go | 30 +++++++++++++++++++++--------- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/main_test.go b/main_test.go index 790c5b0..031453f 100644 --- a/main_test.go +++ b/main_test.go @@ -2209,6 +2209,44 @@ func TestRemove(t *testing.T) { }) }) }) + + Convey("And a set with the same files and name added by a different user", func() { + user, err := user.Current() + So(err, ShouldBeNil) + + setName2 := setName + + exitCode, _ := s.runBinary(t, "add", "--name", setName2, "--transformer", + transformer, "--items", tempTestFileOfPaths.Name(), "--user", "testUser") + + So(exitCode, ShouldEqual, 0) + + s.waitForStatusWithUser(setName2, "\nStatus: complete", "testUser", 20*time.Second) + + Convey("Remove on a file removes the user as a requester but not the file", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) + So(exitCode, ShouldEqual, 0) + + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") + + So(requesters, ShouldNotContainSubstring, user.Username) + }) + + Convey("And a second set with the same files added by the same user", func() { + setName3 := "same_user_set" + + s.addSetForTestingWithItems(t, setName3, transformer, tempTestFileOfPaths.Name()) + + Convey("Remove keeps the user as a requester", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) + So(exitCode, ShouldEqual, 0) + + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") + + So(requesters, ShouldContainSubstring, user.Username) + }) + }) + }) }) //TODO add tests for failed files }) diff --git a/put/baton.go b/put/baton.go index 36e71b6..e0d06f0 100644 --- a/put/baton.go +++ b/put/baton.go @@ -505,7 +505,8 @@ func metaToAVUs(meta map[string]string) []ex.AVU { // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not // associated with any other sets. Otherwise it updates the iRODS metadata for // the path to not include the given set. -func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path string, sets, requesters []string, meta map[string]string) error { +func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path string, + sets, requesters []string, meta map[string]string) error { if len(sets) == 0 { return b.handleHardlinkAndRemoveFromIRODS(path, transformer, meta) } diff --git a/server/setdb.go b/server/setdb.go index 6819a24..34739ea 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -451,28 +451,40 @@ func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { sets := strings.Split(meta[put.MetaKeySets], ",") + requesters := strings.Split(meta[put.MetaKeyRequester], ",") + + userSets, err := s.db.GetByRequester(set.Requester) + + lessRequesters, err := removeElementFromSlice(requesters, set.Requester) + + var otherUserSets []string + for _, requester := range lessRequesters { + otherSets, err := s.db.GetByRequester(requester) + if err != nil { + return nil, nil, err + } + for _, otherSet := range otherSets { + otherUserSets = append(otherUserSets, otherSet.Name) + } - sets, err := removeElementFromSlice(sets, set.Name) - if err != nil { - return nil, nil, err } - userSets, err := s.db.GetByRequester(set.Requester) + sets, err = removeElementFromSlice(sets, set.Name) if err != nil { return nil, nil, err } - requesters := strings.Split(meta[put.MetaKeyRequester], ",") - for _, userSet := range userSets { if slices.Contains(sets, userSet.Name) { - return sets, requesters, nil + lessRequesters = requesters } } - requesters, err = removeElementFromSlice(requesters, set.Requester) + if slices.Contains(otherUserSets, set.Name) { + sets = append(sets, set.Name) + } - return sets, requesters, err + return sets, lessRequesters, err } func removeElementFromSlice(slice []string, element string) ([]string, error) { From 3e418f2ce263101a4bcdda910df0bf685bf3aed7 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Thu, 23 Jan 2025 11:59:56 +0000 Subject: [PATCH 10/35] Refactor handleSetsAndRequesters --- server/setdb.go | 57 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/server/setdb.go b/server/setdb.go index 34739ea..eaf9bc2 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -453,38 +453,59 @@ func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ( sets := strings.Split(meta[put.MetaKeySets], ",") requesters := strings.Split(meta[put.MetaKeyRequester], ",") - userSets, err := s.db.GetByRequester(set.Requester) - - lessRequesters, err := removeElementFromSlice(requesters, set.Requester) + otherUserSets, userSets, err := s.getSetNamesByRequesters(requesters, set.Requester) + if err != nil { + return nil, nil, err + } - var otherUserSets []string - for _, requester := range lessRequesters { - otherSets, err := s.db.GetByRequester(requester) + if len(userSets) == 1 && userSets[0] == set.Name { + requesters, err = removeElementFromSlice(requesters, set.Requester) if err != nil { return nil, nil, err } - for _, otherSet := range otherSets { - otherUserSets = append(otherUserSets, otherSet.Name) - } + } + if slices.Contains(otherUserSets, set.Name) { + return sets, requesters, nil } sets, err = removeElementFromSlice(sets, set.Name) - if err != nil { - return nil, nil, err - } - for _, userSet := range userSets { - if slices.Contains(sets, userSet.Name) { - lessRequesters = requesters + return sets, requesters, err +} + +func (s *Server) getSetNamesByRequesters(requesters []string, user string) ([]string, []string, error) { + var ( + otherUserSets []string + curUserSets []string + ) + + for _, requester := range requesters { + requesterSets, err := s.db.GetByRequester(requester) + if err != nil { + return nil, nil, err } + + if requester == user { + curUserSets = append(curUserSets, getNamesFromSets(requesterSets)...) + + continue + } + + otherUserSets = append(otherUserSets, getNamesFromSets(requesterSets)...) } - if slices.Contains(otherUserSets, set.Name) { - sets = append(sets, set.Name) + return otherUserSets, curUserSets, nil +} + +func getNamesFromSets(sets []*set.Set) []string { + names := make([]string, len(sets)) + + for i, set := range sets { + names[i] = set.Name } - return sets, lessRequesters, err + return names } func removeElementFromSlice(slice []string, element string) ([]string, error) { From 043d554a72681b6a3fdffc48e91b8f57f8aa519c Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 24 Jan 2025 09:58:44 +0000 Subject: [PATCH 11/35] Fix bug when removing a failed file that was successful in a different set --- put/baton.go | 5 +++++ server/setdb.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/put/baton.go b/put/baton.go index e0d06f0..3c8f74f 100644 --- a/put/baton.go +++ b/put/baton.go @@ -521,6 +521,11 @@ func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path strin MetaKeyRequester: strings.Join(requesters, ","), } + if metaToRemove[MetaKeySets] == newMeta[MetaKeySets] && + metaToRemove[MetaKeyRequester] == newMeta[MetaKeyRequester] { + return nil + } + err := b.RemoveMeta(path, metaToRemove) if err != nil { return err diff --git a/server/setdb.go b/server/setdb.go index eaf9bc2..0bddbc2 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -453,6 +453,10 @@ func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ( sets := strings.Split(meta[put.MetaKeySets], ",") requesters := strings.Split(meta[put.MetaKeyRequester], ",") + if !slices.Contains(sets, set.Name) { + return sets, requesters, nil + } + otherUserSets, userSets, err := s.getSetNamesByRequesters(requesters, set.Requester) if err != nil { return nil, nil, err From 26fc4bebe33412cf7b9ef6431d5b3fc07046ec9e Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 24 Jan 2025 12:07:18 +0000 Subject: [PATCH 12/35] Update set counts after removal WIP --- main_test.go | 10 +++++++--- server/setdb.go | 12 +++++++++--- set/db.go | 11 +++++++++++ set/set.go | 15 ++++++++++++--- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/main_test.go b/main_test.go index 031453f..97bb48f 100644 --- a/main_test.go +++ b/main_test.go @@ -1924,7 +1924,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - Convey("Given a server", t, func() { + FocusConvey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1953,7 +1953,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - Convey("And an added set with files and folders", func() { + FocusConvey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -1995,7 +1995,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - Convey("Remove removes the file from the set", func() { + FocusConvey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) @@ -2005,6 +2005,10 @@ func TestRemove(t *testing.T) { s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, 0, file1) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded): 30 B / 30 B\n"+ + "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") }) Convey("Remove removes the dir from the set", func() { diff --git a/server/setdb.go b/server/setdb.go index 0bddbc2..162875c 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -577,13 +577,19 @@ func (s *Server) removeDirs(c *gin.Context) { c.Status(http.StatusOK) } -func (s *Server) removeFromIRODSandDB(set *set.Set, dirpaths, filepaths []string) error { - err := s.removePathsFromIRODS(set, dirpaths, filepaths) +func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { + err := s.removePathsFromIRODS(userSet, dirpaths, filepaths) if err != nil { return err } - return s.removePathsFromDB(set, dirpaths, filepaths) + err = s.removePathsFromDB(userSet, dirpaths, filepaths) + if err != nil { + return err + } + + return s.db.SetNewCounts(userSet.ID(), userSet.NumFiles-uint64(len(filepaths))) + } func (s *Server) removePathsFromIRODS(set *set.Set, dirpaths, filepaths []string) error { diff --git a/set/db.go b/set/db.go index 4af490b..3e8d9bc 100644 --- a/set/db.go +++ b/set/db.go @@ -1149,6 +1149,17 @@ func (d *DB) SetError(setID, errMsg string) error { }) } +func (d *DB) SetNewCounts(setID string, numFiles uint64) error { + return d.updateSetProperties(setID, func(got *Set) { + got.NumFiles = numFiles + + err := got.FixCounts(&Entry{}, d.GetFileEntries) + if err != nil { + // + } + }) +} + // updateSetProperties retrives a set from the database and gives it to your // callback, allowing you to change properties on it. The altered set will then // be stored back in the database. diff --git a/set/set.go b/set/set.go index 69d1d51..a20412d 100644 --- a/set/set.go +++ b/set/set.go @@ -503,7 +503,7 @@ func (s *Set) UpdateBasedOnEntry(entry *Entry, getFileEntries func(string) ([]*E s.adjustBasedOnEntry(entry) - err := s.fixCounts(entry, getFileEntries) + err := s.FixCounts(entry, getFileEntries) if err != nil { return err } @@ -538,10 +538,10 @@ func (s *Set) checkIfComplete() { s.Uploaded, s.Replaced, s.Skipped, s.Failed, s.Missing, s.Abnormal, s.UploadedSize())) } -// fixCounts resets the set counts to 0 and goes through all the entries for +// FixCounts resets the set counts to 0 and goes through all the entries for // the set in the db to recaluclate them. The supplied entry should be one you // newly updated and that wasn't in the db before the transaction we're in. -func (s *Set) fixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, error)) error { +func (s *Set) FixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, error)) error { if s.countsValid() { return nil } @@ -582,6 +582,15 @@ func (s *Set) updateAllCounts(entries []*Entry, entry *Entry) { } } +// updateAllCounts should be called after setting all counts to 0 (because they +// had become invalid), and then recalculates the counts. Also marks the given +// entry as newFail if any entry in entries is Failed. +func (s *Set) updateSetSize(entries []*Entry) { + for _, e := range entries { + s.SizeTotal += e.Size + } +} + // SetError records the given error against the set, indicating it wont work. func (s *Set) SetError(errMsg string) { s.Error = errMsg From ace746bbb8fe288f74ffeb0c59120d267a5cb280 Mon Sep 17 00:00:00 2001 From: Iaroslav Popov Date: Fri, 31 Jan 2025 17:25:57 +0000 Subject: [PATCH 13/35] Refactor SetNewCounts to recalculate file counts --- server/setdb.go | 2 +- set/db.go | 17 +++++++++++------ set/set.go | 24 +++++++++--------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/server/setdb.go b/server/setdb.go index 162875c..468cb0d 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -588,7 +588,7 @@ func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []st return err } - return s.db.SetNewCounts(userSet.ID(), userSet.NumFiles-uint64(len(filepaths))) + return s.db.SetNewCounts(userSet.ID()) } diff --git a/set/db.go b/set/db.go index 3e8d9bc..b34ef0c 100644 --- a/set/db.go +++ b/set/db.go @@ -1149,14 +1149,19 @@ func (d *DB) SetError(setID, errMsg string) error { }) } -func (d *DB) SetNewCounts(setID string, numFiles uint64) error { +func (d *DB) SetNewCounts(setID string) error { + entries, err := d.GetFileEntries(setID) + if err != nil { + return err + } + return d.updateSetProperties(setID, func(got *Set) { - got.NumFiles = numFiles + got.resetCounts() + got.NumFiles = uint64(len(entries)) + got.updateAllCounts(entries, &Entry{}) - err := got.FixCounts(&Entry{}, d.GetFileEntries) - if err != nil { - // - } + got.SizeTotal = 0 + got.updateSetSize(entries) }) } diff --git a/set/set.go b/set/set.go index a20412d..a1b815c 100644 --- a/set/set.go +++ b/set/set.go @@ -551,6 +551,13 @@ func (s *Set) FixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, err return err } + s.resetCounts() + s.updateAllCounts(entries, entry) + + return nil +} + +func (s *Set) resetCounts() { s.Uploaded = 0 s.Replaced = 0 s.Skipped = 0 @@ -559,10 +566,6 @@ func (s *Set) FixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, err s.Abnormal = 0 s.Symlinks = 0 s.Hardlinks = 0 - - s.updateAllCounts(entries, entry) - - return nil } // updateAllCounts should be called after setting all counts to 0 (because they @@ -582,12 +585,10 @@ func (s *Set) updateAllCounts(entries []*Entry, entry *Entry) { } } -// updateAllCounts should be called after setting all counts to 0 (because they -// had become invalid), and then recalculates the counts. Also marks the given -// entry as newFail if any entry in entries is Failed. func (s *Set) updateSetSize(entries []*Entry) { for _, e := range entries { s.SizeTotal += e.Size + // s.SizeUploaded } } @@ -627,17 +628,10 @@ func (s *Set) reset() { s.NumFiles = 0 s.SizeTotal = 0 s.SizeUploaded = 0 - s.Uploaded = 0 - s.Replaced = 0 - s.Skipped = 0 - s.Failed = 0 - s.Missing = 0 - s.Abnormal = 0 - s.Symlinks = 0 - s.Hardlinks = 0 s.Status = PendingDiscovery s.Error = "" s.Warning = "" + s.resetCounts() } func (s *Set) UserMetadata() string { From 73708fed1e837fd1c003184d0b43c6af1ccc7f57 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Mon, 3 Feb 2025 15:33:07 +0000 Subject: [PATCH 14/35] Combine removeFiles and removeDirs endpoints and add recently removed to status --- cmd/remove.go | 10 +-- cmd/status.go | 4 +- main_test.go | 21 ++++- server/client.go | 19 +++- server/setdb.go | 226 +++++++++++++++++++++++++++++++++-------------- set/db.go | 43 +++++---- set/set.go | 41 +++++++++ set/set_test.go | 5 +- 8 files changed, 263 insertions(+), 106 deletions(-) diff --git a/cmd/remove.go b/cmd/remove.go index 84ebaae..2d20706 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -132,13 +132,13 @@ func remove(client *server.Client, user, name string, files, dirs []string) { return } - err := client.RemoveFiles(sets[0].ID(), files) + err := client.RemoveFiles(sets[0].ID(), files, dirs) if err != nil { die("%s", err.Error()) } - err = client.RemoveDirs(sets[0].ID(), dirs) - if err != nil { - die("%s", err.Error()) - } + // err = client.RemoveDirs(sets[0].ID(), dirs) + // if err != nil { + // die("%s", err.Error()) + // } } diff --git a/cmd/status.go b/cmd/status.go index af8c47e..7be5742 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -410,8 +410,8 @@ func displaySet(s *set.Set, showRequesters bool) { //nolint:funlen,gocyclo } cliPrint("Discovery: %s\n", s.Discovered()) - cliPrint("Num files: %s; Symlinks: %d; Hardlinks: %d; Size (total/recently uploaded): %s / %s\n", - s.Count(), s.Symlinks, s.Hardlinks, s.Size(), s.UploadedSize()) + cliPrint("Num files: %s; Symlinks: %d; Hardlinks: %d; Size (total/recently uploaded/recently removed): %s / %s / %s\n", + s.Count(), s.Symlinks, s.Hardlinks, s.Size(), s.UploadedSize(), s.RemovedSize()) cliPrint("Uploaded: %d; Replaced: %d; Skipped: %d; Failed: %d; Missing: %d; Abnormal: %d\n", s.Uploaded, s.Replaced, s.Skipped, s.Failed, s.Missing, s.Abnormal) diff --git a/main_test.go b/main_test.go index 97bb48f..28b7c65 100644 --- a/main_test.go +++ b/main_test.go @@ -1924,7 +1924,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - FocusConvey("Given a server", t, func() { + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1953,7 +1953,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - FocusConvey("And an added set with files and folders", func() { + Convey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -1995,7 +1995,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - FocusConvey("Remove removes the file from the set", func() { + Convey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) @@ -2007,8 +2007,21 @@ func TestRemove(t *testing.T) { 0, file1) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, - 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded): 30 B / 30 B\n"+ + 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") + + Convey("Remove again will remove another file", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file2) + + So(exitCode, ShouldEqual, 0) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, file2) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, "Num files: 4; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 20 B / 40 B / 10 B\n"+ + "Uploaded: 4; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") + }) }) Convey("Remove removes the dir from the set", func() { diff --git a/server/client.go b/server/client.go index 6b038df..2926d45 100644 --- a/server/client.go +++ b/server/client.go @@ -48,6 +48,9 @@ const ( numHandlers = 2 millisecondsInSecond = 1000 + fileKeyForJSON = "files" + dirKeyForJSON = "dirs" + ErrKilledDueToStuck = gas.Error(put.ErrStuckTimeout) ) @@ -615,10 +618,18 @@ func (c *Client) RetryFailedSetUploads(id string) (int, error) { return retried, err } -func (c *Client) RemoveFiles(setID string, files []string) error { - return c.putThing(EndPointAuthRemoveFiles+"/"+setID, stringsToBytes(files)) +func (c *Client) RemoveFiles(setID string, files, dirs []string) error { + return c.putThing(EndPointAuthRemovePaths+"/"+setID, mapToBytes(files, dirs)) } -func (c *Client) RemoveDirs(setID string, dirs []string) error { - return c.putThing(EndPointAuthRemoveDirs+"/"+setID, stringsToBytes(dirs)) +func mapToBytes(files, dirs []string) map[string][][]byte { + newMap := make(map[string][][]byte) + newMap[fileKeyForJSON] = stringsToBytes(files) + newMap[dirKeyForJSON] = stringsToBytes(dirs) + + return newMap } + +// func (c *Client) RemoveDirs(setID string, dirs []string) error { +// return c.putThing(EndPointAuthRemoveDirs+"/"+setID, stringsToBytes(dirs)) +// } diff --git a/server/setdb.go b/server/setdb.go index 468cb0d..dc7bd2c 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -57,7 +57,7 @@ const ( workingPath = "/working" fileStatusPath = "/file_status" fileRetryPath = "/retry" - removeFilesPath = "/remove_files" + removePathsPath = "/remove_paths" removeDirsPath = "/remove_dirs" // EndPointAuthSet is the endpoint for getting and setting sets. @@ -96,7 +96,7 @@ const ( EndPointAuthRetryEntries = gas.EndPointAuth + fileRetryPath // - EndPointAuthRemoveFiles = gas.EndPointAuth + removeFilesPath + EndPointAuthRemovePaths = gas.EndPointAuth + removePathsPath // EndPointAuthRemoveDirs = gas.EndPointAuth + removeDirsPath @@ -280,8 +280,8 @@ func (s *Server) addDBEndpoints(authGroup *gin.RouterGroup) { authGroup.GET(fileRetryPath+idParam, s.retryFailedEntries) - authGroup.PUT(removeFilesPath+idParam, s.removeFiles) - authGroup.PUT(removeDirsPath+idParam, s.removeDirs) + authGroup.PUT(removePathsPath+idParam, s.removePaths) + //authGroup.PUT(removeDirsPath+idParam, s.removeDirs) } // putSet interprets the body as a JSON encoding of a set.Set and stores it in @@ -382,31 +382,49 @@ func (s *Server) putFiles(c *gin.Context) { c.Status(http.StatusOK) } -func (s *Server) removeFiles(c *gin.Context) { - sid, paths, ok := s.bindPathsAndValidateSet(c) +func (s *Server) removePaths(c *gin.Context) { + sid, filePaths, dirPaths, ok := s.parseRemoveParamsAndValidateSet(c) if !ok { return } set := s.db.GetByID(sid) - err := s.db.ValidateFilePaths(set, paths) + err := s.db.ValidateFilePaths(set, filePaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } - err = s.removeFromIRODSandDB(set, []string{}, paths) + err = s.db.ValidateDirPaths(set, dirPaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } + err = s.db.ResetRemoveSize(sid) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + s.removeFiles(set, filePaths) + + s.removeDirs(set, dirPaths) +} + +func (s *Server) removeFiles(set *set.Set, paths []string) error { + err := s.removeFileFromIRODSandDB(set, paths) + if err != nil { + return err + } + // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) - c.Status(http.StatusOK) + return nil } // getBatonAndTransformer returns a baton with a meta client and a transformer @@ -449,6 +467,26 @@ func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, return nil } +func (s *Server) removeFileFromIRODS(set *set.Set, path string, + baton *put.Baton, transformer put.PathTransformer) error { + rpath, err := transformer(path) + if err != nil { + return err + } + + remoteMeta, err := baton.GetMeta(rpath) + if err != nil { + return err + } + + sets, requesters, err := s.handleSetsAndRequesters(set, remoteMeta) + if err != nil { + return err + } + + return baton.RemovePathFromSetInIRODS(transformer, rpath, sets, requesters, remoteMeta) +} + func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { sets := strings.Split(meta[put.MetaKeySets], ",") requesters := strings.Split(meta[put.MetaKeyRequester], ",") @@ -524,96 +562,130 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { return slice[:len(slice)-1], nil } -func (s *Server) removeDirsFromIRODS(set *set.Set, dirpaths []string, +func (s *Server) removeDirFromIRODS(set *set.Set, path string, baton *put.Baton, transformer put.PathTransformer) error { - for _, path := range dirpaths { - rpath, err := transformer(path) - if err != nil { - return err - } - err = baton.RemoveDirFromIRODS(rpath) - if err != nil { - return err - } + rpath, err := transformer(path) + if err != nil { + return err + } + + err = baton.RemoveDirFromIRODS(rpath) + if err != nil { + return err } return nil } -func (s *Server) removeDirs(c *gin.Context) { - sid, paths, ok := s.bindPathsAndValidateSet(c) - if !ok { - return - } +func (s *Server) removeDirs(set *set.Set, paths []string) error { + var filepaths []string + var err error - set := s.db.GetByID(sid) + for _, path := range paths { + filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) + if err != nil { + return err + } + } - err := s.db.ValidateDirPaths(set, paths) + err = s.removeFileFromIRODSandDB(set, filepaths) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return + return err } - var filepaths []string + return s.removeDirFromIRODSandDB(set, paths) +} +func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, paths []string) error { for _, path := range paths { - filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) + entry, err := s.db.GetFileEntryForSet(userSet.ID(), path) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + return err + } - return + baton, transformer, err := s.getBatonAndTransformer(userSet) + if err != nil { + return err } - } - err = s.removeFromIRODSandDB(set, paths, filepaths) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + err = s.removeFileFromIRODS(userSet, path, baton, transformer) + if err != nil { + return err + } - return + err = s.db.RemoveFileEntries(userSet.ID(), path) + if err != nil { + return err + } + + err = s.db.UpdateBasedOnRemovedEntry(userSet.ID(), entry) + if err != nil { + return err + } } - c.Status(http.StatusOK) + return nil } -func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { - err := s.removePathsFromIRODS(userSet, dirpaths, filepaths) - if err != nil { - return err - } +func (s *Server) removeDirFromIRODSandDB(userSet *set.Set, paths []string) error { + for _, path := range paths { + baton, transformer, err := s.getBatonAndTransformer(userSet) + if err != nil { + return err + } - err = s.removePathsFromDB(userSet, dirpaths, filepaths) - if err != nil { - return err - } + err = s.removeDirFromIRODS(userSet, path, baton, transformer) + if err != nil { + return err + } - return s.db.SetNewCounts(userSet.ID()) + err = s.db.RemoveDirEntries(userSet.ID(), path) + if err != nil { + return err + } + } + return nil } -func (s *Server) removePathsFromIRODS(set *set.Set, dirpaths, filepaths []string) error { - baton, transformer, err := s.getBatonAndTransformer(set) - if err != nil { - return err - } +// func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { +// err := s.removePathsFromIRODS(userSet, dirpaths, filepaths) +// if err != nil { +// return err +// } - err = s.removeFilesFromIRODS(set, filepaths, baton, transformer) - if err != nil { - return err - } +// err = s.removePathsFromDB(userSet, dirpaths, filepaths) +// if err != nil { +// return err +// } - return s.removeDirsFromIRODS(set, dirpaths, baton, transformer) -} +// return s.db.SetNewCounts(userSet.ID()) -func (s *Server) removePathsFromDB(set *set.Set, dirpaths, filepaths []string) error { - err := s.db.RemoveDirEntries(set.ID(), dirpaths) - if err != nil { - return err - } +// } - return s.db.RemoveFileEntries(set.ID(), filepaths) -} +// func (s *Server) removePathsFromIRODS(set *set.Set, dirpaths, filepaths []string) error { +// baton, transformer, err := s.getBatonAndTransformer(set) +// if err != nil { +// return err +// } + +// err = s.removeFilesFromIRODS(set, filepaths, baton, transformer) +// if err != nil { +// return err +// } + +// return s.removeDirsFromIRODS(set, dirpaths, baton, transformer) +// } + +// func (s *Server) removePathsFromDB(set *set.Set, dirpaths, filepaths []string) error { +// err := s.db.RemoveDirEntries(set.ID(), dirpaths) +// if err != nil { +// return err +// } + +// return s.db.RemoveFileEntries(set.ID(), filepaths) +// } // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id // from the URL parameter if Requester matches logged-in username. @@ -634,6 +706,26 @@ func (s *Server) bindPathsAndValidateSet(c *gin.Context) (string, []string, bool return set.ID(), bytesToStrings(bpaths), true } +// parseRemoveParamsAndValidateSet gets the file paths and dir paths out of the +// JSON body, and the set id from the URL parameter if Requester matches +// logged-in username. +func (s *Server) parseRemoveParamsAndValidateSet(c *gin.Context) (string, []string, []string, bool) { + bmap := make(map[string][][]byte) + + if err := c.BindJSON(&bmap); err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return "", nil, nil, false + } + + set, ok := s.validateSet(c) + if !ok { + return "", nil, nil, false + } + + return set.ID(), bytesToStrings(bmap[fileKeyForJSON]), bytesToStrings(bmap[dirKeyForJSON]), true +} + // validateSet gets the id parameter from the given context and checks a // corresponding set exists and the logged-in user is the same as the set's // Requester. If so, returns the set and true. If not, Aborts with an error diff --git a/set/db.go b/set/db.go index b34ef0c..617c6a7 100644 --- a/set/db.go +++ b/set/db.go @@ -285,29 +285,27 @@ func (d *DB) ValidateDirPaths(set *Set, paths []string) error { } // RemoveFileEntries removes the provided files from a given set. -func (d *DB) RemoveFileEntries(setID string, paths []string) error { - err := d.removeEntries(setID, paths, fileBucket) +func (d *DB) RemoveFileEntries(setID string, path string) error { + err := d.removeEntries(setID, path, fileBucket) if err != nil { return err } - return d.removeEntries(setID, paths, discoveredBucket) + return d.removeEntries(setID, path, discoveredBucket) } // removeEntries removes the entries with the provided entry keys from a given // bucket of a given set. -func (d *DB) removeEntries(setID string, entryKeys []string, bucketName string) error { +func (d *DB) removeEntries(setID string, entryKey string, bucketName string) error { return d.db.Update(func(tx *bolt.Tx) error { subBucketName := []byte(bucketName + separator + setID) setsBucket := tx.Bucket([]byte(setsBucket)) entriesBucket := setsBucket.Bucket(subBucketName) - for _, v := range entryKeys { - err := entriesBucket.Delete([]byte(v)) - if err != nil { - return err - } + err := entriesBucket.Delete([]byte(entryKey)) + if err != nil { + return err } return nil @@ -315,8 +313,8 @@ func (d *DB) removeEntries(setID string, entryKeys []string, bucketName string) } // RemoveDirEntries removes the provided directories from a given set. -func (d *DB) RemoveDirEntries(setID string, paths []string) error { - return d.removeEntries(setID, paths, dirBucket) +func (d *DB) RemoveDirEntries(setID string, path string) error { + return d.removeEntries(setID, path, dirBucket) } func (d *DB) GetFilesInDir(setID string, dirpath string, filepaths []string) ([]string, error) { @@ -1149,19 +1147,20 @@ func (d *DB) SetError(setID, errMsg string) error { }) } -func (d *DB) SetNewCounts(setID string) error { - entries, err := d.GetFileEntries(setID) - if err != nil { - return err - } - +func (d *DB) UpdateBasedOnRemovedEntry(setID string, entry *Entry) error { return d.updateSetProperties(setID, func(got *Set) { - got.resetCounts() - got.NumFiles = uint64(len(entries)) - got.updateAllCounts(entries, &Entry{}) + got.SizeRemoved += entry.Size + got.SizeTotal -= entry.Size + + got.NumFiles -= 1 + + got.removedEntryToSetCounts(entry) + }) +} - got.SizeTotal = 0 - got.updateSetSize(entries) +func (d *DB) ResetRemoveSize(setID string) error { + return d.updateSetProperties(setID, func(got *Set) { + got.SizeRemoved = 0 }) } diff --git a/set/set.go b/set/set.go index a1b815c..175cddc 100644 --- a/set/set.go +++ b/set/set.go @@ -197,6 +197,10 @@ type Set struct { // skipped) since the last discovery. This is a read-only value. SizeUploaded uint64 + // SizeRemoved provides the size of files (bytes) part of the most recent + // remove. This is a read-only value. + SizeRemoved uint64 + // Error holds any error that applies to the whole set, such as an issue // with the Transformer. This is a read-only value. Error string @@ -279,6 +283,12 @@ func (s *Set) UploadedSize() string { return humanize.IBytes(s.SizeUploaded) //nolint:misspell } +// RemovedSize provides a string representation of SizeRemoved in a human +// readable format. This is the size of files part of the most recent remove. +func (s *Set) RemovedSize() string { + return humanize.IBytes(s.SizeRemoved) //nolint:misspell +} + func (s *Set) TransformPath(path string) (string, error) { transformer, err := s.MakeTransformer() if err != nil { @@ -418,6 +428,11 @@ func (s *Set) entryToSetCounts(entry *Entry) { s.entryTypeToSetCounts(entry) } +func (s *Set) removedEntryToSetCounts(entry *Entry) { + s.removedEntryStatusToSetCounts(entry) + s.removedEntryTypeToSetCounts(entry) +} + func (s *Set) entryStatusToSetCounts(entry *Entry) { //nolint:gocyclo switch entry.Status { //nolint:exhaustive case Uploaded: @@ -442,6 +457,23 @@ func (s *Set) entryStatusToSetCounts(entry *Entry) { //nolint:gocyclo } } +func (s *Set) removedEntryStatusToSetCounts(entry *Entry) { //nolint:gocyclo + switch entry.Status { //nolint:exhaustive + case Uploaded: + s.Uploaded-- + case Replaced: + s.Replaced-- + case Skipped: + s.Skipped-- + case Failed: + s.Failed-- + case Missing: + s.Missing-- + case AbnormalEntry: + s.Abnormal-- + } +} + func (s *Set) sendSlackMessage(level slack.Level, msg string) { if s.slacker == nil { return @@ -463,6 +495,15 @@ func (s *Set) entryTypeToSetCounts(entry *Entry) { } } +func (s *Set) removedEntryTypeToSetCounts(entry *Entry) { + switch entry.Type { //nolint:exhaustive + case Symlink: + s.Symlinks-- + case Hardlink: + s.Hardlinks-- + } +} + // LogChangesToSlack will cause the set to use the slacker when significant // events happen to the set. func (s *Set) LogChangesToSlack(slacker Slacker) { diff --git a/set/set_test.go b/set/set_test.go index 89c0134..90fc285 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -358,7 +358,8 @@ func TestSetDB(t *testing.T) { So(err, ShouldBeNil) Convey("Then remove files and dirs from the sets", func() { - db.RemoveFileEntries(set.ID(), []string{"/a/b.txt"}) + err := db.removeEntries(set.ID(), "/a/b.txt", fileBucket) + So(err, ShouldBeNil) fEntries, errg := db.GetFileEntries(set.ID()) So(errg, ShouldBeNil) @@ -366,7 +367,7 @@ func TestSetDB(t *testing.T) { So(fEntries[0], ShouldResemble, &Entry{Path: "/c/d.txt"}) So(fEntries[1], ShouldResemble, &Entry{Path: "/e/f.txt"}) - db.RemoveDirEntries(set.ID(), []string{"/g/h"}) + db.RemoveDirEntries(set.ID(), "/g/h") dEntries, errg := db.GetDirEntries(set.ID()) So(errg, ShouldBeNil) From 48dc79dc264431c6380082abe71543c3e6a7bb78 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Mon, 3 Feb 2025 16:17:38 +0000 Subject: [PATCH 15/35] Implement removeQueue WIP --- main_test.go | 11 +++++-- server/server.go | 29 ++++++++++++++---- server/setdb.go | 76 +++++++++++++++++++++++++++++------------------- 3 files changed, 77 insertions(+), 39 deletions(-) diff --git a/main_test.go b/main_test.go index 28b7c65..d07dffb 100644 --- a/main_test.go +++ b/main_test.go @@ -1924,7 +1924,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - Convey("Given a server", t, func() { + FocusConvey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1953,7 +1953,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - Convey("And an added set with files and folders", func() { + FocusConvey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -1995,11 +1995,16 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - Convey("Remove removes the file from the set", func() { + FocusConvey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + + // s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + // 0, "Removal status: 0 / 100 files removed") + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, file2) diff --git a/server/server.go b/server/server.go index 7fb992d..b7ecf40 100644 --- a/server/server.go +++ b/server/server.go @@ -96,7 +96,7 @@ type Server struct { cacheMu sync.Mutex dirPool *workerpool.WorkerPool queue *queue.Queue - // removeQueue *queue.Queue + removeQueue *queue.Queue sched *scheduler.Scheduler putCmd string req *jqs.Requirements @@ -129,7 +129,7 @@ func New(conf Config) (*Server, error) { numClients: 1, dirPool: workerpool.New(workerPoolSizeDir), queue: queue.New(context.Background(), "put"), - // removeQueue: queue.New(context.Background(), "remove"), + removeQueue: queue.New(context.Background(), "remove"), creatingCollections: make(map[string]bool), slacker: conf.Slacker, stillRunningMsgFreq: conf.StillRunningMsgFreq, @@ -141,7 +141,7 @@ func New(conf Config) (*Server, error) { s.clientQueue.SetTTRCallback(s.clientTTRC) s.Server.Router().Use(gas.IncludeAbortErrorsInBody) - // s.removeQueue.SetReadyAddedCallback(s.removeCallback) + s.removeQueue.SetReadyAddedCallback(s.removeCallback) s.monitor = NewMonitor(func(given *set.Set) { if err := s.discoverSet(given); err != nil { @@ -203,9 +203,26 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC return nil } -// func (s *Server) removeCallback(_ string, item []interface{}) { -// fmt.Println("removeQueue callback called") -// } +func (s *Server) removeCallback(_ string, allitemdata []interface{}) { + fmt.Println("removeQueue callback called") + + item := allitemdata[0] + removeReq, _ := item.(removeReq) + + fmt.Println(removeReq) + + if removeReq.isDir { + // handle dir + return + } + + err := s.removeFileFromIRODSandDB(removeReq.set, removeReq.path) + if err != nil { + // + } + + // handle file +} // rac is our queue's ready added callback which will get all ready put Requests // and ensure there are enough put jobs added to wr. diff --git a/server/setdb.go b/server/setdb.go index dc7bd2c..5f1552b 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -416,15 +416,38 @@ func (s *Server) removePaths(c *gin.Context) { s.removeDirs(set, dirPaths) } +type removeReq struct { + path string + set *set.Set + isDir bool +} + func (s *Server) removeFiles(set *set.Set, paths []string) error { - err := s.removeFileFromIRODSandDB(set, paths) - if err != nil { - return err + // err := s.removeFileFromIRODSandDB(set, paths) + // if err != nil { + // return err + // } + if len(paths) == 0 { + return nil } - // s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) + defs := make([]*queue.ItemDef, len(paths)) - return nil + for i, path := range paths { + defs[i] = &queue.ItemDef{ + Key: strings.Join([]string{set.ID(), path}, ":"), + Data: removeReq{path: path, set: set, isDir: false}, + TTR: ttr, + } + //s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) + } + + _, dups, err := s.removeQueue.AddMany(context.Background(), defs) + if dups != 0 { + return fmt.Errorf("dups??") + } + + return err } // getBatonAndTransformer returns a baton with a meta client and a transformer @@ -589,7 +612,7 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { } } - err = s.removeFileFromIRODSandDB(set, filepaths) + //err = s.removeFileFromIRODSandDB(set, filepaths) if err != nil { return err } @@ -597,35 +620,28 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { return s.removeDirFromIRODSandDB(set, paths) } -func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, paths []string) error { - for _, path := range paths { - entry, err := s.db.GetFileEntryForSet(userSet.ID(), path) - if err != nil { - return err - } - - baton, transformer, err := s.getBatonAndTransformer(userSet) - if err != nil { - return err - } +func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, path string) error { + entry, err := s.db.GetFileEntryForSet(userSet.ID(), path) + if err != nil { + return err + } - err = s.removeFileFromIRODS(userSet, path, baton, transformer) - if err != nil { - return err - } + baton, transformer, err := s.getBatonAndTransformer(userSet) + if err != nil { + return err + } - err = s.db.RemoveFileEntries(userSet.ID(), path) - if err != nil { - return err - } + err = s.removeFileFromIRODS(userSet, path, baton, transformer) + if err != nil { + return err + } - err = s.db.UpdateBasedOnRemovedEntry(userSet.ID(), entry) - if err != nil { - return err - } + err = s.db.RemoveFileEntries(userSet.ID(), path) + if err != nil { + return err } - return nil + return s.db.UpdateBasedOnRemovedEntry(userSet.ID(), entry) } func (s *Server) removeDirFromIRODSandDB(userSet *set.Set, paths []string) error { From 25131d3fa6465766f264c959aea0a6c8c1d7e87b Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 4 Feb 2025 13:38:36 +0000 Subject: [PATCH 16/35] Finish implentation for removeQueue WIP --- main_test.go | 45 ++++++++++++++++++++++- server/server.go | 37 ++++++++++++------- server/setdb.go | 95 +++++++++++++++++++++++++++--------------------- 3 files changed, 121 insertions(+), 56 deletions(-) diff --git a/main_test.go b/main_test.go index d07dffb..eeadc1a 100644 --- a/main_test.go +++ b/main_test.go @@ -1995,7 +1995,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - FocusConvey("Remove removes the file from the set", func() { + Convey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) @@ -2020,6 +2020,8 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) + time.Sleep(10 * time.Second) + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, 0, file2) @@ -2034,6 +2036,8 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, dir2) @@ -2044,6 +2048,23 @@ func TestRemove(t *testing.T) { 0, dir1+" => ") }) + FocusConvey("Remove removes an empty dir from the set", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir2) + + So(exitCode, ShouldEqual, 0) + + time.Sleep(2 * time.Second) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, dir1) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir2+"/") + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir2+" => ") + }) + Convey("Remove takes a flag --items and removes all provided files and dirs from the set", func() { tempTestFileOfPathsToRemove, err := os.CreateTemp(dir, "testFileSet") So(err, ShouldBeNil) @@ -2056,6 +2077,8 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, file2) @@ -2080,6 +2103,8 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + output, err = exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) So(string(output), ShouldNotContainSubstring, "file1") @@ -2094,6 +2119,8 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2120,6 +2147,8 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2137,6 +2166,8 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + output, err = exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) So(string(output), ShouldContainSubstring, "dir_not_removed\n") @@ -2193,6 +2224,8 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + output = getRemoteMeta(filepath.Join(remotePath, "file1")) So(output, ShouldNotContainSubstring, setName) So(output, ShouldContainSubstring, setName2) @@ -2202,6 +2235,8 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + output, err := exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) So(string(output), ShouldContainSubstring, "file1") @@ -2211,6 +2246,8 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") So(requesters, ShouldNotContainSubstring, user.Username) @@ -2225,6 +2262,8 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") So(requesters, ShouldContainSubstring, user.Username) @@ -2249,6 +2288,8 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") So(requesters, ShouldNotContainSubstring, user.Username) @@ -2263,6 +2304,8 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) + time.Sleep(2 * time.Second) + requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") So(requesters, ShouldContainSubstring, user.Username) diff --git a/server/server.go b/server/server.go index b7ecf40..2fc482b 100644 --- a/server/server.go +++ b/server/server.go @@ -203,25 +203,34 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC return nil } -func (s *Server) removeCallback(_ string, allitemdata []interface{}) { - fmt.Println("removeQueue callback called") +func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { + baton, err := put.GetBatonHandlerWithMetaClient() + if err != nil { + fmt.Println("error: ", err.Error()) + // + } - item := allitemdata[0] - removeReq, _ := item.(removeReq) + for _, item := range allitemdata { + removeReq, _ := item.(removeReq) + var err error - fmt.Println(removeReq) + if removeReq.isDir { + err = s.removeDirFromIRODSandDB(removeReq.set, removeReq.path, baton) + } else { + err = s.removeFileFromIRODSandDB(removeReq.set, removeReq.path, baton) + } - if removeReq.isDir { - // handle dir - return - } + if err != nil { + fmt.Println("error: ", err.Error()) + // + } - err := s.removeFileFromIRODSandDB(removeReq.set, removeReq.path) - if err != nil { - // + err = s.removeQueue.Remove(context.Background(), removeReq.key()) + if err != nil { + fmt.Println("error: ", err.Error()) + // + } } - - // handle file } // rac is our queue's ready added callback which will get all ready put Requests diff --git a/server/setdb.go b/server/setdb.go index 5f1552b..e2bc711 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -422,11 +422,11 @@ type removeReq struct { isDir bool } +func (rq removeReq) key() string { + return strings.Join([]string{rq.set.ID(), rq.path}, ":") +} + func (s *Server) removeFiles(set *set.Set, paths []string) error { - // err := s.removeFileFromIRODSandDB(set, paths) - // if err != nil { - // return err - // } if len(paths) == 0 { return nil } @@ -434,12 +434,13 @@ func (s *Server) removeFiles(set *set.Set, paths []string) error { defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { + rq := removeReq{path: path, set: set, isDir: false} + defs[i] = &queue.ItemDef{ - Key: strings.Join([]string{set.ID(), path}, ":"), - Data: removeReq{path: path, set: set, isDir: false}, + Key: rq.key(), + Data: rq, TTR: ttr, } - //s.removeQueue.Add(context.Background(), "key", "", "data", 0, 0, 1*time.Hour, queue.SubQueueReady, []string{}) } _, dups, err := s.removeQueue.AddMany(context.Background(), defs) @@ -450,19 +451,6 @@ func (s *Server) removeFiles(set *set.Set, paths []string) error { return err } -// getBatonAndTransformer returns a baton with a meta client and a transformer -// for the given set. -func (s *Server) getBatonAndTransformer(set *set.Set) (*put.Baton, put.PathTransformer, error) { - baton, err := put.GetBatonHandlerWithMetaClient() - if err != nil { - return nil, nil, err - } - - tranformer, err := set.MakeTransformer() - - return baton, tranformer, err -} - func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, baton *put.Baton, transformer put.PathTransformer) error { for _, path := range paths { @@ -605,28 +593,60 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { var filepaths []string var err error - for _, path := range paths { + if len(paths) == 0 { + return nil + } + + dirDefs := make([]*queue.ItemDef, len(paths)) + + for i, path := range paths { + rq := removeReq{path: path, set: set, isDir: true} + + dirDefs[i] = &queue.ItemDef{ + Key: rq.key(), + Data: rq, + TTR: ttr, + } + filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) if err != nil { return err } } - //err = s.removeFileFromIRODSandDB(set, filepaths) + if len(filepaths) == 0 { + return nil + } + + fileDefs := make([]*queue.ItemDef, len(filepaths)) + + for i, path := range filepaths { + rq := removeReq{path: path, set: set, isDir: false} + + fileDefs[i] = &queue.ItemDef{ + Key: rq.key(), + Data: rq, + TTR: ttr, + } + } + + _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) if err != nil { return err } - return s.removeDirFromIRODSandDB(set, paths) + _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) + + return err } -func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, path string) error { +func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, path string, baton *put.Baton) error { entry, err := s.db.GetFileEntryForSet(userSet.ID(), path) if err != nil { return err } - baton, transformer, err := s.getBatonAndTransformer(userSet) + transformer, err := userSet.MakeTransformer() if err != nil { return err } @@ -644,25 +664,18 @@ func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, path string) error { return s.db.UpdateBasedOnRemovedEntry(userSet.ID(), entry) } -func (s *Server) removeDirFromIRODSandDB(userSet *set.Set, paths []string) error { - for _, path := range paths { - baton, transformer, err := s.getBatonAndTransformer(userSet) - if err != nil { - return err - } - - err = s.removeDirFromIRODS(userSet, path, baton, transformer) - if err != nil { - return err - } +func (s *Server) removeDirFromIRODSandDB(userSet *set.Set, path string, baton *put.Baton) error { + transformer, err := userSet.MakeTransformer() + if err != nil { + return err + } - err = s.db.RemoveDirEntries(userSet.ID(), path) - if err != nil { - return err - } + err = s.removeDirFromIRODS(userSet, path, baton, transformer) + if err != nil { + return err } - return nil + return s.db.RemoveDirEntries(userSet.ID(), path) } // func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { From 973a34dabbf8ca2d33ad50795f0ebc5114a646c7 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 4 Feb 2025 15:19:02 +0000 Subject: [PATCH 17/35] Finish removeQueue implementation --- main_test.go | 23 +++++++++++++++--- server/server.go | 21 ++++++++++------- server/setdb.go | 61 +++++++++++++++++++++++++++++------------------- set/db.go | 17 ++++++++++++-- 4 files changed, 85 insertions(+), 37 deletions(-) diff --git a/main_test.go b/main_test.go index eeadc1a..e3774c3 100644 --- a/main_test.go +++ b/main_test.go @@ -1824,6 +1824,11 @@ func getRemoteMeta(path string) string { return string(output) } +func removeFileFromIRODS(path string) { + _, err := exec.Command("irm", "-f", path).CombinedOutput() + So(err, ShouldBeNil) +} + func TestManualMode(t *testing.T) { resetIRODS() @@ -1924,7 +1929,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - FocusConvey("Given a server", t, func() { + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1953,7 +1958,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - FocusConvey("And an added set with files and folders", func() { + Convey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -2048,7 +2053,7 @@ func TestRemove(t *testing.T) { 0, dir1+" => ") }) - FocusConvey("Remove removes an empty dir from the set", func() { + Convey("Remove removes an empty dir from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir2) So(exitCode, ShouldEqual, 0) @@ -2312,6 +2317,18 @@ func TestRemove(t *testing.T) { }) }) }) + + Convey("Failing to remove from iRODS displays the error in status", func() { + removeFileFromIRODS(filepath.Join(remotePath, "file1")) + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) + + So(exitCode, ShouldEqual, 0) + + time.Sleep(2 * time.Second) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, fmt.Sprintf("list operation failed: Path '%s'", filepath.Join(remotePath, "file1"))) + }) }) //TODO add tests for failed files }) diff --git a/server/server.go b/server/server.go index 2fc482b..c10bcee 100644 --- a/server/server.go +++ b/server/server.go @@ -206,29 +206,34 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { baton, err := put.GetBatonHandlerWithMetaClient() if err != nil { - fmt.Println("error: ", err.Error()) - // + s.Logger.Printf("%s", err.Error()) + + return } for _, item := range allitemdata { removeReq, _ := item.(removeReq) + var err error if removeReq.isDir { - err = s.removeDirFromIRODSandDB(removeReq.set, removeReq.path, baton) + err = s.removeDirFromIRODSandDB(removeReq, baton) } else { - err = s.removeFileFromIRODSandDB(removeReq.set, removeReq.path, baton) + err = s.removeFileFromIRODSandDB(removeReq, baton) } if err != nil { - fmt.Println("error: ", err.Error()) - // + s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + + // TODO if not max attempts + // continue } err = s.removeQueue.Remove(context.Background(), removeReq.key()) if err != nil { - fmt.Println("error: ", err.Error()) - // + s.Logger.Printf("%s", err.Error()) + + continue } } } diff --git a/server/setdb.go b/server/setdb.go index e2bc711..6a1aeea 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -417,9 +417,10 @@ func (s *Server) removePaths(c *gin.Context) { } type removeReq struct { - path string - set *set.Set - isDir bool + path string + set *set.Set + isDir bool + isDirEmpty bool } func (rq removeReq) key() string { @@ -602,20 +603,22 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { for i, path := range paths { rq := removeReq{path: path, set: set, isDir: true} + dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) + if err != nil { + return err + } + + if len(dirFilepaths) == 0 { + rq.isDirEmpty = true + } + + filepaths = append(filepaths, dirFilepaths...) + dirDefs[i] = &queue.ItemDef{ Key: rq.key(), Data: rq, TTR: ttr, } - - filepaths, err = s.db.GetFilesInDir(set.ID(), path, filepaths) - if err != nil { - return err - } - } - - if len(filepaths) == 0 { - return nil } fileDefs := make([]*queue.ItemDef, len(filepaths)) @@ -640,42 +643,52 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { return err } -func (s *Server) removeFileFromIRODSandDB(userSet *set.Set, path string, baton *put.Baton) error { - entry, err := s.db.GetFileEntryForSet(userSet.ID(), path) +func (s *Server) removeFileFromIRODSandDB(removeReq removeReq, baton *put.Baton) error { + entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) if err != nil { return err } - transformer, err := userSet.MakeTransformer() + transformer, err := removeReq.set.MakeTransformer() if err != nil { return err } - err = s.removeFileFromIRODS(userSet, path, baton, transformer) + err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) if err != nil { + entry.LastError = err.Error() + + err := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) + if err != nil { + fmt.Println("!! error: ", err.Error()) + } + return err } - err = s.db.RemoveFileEntries(userSet.ID(), path) + err = s.db.RemoveFileEntries(removeReq.set.ID(), removeReq.path) if err != nil { + // TODO - do something to say removed from irods but no db return err } - return s.db.UpdateBasedOnRemovedEntry(userSet.ID(), entry) + return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) } -func (s *Server) removeDirFromIRODSandDB(userSet *set.Set, path string, baton *put.Baton) error { - transformer, err := userSet.MakeTransformer() +func (s *Server) removeDirFromIRODSandDB(removeReq removeReq, baton *put.Baton) error { + transformer, err := removeReq.set.MakeTransformer() if err != nil { return err } - err = s.removeDirFromIRODS(userSet, path, baton, transformer) - if err != nil { - return err + if !removeReq.isDirEmpty { + err = s.removeDirFromIRODS(removeReq.set, removeReq.path, baton, transformer) + if err != nil { + return err + } } - return s.db.RemoveDirEntries(userSet.ID(), path) + return s.db.RemoveDirEntries(removeReq.set.ID(), removeReq.path) } // func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { diff --git a/set/db.go b/set/db.go index 617c6a7..58c45a5 100644 --- a/set/db.go +++ b/set/db.go @@ -243,6 +243,17 @@ func updateDatabaseSetWithUserSetDetails(dbSet, userSet *Set) error { return nil } +func (d *DB) UploadEntry(sid, key string, entry *Entry) error { + return d.db.Update(func(tx *bolt.Tx) error { + _, b, err := d.getEntry(tx, sid, key) + if err != nil { + return err + } + + return b.Put([]byte(key), d.encodeToBytes(entry)) + }) +} + // encodeToBytes encodes the given thing as a byte slice, suitable for storing // in a database. func (d *DB) encodeToBytes(thing interface{}) []byte { @@ -317,7 +328,9 @@ func (d *DB) RemoveDirEntries(setID string, path string) error { return d.removeEntries(setID, path, dirBucket) } -func (d *DB) GetFilesInDir(setID string, dirpath string, filepaths []string) ([]string, error) { +func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) { + var filepaths []string + entries, err := d.getEntries(setID, discoveredBucket) if err != nil { return nil, err @@ -1003,7 +1016,7 @@ func (d *DBRO) GetFileEntries(setID string) ([]*Entry, error) { func (d *DBRO) GetFileEntryForSet(setID, filePath string) (*Entry, error) { var entry *Entry - if err := d.db.View(func(tx *bolt.Tx) error { + if err := d.db.Update(func(tx *bolt.Tx) error { var err error entry, _, err = d.getEntry(tx, setID, filePath) From 53aa3bb1cf59b71f66e023b5ff6cc5c7e5403f9b Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 4 Feb 2025 16:29:01 +0000 Subject: [PATCH 18/35] Update status to have a `Removal status` line when removing --- cmd/status.go | 4 ++++ main_test.go | 45 ++++++++++++++++++++++++--------------------- server/setdb.go | 33 +++++++++++++++++++++++++-------- set/db.go | 17 +++++++++++++++-- set/set.go | 4 ++++ 5 files changed, 72 insertions(+), 31 deletions(-) diff --git a/cmd/status.go b/cmd/status.go index 7be5742..ba1e391 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -405,6 +405,10 @@ func displaySet(s *set.Set, showRequesters bool) { //nolint:funlen,gocyclo cliPrint("Status: %s\n", s.Status) } + if s.NumObjectsToBeRemoved > 0 { + cliPrint("Removal status: %d / %d objects removed\n", s.NumObjectsRemoved, s.NumObjectsToBeRemoved) + } + if s.Warning != "" { cliPrint("Warning: %s\n", s.Warning) } diff --git a/main_test.go b/main_test.go index e3774c3..fd5cf3c 100644 --- a/main_test.go +++ b/main_test.go @@ -1929,7 +1929,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - Convey("Given a server", t, func() { + FocusConvey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1958,7 +1958,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - Convey("And an added set with files and folders", func() { + FocusConvey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -2000,15 +2000,15 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - Convey("Remove removes the file from the set", func() { + FocusConvey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, "Removal status: 0 / 1 objects removed") - // s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, - // 0, "Removal status: 0 / 100 files removed") + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, file2) @@ -2020,12 +2020,15 @@ func TestRemove(t *testing.T) { 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") - Convey("Remove again will remove another file", func() { + FocusConvey("Remove again will remove another file", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file2) So(exitCode, ShouldEqual, 0) - time.Sleep(10 * time.Second) + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, "Removal status: 0 / 1 objects removed") + + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, 0, file2) @@ -2041,7 +2044,7 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 2 / 2 objects removed", 5*time.Second) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, dir2) @@ -2058,7 +2061,7 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, dir1) @@ -2082,7 +2085,7 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 3 / 3 objects removed", 5*time.Second) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, file2) @@ -2108,7 +2111,7 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) output, err = exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) @@ -2124,7 +2127,7 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2152,7 +2155,7 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2171,7 +2174,7 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 2 / 2 objects removed", 5*time.Second) output, err = exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) @@ -2229,7 +2232,7 @@ func TestRemove(t *testing.T) { So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) output = getRemoteMeta(filepath.Join(remotePath, "file1")) So(output, ShouldNotContainSubstring, setName) @@ -2240,7 +2243,7 @@ func TestRemove(t *testing.T) { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) output, err := exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) @@ -2251,7 +2254,7 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2267,7 +2270,7 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2293,7 +2296,7 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2309,7 +2312,7 @@ func TestRemove(t *testing.T) { exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) - time.Sleep(2 * time.Second) + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") diff --git a/server/setdb.go b/server/setdb.go index 6a1aeea..4ad4bc0 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -411,9 +411,21 @@ func (s *Server) removePaths(c *gin.Context) { return } - s.removeFiles(set, filePaths) + err = s.removeFiles(set, filePaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } + + dirFilePaths, err := s.removeDirs(set, dirPaths) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck + + return + } - s.removeDirs(set, dirPaths) + s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) } type removeReq struct { @@ -590,12 +602,12 @@ func (s *Server) removeDirFromIRODS(set *set.Set, path string, return nil } -func (s *Server) removeDirs(set *set.Set, paths []string) error { +func (s *Server) removeDirs(set *set.Set, paths []string) ([]string, error) { var filepaths []string var err error if len(paths) == 0 { - return nil + return filepaths, nil } dirDefs := make([]*queue.ItemDef, len(paths)) @@ -605,7 +617,7 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) if err != nil { - return err + return nil, err } if len(dirFilepaths) == 0 { @@ -635,12 +647,12 @@ func (s *Server) removeDirs(set *set.Set, paths []string) error { _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) if err != nil { - return err + return nil, err } _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) - return err + return filepaths, err } func (s *Server) removeFileFromIRODSandDB(removeReq removeReq, baton *put.Baton) error { @@ -688,7 +700,12 @@ func (s *Server) removeDirFromIRODSandDB(removeReq removeReq, baton *put.Baton) } } - return s.db.RemoveDirEntries(removeReq.set.ID(), removeReq.path) + err = s.db.RemoveDirEntries(removeReq.set.ID(), removeReq.path) + if err != nil { + return err + } + + return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) } // func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { diff --git a/set/db.go b/set/db.go index 58c45a5..77e3dce 100644 --- a/set/db.go +++ b/set/db.go @@ -1164,13 +1164,26 @@ func (d *DB) UpdateBasedOnRemovedEntry(setID string, entry *Entry) error { return d.updateSetProperties(setID, func(got *Set) { got.SizeRemoved += entry.Size got.SizeTotal -= entry.Size - - got.NumFiles -= 1 + got.NumFiles-- + got.NumObjectsRemoved++ got.removedEntryToSetCounts(entry) }) } +func (d *DB) UpdateSetTotalToRemove(setID string, num uint64) error { + return d.updateSetProperties(setID, func(got *Set) { + got.NumObjectsToBeRemoved = num + got.NumObjectsRemoved = 0 + }) +} + +func (d *DB) IncrementSetTotalRemoved(setID string) error { + return d.updateSetProperties(setID, func(got *Set) { + got.NumObjectsRemoved++ + }) +} + func (d *DB) ResetRemoveSize(setID string) error { return d.updateSetProperties(setID, func(got *Set) { got.SizeRemoved = 0 diff --git a/set/set.go b/set/set.go index 175cddc..2176acd 100644 --- a/set/set.go +++ b/set/set.go @@ -201,6 +201,10 @@ type Set struct { // remove. This is a read-only value. SizeRemoved uint64 + NumObjectsToBeRemoved uint64 + + NumObjectsRemoved uint64 + // Error holds any error that applies to the whole set, such as an issue // with the Transformer. This is a read-only value. Error string From cd1bae355699db2e62ef429e59c46fdca8fac1a7 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 5 Feb 2025 14:53:22 +0000 Subject: [PATCH 19/35] Handle removal retries --- main_test.go | 27 ++++++++++++++++++++++----- server/server.go | 26 ++++++++++++++++++-------- server/setdb.go | 12 +++++------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/main_test.go b/main_test.go index fd5cf3c..95ba6f4 100644 --- a/main_test.go +++ b/main_test.go @@ -1829,6 +1829,11 @@ func removeFileFromIRODS(path string) { So(err, ShouldBeNil) } +func addFileToIRODS(localPath, remotePath string) { + _, err := exec.Command("iput", localPath, remotePath).CombinedOutput() + So(err, ShouldBeNil) +} + func TestManualMode(t *testing.T) { resetIRODS() @@ -1929,7 +1934,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - FocusConvey("Given a server", t, func() { + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1958,7 +1963,7 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath - FocusConvey("And an added set with files and folders", func() { + Convey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -2000,7 +2005,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName, transformer, tempTestFileOfPaths.Name()) - FocusConvey("Remove removes the file from the set", func() { + Convey("Remove removes the file from the set", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) @@ -2020,7 +2025,7 @@ func TestRemove(t *testing.T) { 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") - FocusConvey("Remove again will remove another file", func() { + Convey("Remove again will remove another file", func() { exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file2) So(exitCode, ShouldEqual, 0) @@ -2321,8 +2326,9 @@ func TestRemove(t *testing.T) { }) }) - Convey("Failing to remove from iRODS displays the error in status", func() { + Convey("Remove on a failing file displays the error on the file", func() { removeFileFromIRODS(filepath.Join(remotePath, "file1")) + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) So(exitCode, ShouldEqual, 0) @@ -2331,6 +2337,17 @@ func TestRemove(t *testing.T) { s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, fmt.Sprintf("list operation failed: Path '%s'", filepath.Join(remotePath, "file1"))) + + Convey("And displays the error in set status if not fixed", func() { + s.waitForStatus(setName, fmt.Sprintf("Error: Error when removing: list operation failed: Path '%s'", + filepath.Join(remotePath, "file1")), 30*time.Second) + }) + + Convey("And succeeds if issue is fixed during retries", func() { + addFileToIRODS(file1, filepath.Join(remotePath, "file1")) + + s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 30*time.Second) + }) }) }) //TODO add tests for failed files diff --git a/server/server.go b/server/server.go index c10bcee..3d1a668 100644 --- a/server/server.go +++ b/server/server.go @@ -66,6 +66,8 @@ const ( jobLimitGroup = "irods" maxJobsToSubmit = 100 racRetriggerDelay = 1 * time.Minute + + retryDelay = 5 * time.Second ) // Config configures the server. @@ -222,18 +224,26 @@ func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { err = s.removeFileFromIRODSandDB(removeReq, baton) } - if err != nil { - s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + errr := s.removeQueue.Remove(context.Background(), removeReq.key()) + if errr != nil { + s.Logger.Printf("%s", err.Error()) - // TODO if not max attempts - // continue + continue } - err = s.removeQueue.Remove(context.Background(), removeReq.key()) if err != nil { - s.Logger.Printf("%s", err.Error()) - - continue + removeReq.attempts++ + if removeReq.attempts == jobRetries { + s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + + continue + } + + _, err = s.removeQueue.Add(context.Background(), removeReq.key(), "", + removeReq, 0, retryDelay, ttr, queue.SubQueueDelay, nil) + if err != nil { + s.Logger.Printf("%s", err.Error()) + } } } } diff --git a/server/setdb.go b/server/setdb.go index 4ad4bc0..4f4fa5e 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -433,6 +433,7 @@ type removeReq struct { set *set.Set isDir bool isDirEmpty bool + attempts uint8 } func (rq removeReq) key() string { @@ -456,10 +457,7 @@ func (s *Server) removeFiles(set *set.Set, paths []string) error { } } - _, dups, err := s.removeQueue.AddMany(context.Background(), defs) - if dups != 0 { - return fmt.Errorf("dups??") - } + _, _, err := s.removeQueue.AddMany(context.Background(), defs) return err } @@ -670,9 +668,9 @@ func (s *Server) removeFileFromIRODSandDB(removeReq removeReq, baton *put.Baton) if err != nil { entry.LastError = err.Error() - err := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) - if err != nil { - fmt.Println("!! error: ", err.Error()) + erru := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) + if erru != nil { + s.Logger.Printf("%s", erru.Error()) } return err From 18c0cb86f37b12ffc578a8fcb6455cebfe384f14 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 5 Feb 2025 15:49:39 +0000 Subject: [PATCH 20/35] Refactor removeQueue implementation --- main_test.go | 1 - server/server.go | 43 +++++++++++++++++++++++-------------------- server/setdb.go | 34 ++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/main_test.go b/main_test.go index 95ba6f4..7c7fe8d 100644 --- a/main_test.go +++ b/main_test.go @@ -2350,7 +2350,6 @@ func TestRemove(t *testing.T) { }) }) }) - //TODO add tests for failed files }) } diff --git a/server/server.go b/server/server.go index 3d1a668..436c3fb 100644 --- a/server/server.go +++ b/server/server.go @@ -143,7 +143,7 @@ func New(conf Config) (*Server, error) { s.clientQueue.SetTTRCallback(s.clientTTRC) s.Server.Router().Use(gas.IncludeAbortErrorsInBody) - s.removeQueue.SetReadyAddedCallback(s.removeCallback) + // s.removeQueue.SetReadyAddedCallback(s.removeCallback) s.monitor = NewMonitor(func(given *set.Set) { if err := s.discoverSet(given); err != nil { @@ -205,7 +205,7 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC return nil } -func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { +func (s *Server) handleRemoveRequests(reserveGroup string) { baton, err := put.GetBatonHandlerWithMetaClient() if err != nil { s.Logger.Printf("%s", err.Error()) @@ -213,10 +213,13 @@ func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { return } - for _, item := range allitemdata { - removeReq, _ := item.(removeReq) + for { + item, err := s.removeQueue.Reserve(reserveGroup, 6*time.Second) + if item == nil { + break + } - var err error + removeReq := item.Data().(removeReq) if removeReq.isDir { err = s.removeDirFromIRODSandDB(removeReq, baton) @@ -224,26 +227,26 @@ func (s *Server) removeCallback(queueName string, allitemdata []interface{}) { err = s.removeFileFromIRODSandDB(removeReq, baton) } - errr := s.removeQueue.Remove(context.Background(), removeReq.key()) - if errr != nil { - s.Logger.Printf("%s", err.Error()) - - continue - } - if err != nil { - removeReq.attempts++ - if removeReq.attempts == jobRetries { - s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + if uint8(item.Stats().Releases) < jobRetries { + s.removeQueue.SetDelay(removeReq.key(), retryDelay) + + err := s.removeQueue.Release(context.Background(), removeReq.key()) + if err != nil { + s.Logger.Printf("%s", err.Error()) + } continue } - _, err = s.removeQueue.Add(context.Background(), removeReq.key(), "", - removeReq, 0, retryDelay, ttr, queue.SubQueueDelay, nil) - if err != nil { - s.Logger.Printf("%s", err.Error()) - } + s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + } + + errr := s.removeQueue.Remove(context.Background(), removeReq.key()) + if errr != nil { + s.Logger.Printf("%s", err.Error()) + + continue } } } diff --git a/server/setdb.go b/server/setdb.go index 4f4fa5e..b405ddb 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -411,20 +411,24 @@ func (s *Server) removePaths(c *gin.Context) { return } - err = s.removeFiles(set, filePaths) + reserveGroup := set.ID() + time.Now().String() + + err = s.removeFiles(set, filePaths, reserveGroup) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } - dirFilePaths, err := s.removeDirs(set, dirPaths) + dirFilePaths, err := s.removeDirs(set, dirPaths, reserveGroup) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } + go s.handleRemoveRequests(reserveGroup) + s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) } @@ -433,14 +437,13 @@ type removeReq struct { set *set.Set isDir bool isDirEmpty bool - attempts uint8 } func (rq removeReq) key() string { return strings.Join([]string{rq.set.ID(), rq.path}, ":") } -func (s *Server) removeFiles(set *set.Set, paths []string) error { +func (s *Server) removeFiles(set *set.Set, paths []string, reserveGroup string) error { if len(paths) == 0 { return nil } @@ -451,9 +454,10 @@ func (s *Server) removeFiles(set *set.Set, paths []string) error { rq := removeReq{path: path, set: set, isDir: false} defs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, + Key: rq.key(), + Data: rq, + TTR: ttr, + ReserveGroup: reserveGroup, } } @@ -600,7 +604,7 @@ func (s *Server) removeDirFromIRODS(set *set.Set, path string, return nil } -func (s *Server) removeDirs(set *set.Set, paths []string) ([]string, error) { +func (s *Server) removeDirs(set *set.Set, paths []string, reserveGroup string) ([]string, error) { var filepaths []string var err error @@ -625,9 +629,10 @@ func (s *Server) removeDirs(set *set.Set, paths []string) ([]string, error) { filepaths = append(filepaths, dirFilepaths...) dirDefs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, + Key: rq.key(), + Data: rq, + TTR: ttr, + ReserveGroup: reserveGroup, } } @@ -637,9 +642,10 @@ func (s *Server) removeDirs(set *set.Set, paths []string) ([]string, error) { rq := removeReq{path: path, set: set, isDir: false} fileDefs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, + Key: rq.key(), + Data: rq, + TTR: ttr, + ReserveGroup: reserveGroup, } } From 812e3a053fefb8df1ad3bf95fee6ce8a92f5a8fd Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Thu, 6 Feb 2025 17:06:42 +0000 Subject: [PATCH 21/35] Add handling for when an item is removed from iRODS but fails to be removed from DB and delint --- cmd/put.go | 2 +- cmd/remove.go | 5 -- main_test.go | 110 ++++++++++++-------------------- server/server.go | 79 +++++++++++++++++------ server/setdb.go | 162 ++++++++++++++++++++--------------------------- set/db.go | 15 ++++- set/set.go | 7 -- set/set_test.go | 5 +- 8 files changed, 186 insertions(+), 199 deletions(-) diff --git a/cmd/put.go b/cmd/put.go index 3e58700..d12f76a 100644 --- a/cmd/put.go +++ b/cmd/put.go @@ -301,7 +301,7 @@ func createCollection(p *put.Putter) { func handleManualMode() { requests, err := getRequestsFromFile(putFile, putMeta, putBase64) if err != nil { - die(err.Error()) + die("%s", err.Error()) } _, uploadResults, skipResults, dfunc := handlePut(nil, requests) diff --git a/cmd/remove.go b/cmd/remove.go index 2d20706..34d2f29 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -136,9 +136,4 @@ func remove(client *server.Client, user, name string, files, dirs []string) { if err != nil { die("%s", err.Error()) } - - // err = client.RemoveDirs(sets[0].ID(), dirs) - // if err != nil { - // die("%s", err.Error()) - // } } diff --git a/main_test.go b/main_test.go index 7c7fe8d..0f8af6f 100644 --- a/main_test.go +++ b/main_test.go @@ -320,6 +320,15 @@ func (s *TestServer) addSetForTestingWithFlag(t *testing.T, name, transformer, p s.waitForStatus(name, "\nStatus: complete", 5*time.Second) } +func (s *TestServer) removePath(t *testing.T, name, path string, numFiles int) { + t.Helper() + + exitCode, _ := s.runBinary(t, "remove", "--name", name, "--path", path) + So(exitCode, ShouldEqual, 0) + + s.waitForStatus(name, fmt.Sprintf("Removal status: %d / %d objects removed", numFiles, numFiles), 10*time.Second) +} + func (s *TestServer) waitForStatus(name, statusToFind string, timeout time.Duration) { ctx, cancelFn := context.WithTimeout(context.Background(), timeout) defer cancelFn() @@ -2022,11 +2031,12 @@ func TestRemove(t *testing.T) { 0, file1) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, - 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ + 0, "Num files: 5; Symlinks: 1; Hardlinks: 1; Size "+ + "(total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") Convey("Remove again will remove another file", func() { - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file2) + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file2) So(exitCode, ShouldEqual, 0) @@ -2039,17 +2049,14 @@ func TestRemove(t *testing.T) { 0, file2) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, - 0, "Num files: 4; Symlinks: 1; Hardlinks: 1; Size (total/recently uploaded/recently removed): 20 B / 40 B / 10 B\n"+ + 0, "Num files: 4; Symlinks: 1; Hardlinks: 1; Size "+ + "(total/recently uploaded/recently removed): 20 B / 40 B / 10 B\n"+ "Uploaded: 4; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") }) }) Convey("Remove removes the dir from the set", func() { - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) - - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 2 / 2 objects removed", 5*time.Second) + s.removePath(t, setName, dir1, 2) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, dir2) @@ -2062,11 +2069,7 @@ func TestRemove(t *testing.T) { }) Convey("Remove removes an empty dir from the set", func() { - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir2) - - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, dir2, 1) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, dir1) @@ -2079,8 +2082,8 @@ func TestRemove(t *testing.T) { }) Convey("Remove takes a flag --items and removes all provided files and dirs from the set", func() { - tempTestFileOfPathsToRemove, err := os.CreateTemp(dir, "testFileSet") - So(err, ShouldBeNil) + tempTestFileOfPathsToRemove, errt := os.CreateTemp(dir, "testFileSet") + So(errt, ShouldBeNil) _, err = io.WriteString(tempTestFileOfPathsToRemove, fmt.Sprintf("%s\n%s", file1, dir1)) @@ -2109,14 +2112,11 @@ func TestRemove(t *testing.T) { }) Convey("Remove removes the provided file from iRODS", func() { - output, err := exec.Command("ils", remotePath).CombinedOutput() - So(err, ShouldBeNil) + output, erro := exec.Command("ils", remotePath).CombinedOutput() + So(erro, ShouldBeNil) So(string(output), ShouldContainSubstring, "file1") - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) output, err = exec.Command("ils", remotePath).CombinedOutput() So(err, ShouldBeNil) @@ -2126,13 +2126,10 @@ func TestRemove(t *testing.T) { Convey("Remove with a hardlink removes both the hardlink file and inode file", func() { remoteInode := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "link")), "ibackup:remotehardlink") - _, err := exec.Command("ils", remoteInode).CombinedOutput() + _, err = exec.Command("ils", remoteInode).CombinedOutput() So(err, ShouldBeNil) - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, linkPath, 1) _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2152,15 +2149,12 @@ func TestRemove(t *testing.T) { s.waitForStatus("testHardlinks", "\nStatus: complete", 10*time.Second) Convey("Removing a hardlink does not remove the inode file", func() { - remoteInode := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "link2")), "ibackup:remotehardlink") + remoteInode := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "link")), "ibackup:remotehardlink") - _, err := exec.Command("ils", remoteInode).CombinedOutput() + _, err = exec.Command("ils", remoteInode).CombinedOutput() So(err, ShouldBeNil) - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", linkPath) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, linkPath, 1) _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() So(err, ShouldNotBeNil) @@ -2171,15 +2165,12 @@ func TestRemove(t *testing.T) { }) Convey("Remove removes the provided dir from iRODS", func() { - output, err := exec.Command("ils", "-r", remotePath).CombinedOutput() - So(err, ShouldBeNil) + output, errc := exec.Command("ils", "-r", remotePath).CombinedOutput() + So(errc, ShouldBeNil) So(string(output), ShouldContainSubstring, "path/to/some/dir") So(string(output), ShouldContainSubstring, "file3\n") - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", dir1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 2 / 2 objects removed", 5*time.Second) + s.removePath(t, setName, dir1, 2) output, err = exec.Command("ils", "-r", remotePath).CombinedOutput() So(err, ShouldBeNil) @@ -2216,8 +2207,8 @@ func TestRemove(t *testing.T) { }) Convey("And a set with the same files added by a different user", func() { - user, err := user.Current() - So(err, ShouldBeNil) + user, erru := user.Current() + So(erru, ShouldBeNil) setName2 := "different_user_set" @@ -2233,11 +2224,7 @@ func TestRemove(t *testing.T) { So(output, ShouldContainSubstring, setName) So(output, ShouldContainSubstring, setName2) - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) - - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) output = getRemoteMeta(filepath.Join(remotePath, "file1")) So(output, ShouldNotContainSubstring, setName) @@ -2245,21 +2232,15 @@ func TestRemove(t *testing.T) { }) Convey("Remove does not remove the provided file from iRODS", func() { - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) + s.removePath(t, setName, file1, 1) - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) - - output, err := exec.Command("ils", remotePath).CombinedOutput() - So(err, ShouldBeNil) + output, errc := exec.Command("ils", remotePath).CombinedOutput() + So(errc, ShouldBeNil) So(string(output), ShouldContainSubstring, "file1") }) Convey("Remove on a file removes the user as a requester", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2272,10 +2253,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName3, transformer, tempTestFileOfPaths.Name()) Convey("Remove keeps the user as a requester", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2285,8 +2263,8 @@ func TestRemove(t *testing.T) { }) Convey("And a set with the same files and name added by a different user", func() { - user, err := user.Current() - So(err, ShouldBeNil) + user, erru := user.Current() + So(erru, ShouldBeNil) setName2 := setName @@ -2298,10 +2276,7 @@ func TestRemove(t *testing.T) { s.waitForStatusWithUser(setName2, "\nStatus: complete", "testUser", 20*time.Second) Convey("Remove on a file removes the user as a requester but not the file", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") @@ -2314,10 +2289,7 @@ func TestRemove(t *testing.T) { s.addSetForTestingWithItems(t, setName3, transformer, tempTestFileOfPaths.Name()) Convey("Remove keeps the user as a requester", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file1) - So(exitCode, ShouldEqual, 0) - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.removePath(t, setName, file1, 1) requesters := getMetaValue(getRemoteMeta(filepath.Join(remotePath, "file1")), "ibackup:requesters") diff --git a/server/server.go b/server/server.go index 436c3fb..6126a10 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,7 @@ package server import ( "context" + "errors" "fmt" "io" "strings" @@ -214,41 +215,79 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { } for { - item, err := s.removeQueue.Reserve(reserveGroup, 6*time.Second) - if item == nil { + item, removeReq, err := s.reserveRemoveRequest(reserveGroup) + if err != nil { break } - removeReq := item.Data().(removeReq) + err = s.removeRequestFromIRODSandDB(&removeReq, baton) + if err != nil { + beenReleased := s.setErrorOrReleaseItem(item, removeReq, err) + if beenReleased { + continue + } + } - if removeReq.isDir { - err = s.removeDirFromIRODSandDB(removeReq, baton) - } else { - err = s.removeFileFromIRODSandDB(removeReq, baton) + errr := s.removeQueue.Remove(context.Background(), removeReq.key()) + if errr != nil { + s.Logger.Printf("%s", err.Error()) } + } +} - if err != nil { - if uint8(item.Stats().Releases) < jobRetries { - s.removeQueue.SetDelay(removeReq.key(), retryDelay) +func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { + if removeReq.isDir { + return s.removeDirFromIRODSandDB(removeReq, baton) + } - err := s.removeQueue.Release(context.Background(), removeReq.key()) - if err != nil { - s.Logger.Printf("%s", err.Error()) - } + return s.removeFileFromIRODSandDB(removeReq, baton) +} - continue - } +func (s *Server) setErrorOrReleaseItem(item *queue.Item, removeReq removeReq, err error) bool { + if uint8(item.Stats().Releases) < jobRetries { + item.SetData(removeReq) - s.db.SetError(removeReq.set.ID(), fmt.Sprintf("Error when removing: %s", err.Error())) + err = s.removeQueue.SetDelay(removeReq.key(), retryDelay) + if err != nil { + s.Logger.Printf("%s", err.Error()) } - errr := s.removeQueue.Remove(context.Background(), removeReq.key()) - if errr != nil { + err = s.removeQueue.Release(context.Background(), removeReq.key()) + if err != nil { s.Logger.Printf("%s", err.Error()) + } - continue + return true + } + + errs := s.db.SetError(removeReq.set.ID(), "Error when removing: "+err.Error()) + if errs != nil { + s.Logger.Printf("Could not put error on set due to: %s\nError was: %s\n", errs.Error(), err.Error()) + } + + return false +} + +// reserveRemoveRequest reserves an item from the given reserve group from the +// remove queue and converts it to a removeRequest. Returns nil and no error if +// the queue is empty. +func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeReq, error) { + item, err := s.removeQueue.Reserve(reserveGroup, retryDelay+2*time.Second) + if err != nil { + qerr, ok := err.(queue.Error) //nolint:errorlint + if ok && errors.Is(qerr.Err, queue.ErrNothingReady) { + return nil, removeReq{}, err } + + s.Logger.Printf("%s", err.Error()) } + + removeReq, ok := item.Data().(removeReq) + if !ok { + s.Logger.Printf("Invalid data type in remove queue") + } + + return item, removeReq, nil } // rac is our queue's ready added callback which will get all ready put Requests diff --git a/server/setdb.go b/server/setdb.go index b405ddb..221001d 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -281,7 +281,6 @@ func (s *Server) addDBEndpoints(authGroup *gin.RouterGroup) { authGroup.GET(fileRetryPath+idParam, s.retryFailedEntries) authGroup.PUT(removePathsPath+idParam, s.removePaths) - //authGroup.PUT(removeDirsPath+idParam, s.removeDirs) } // putSet interprets the body as a JSON encoding of a set.Set and stores it in @@ -390,64 +389,71 @@ func (s *Server) removePaths(c *gin.Context) { set := s.db.GetByID(sid) - err := s.db.ValidateFilePaths(set, filePaths) + err := s.db.ValidateFileAndDirPaths(set, filePaths, dirPaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } - err = s.db.ValidateDirPaths(set, dirPaths) + err = s.db.ResetRemoveSize(sid) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } - err = s.db.ResetRemoveSize(sid) + err = s.removeFilesAndDirs(set, filePaths, dirPaths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck return } +} +func (s *Server) removeFilesAndDirs(set *set.Set, filePaths, dirPaths []string) error { reserveGroup := set.ID() + time.Now().String() - err = s.removeFiles(set, filePaths, reserveGroup) + err := s.submitFilesForRemoval(set, filePaths, reserveGroup) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return + return err } - dirFilePaths, err := s.removeDirs(set, dirPaths, reserveGroup) + dirFilePaths, err := s.submitDirsForRemoval(set, dirPaths, reserveGroup) if err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return + return err } go s.handleRemoveRequests(reserveGroup) - s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) + return s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) } type removeReq struct { - path string - set *set.Set - isDir bool - isDirEmpty bool + path string + set *set.Set + isDir bool + isDirEmpty bool + isRemovedFromIRODS bool } func (rq removeReq) key() string { return strings.Join([]string{rq.set.ID(), rq.path}, ":") } -func (s *Server) removeFiles(set *set.Set, paths []string, reserveGroup string) error { +func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGroup string) error { if len(paths) == 0 { return nil } + defs := makeItemsDefsFromFilePaths(set, reserveGroup, paths) + + _, _, err := s.removeQueue.AddMany(context.Background(), defs) + + return err +} + +func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []string) []*queue.ItemDef { defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { @@ -461,36 +467,7 @@ func (s *Server) removeFiles(set *set.Set, paths []string, reserveGroup string) } } - _, _, err := s.removeQueue.AddMany(context.Background(), defs) - - return err -} - -func (s *Server) removeFilesFromIRODS(set *set.Set, paths []string, - baton *put.Baton, transformer put.PathTransformer) error { - for _, path := range paths { - rpath, err := transformer(path) - if err != nil { - return err - } - - remoteMeta, err := baton.GetMeta(rpath) - if err != nil { - return err - } - - sets, requesters, err := s.handleSetsAndRequesters(set, remoteMeta) - if err != nil { - return err - } - - err = baton.RemovePathFromSetInIRODS(transformer, rpath, sets, requesters, remoteMeta) - if err != nil { - return err - } - } - - return nil + return defs } func (s *Server) removeFileFromIRODS(set *set.Set, path string, @@ -588,9 +565,8 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { return slice[:len(slice)-1], nil } -func (s *Server) removeDirFromIRODS(set *set.Set, path string, - baton *put.Baton, transformer put.PathTransformer) error { - +func (s *Server) removeDirFromIRODS(path string, baton *put.Baton, + transformer put.PathTransformer) error { rpath, err := transformer(path) if err != nil { return err @@ -604,22 +580,39 @@ func (s *Server) removeDirFromIRODS(set *set.Set, path string, return nil } -func (s *Server) removeDirs(set *set.Set, paths []string, reserveGroup string) ([]string, error) { - var filepaths []string - var err error - +func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup string) ([]string, error) { if len(paths) == 0 { - return filepaths, nil + return []string{}, nil } - dirDefs := make([]*queue.ItemDef, len(paths)) + filepaths, dirDefs, err := s.makeItemsDefsAndFilePathsFromDirPaths(set, reserveGroup, paths) + if err != nil { + return nil, err + } + + fileDefs := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) + + _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) + if err != nil { + return nil, err + } + + _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) + + return filepaths, err +} + +func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, reserveGroup string, paths []string) ([]string, []*queue.ItemDef, error) { + var filepaths []string + + defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { rq := removeReq{path: path, set: set, isDir: true} dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) if err != nil { - return nil, err + return nil, nil, err } if len(dirFilepaths) == 0 { @@ -628,20 +621,7 @@ func (s *Server) removeDirs(set *set.Set, paths []string, reserveGroup string) ( filepaths = append(filepaths, dirFilepaths...) - dirDefs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, - ReserveGroup: reserveGroup, - } - } - - fileDefs := make([]*queue.ItemDef, len(filepaths)) - - for i, path := range filepaths { - rq := removeReq{path: path, set: set, isDir: false} - - fileDefs[i] = &queue.ItemDef{ + defs[i] = &queue.ItemDef{ Key: rq.key(), Data: rq, TTR: ttr, @@ -649,17 +629,10 @@ func (s *Server) removeDirs(set *set.Set, paths []string, reserveGroup string) ( } } - _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) - if err != nil { - return nil, err - } - - _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) - - return filepaths, err + return filepaths, defs, nil } -func (s *Server) removeFileFromIRODSandDB(removeReq removeReq, baton *put.Baton) error { +func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) if err != nil { return err @@ -670,38 +643,43 @@ func (s *Server) removeFileFromIRODSandDB(removeReq removeReq, baton *put.Baton) return err } - err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) - if err != nil { - entry.LastError = err.Error() + if !removeReq.isRemovedFromIRODS { + err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) + if err != nil { + entry.LastError = err.Error() + + erru := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) + if erru != nil { + s.Logger.Printf("%s", erru.Error()) + } - erru := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) - if erru != nil { - s.Logger.Printf("%s", erru.Error()) + return err } - return err + removeReq.isRemovedFromIRODS = true } err = s.db.RemoveFileEntries(removeReq.set.ID(), removeReq.path) if err != nil { - // TODO - do something to say removed from irods but no db return err } return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) } -func (s *Server) removeDirFromIRODSandDB(removeReq removeReq, baton *put.Baton) error { +func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { transformer, err := removeReq.set.MakeTransformer() if err != nil { return err } - if !removeReq.isDirEmpty { - err = s.removeDirFromIRODS(removeReq.set, removeReq.path, baton, transformer) + if !removeReq.isDirEmpty && !removeReq.isRemovedFromIRODS { + err = s.removeDirFromIRODS(removeReq.path, baton, transformer) if err != nil { return err } + + removeReq.isRemovedFromIRODS = true } err = s.db.RemoveDirEntries(removeReq.set.ID(), removeReq.path) diff --git a/set/db.go b/set/db.go index 77e3dce..e6306b7 100644 --- a/set/db.go +++ b/set/db.go @@ -264,7 +264,16 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } -func (d *DB) ValidateFilePaths(set *Set, paths []string) error { +func (d *DB) ValidateFileAndDirPaths(set *Set, filePaths, dirPaths []string) error { + err := d.validateFilePaths(set, filePaths) + if err != nil { + return err + } + + return d.validateDirPaths(set, dirPaths) +} + +func (d *DB) validateFilePaths(set *Set, paths []string) error { return d.validatePaths(set, fileBucket, discoveredBucket, paths) } @@ -284,14 +293,14 @@ func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) er for _, path := range paths { if _, ok := entriesMap[path]; !ok { - return Error{fmt.Sprintf("%s is not part of the backup set", path), set.Name} + return Error{path + " is not part of the backup set", set.Name} } } return nil } -func (d *DB) ValidateDirPaths(set *Set, paths []string) error { +func (d *DB) validateDirPaths(set *Set, paths []string) error { return d.validatePaths(set, dirBucket, discoveredBucket, paths) } diff --git a/set/set.go b/set/set.go index 2176acd..23f3eaa 100644 --- a/set/set.go +++ b/set/set.go @@ -630,13 +630,6 @@ func (s *Set) updateAllCounts(entries []*Entry, entry *Entry) { } } -func (s *Set) updateSetSize(entries []*Entry) { - for _, e := range entries { - s.SizeTotal += e.Size - // s.SizeUploaded - } -} - // SetError records the given error against the set, indicating it wont work. func (s *Set) SetError(errMsg string) { s.Error = errMsg diff --git a/set/set_test.go b/set/set_test.go index 90fc285..77e1d7c 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -358,7 +358,7 @@ func TestSetDB(t *testing.T) { So(err, ShouldBeNil) Convey("Then remove files and dirs from the sets", func() { - err := db.removeEntries(set.ID(), "/a/b.txt", fileBucket) + err = db.removeEntries(set.ID(), "/a/b.txt", fileBucket) So(err, ShouldBeNil) fEntries, errg := db.GetFileEntries(set.ID()) @@ -367,7 +367,8 @@ func TestSetDB(t *testing.T) { So(fEntries[0], ShouldResemble, &Entry{Path: "/c/d.txt"}) So(fEntries[1], ShouldResemble, &Entry{Path: "/e/f.txt"}) - db.RemoveDirEntries(set.ID(), "/g/h") + err = db.RemoveDirEntries(set.ID(), "/g/h") + So(err, ShouldBeNil) dEntries, errg := db.GetDirEntries(set.ID()) So(errg, ShouldBeNil) From 68182314ca96d0f98f2d81be43c7d80941f884e1 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 7 Feb 2025 16:51:06 +0000 Subject: [PATCH 22/35] Delint and fix tests --- cmd/filestatus.go | 3 ++ cmd/remove.go | 3 +- main_test.go | 76 ++++++++++++++++++++++++++------------------- put/baton.go | 4 +-- server/server.go | 53 +++++++++++++++++++------------- server/setdb.go | 78 ++++++++++++++--------------------------------- set/db.go | 2 +- set/set.go | 2 +- 8 files changed, 108 insertions(+), 113 deletions(-) diff --git a/cmd/filestatus.go b/cmd/filestatus.go index 9e909c7..170fdbc 100644 --- a/cmd/filestatus.go +++ b/cmd/filestatus.go @@ -60,11 +60,13 @@ func init() { func fileSummary(dbPath, filePath string, useIrods bool) error { db, err := set.NewRO(dbPath) if err != nil { + println("fail 1") return err } sets, err := db.GetAll() if err != nil { + println("fail 2") return err } @@ -75,6 +77,7 @@ func fileSummary(dbPath, filePath string, useIrods bool) error { } if err := fsg.printFileStatuses(sets); err != nil { + println("fail 3") return err } diff --git a/cmd/remove.go b/cmd/remove.go index 34d2f29..b5d91bc 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -67,7 +67,7 @@ var removeCmd = &cobra.Command{ --path: if you want to remove a single file or directory, provide its absolute path. `, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { ensureURLandCert() if (removeItems == "") == (removePath == "") { @@ -93,6 +93,7 @@ var removeCmd = &cobra.Command{ die("%s", err.Error()) } + //TODO if dir does not exist then we will add it as a file if pathIsDir(removePath) { dirs = append(dirs, removePath) } else { diff --git a/main_test.go b/main_test.go index 0f8af6f..cafb243 100644 --- a/main_test.go +++ b/main_test.go @@ -275,7 +275,7 @@ func (s *TestServer) confirmOutputContains(t *testing.T, args []string, expected So(actual, ShouldContainSubstring, expected) } -func (s *TestServer) confirmOutputDoesNotContain(t *testing.T, args []string, expectedCode int, +func (s *TestServer) confirmOutputDoesNotContain(t *testing.T, args []string, expectedCode int, //nolint:unparam expected string) { t.Helper() @@ -598,7 +598,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -627,7 +627,7 @@ User metadata: testKey=testVal;testKey2=testVal2 Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -653,7 +653,7 @@ User metadata: `+meta+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -679,7 +679,7 @@ User metadata: `+meta+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -703,7 +703,7 @@ User metadata: `+meta+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -729,7 +729,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -745,7 +745,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -761,7 +761,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -781,7 +781,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -797,7 +797,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -813,7 +813,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -849,7 +849,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 2; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 2; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 2; Abnormal: 0 Completed in: 0s Example File: `+dir+`/path/to/other/file => /remote/path/to/other/file`) @@ -869,7 +869,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 2; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 2; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 2; Abnormal: 0 Completed in: 0s Example File: `+dir+`/path/to/other/file => /remote/path/to/other/file @@ -895,7 +895,7 @@ Removal date: ` + removalDate + ` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -932,7 +932,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -968,7 +968,7 @@ Monitored: false; Archive: false Status: complete Warning: `+badPermDir+`/: permission denied Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -999,7 +999,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: pending upload Discovery: -Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B (and counting) / 0 B +Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B (and counting) / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Example File: `+humgenFile+" => /humgen/teams/hgi/scratch125/mercury/ibackup/file_for_testsuite.do_not_delete") }) @@ -1028,7 +1028,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: pending upload Discovery: -Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B (and counting) / 0 B +Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B (and counting) / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Example File: `+gengenFile+" => /humgen/gengen/teams/hgi/scratch126/mercury/ibackup/file_for_testsuite.do_not_delete") }) @@ -1068,7 +1068,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: pending upload Discovery: -Num files: 4; Symlinks: 2; Hardlinks: 1; Size (total/recently uploaded): 0 B (and counting) / 0 B +Num files: 4; Symlinks: 2; Hardlinks: 1; Size (total/recently uploaded/recently removed): 0 B (and counting) / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Directories: `+dir+toRemote) @@ -1118,7 +1118,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -1150,7 +1150,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 1; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 1 Completed in: 0s Example File: `+dir+`/fifo => /remote/fifo @@ -1178,7 +1178,7 @@ Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -1252,6 +1252,9 @@ func TestBackup(t *testing.T) { return } + reviewDate := time.Now().AddDate(0, 6, 0).Format("2006-01-02") + removalDate := time.Now().AddDate(1, 0, 0).Format("2006-01-02") + dir := t.TempDir() s := new(TestServer) s.prepareFilePaths(dir) @@ -1354,10 +1357,13 @@ Global put client status (/10): 0 iRODS connections; 0 creating collections; 0 c Name: testForBackup Transformer: `+transformer+` +Reason: backup +Review date: `+reviewDate+` +Removal date: `+removalDate+` Monitored: false; Archive: false Status: complete Discovery: -Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 0 B / 0 B +Num files: 0; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 0 B / 0 B / 0 B Uploaded: 0; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: @@ -1641,12 +1647,16 @@ Local Path Status Size Attempts Date Error`+"\n"+ So(output, ShouldContainSubstring, attributePrefix+"testKey1\n") So(output, ShouldContainSubstring, valuePrefix+"testValue1Updated\n") So(output, ShouldContainSubstring, attributePrefix+"testKey2\n") + So(output, ShouldContainSubstring, valuePrefix+"testValue2Updated\n") So(output, ShouldNotContainSubstring, "testValue1\n") } }) Convey("Repeatedly uploading files that are changed or not changes status details", func() { + resetIRODS() + setName = "changingFilesTest" + s.addSetForTesting(t, setName, transformer, path) statusCmd := []string{"status", "--name", setName} @@ -1654,14 +1664,14 @@ Local Path Status Size Attempts Date Error`+"\n"+ s.waitForStatus(setName, "\nStatus: uploading", 60*time.Second) s.confirmOutputContains(t, statusCmd, 0, `Global put queue status: 3 queued; 3 reserved to be worked on; 0 failed - Global put client status (/10): 6 iRODS connections`) +Global put client status (/10): 6 iRODS connections`) s.waitForStatus(setName, "\nStatus: complete", 60*time.Second) s.confirmOutputContains(t, statusCmd, 0, "Uploaded: 3; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") s.confirmOutputContains(t, statusCmd, 0, - "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 30 B / 30 B") + "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 30 B / 30 B / 0 B") s.confirmOutputContains(t, statusCmd, 0, "") @@ -1673,7 +1683,7 @@ Local Path Status Size Attempts Date Error`+"\n"+ s.confirmOutputContains(t, statusCmd, 0, "Uploaded: 0; Replaced: 0; Skipped: 3; Failed: 0; Missing: 0; Abnormal: 0") s.confirmOutputContains(t, statusCmd, 0, - "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 30 B / 0 B") + "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 30 B / 0 B / 0 B") newName = setName + ".v3" statusCmd[2] = newName @@ -1685,7 +1695,7 @@ Local Path Status Size Attempts Date Error`+"\n"+ s.confirmOutputContains(t, statusCmd, 0, "Uploaded: 0; Replaced: 1; Skipped: 2; Failed: 0; Missing: 0; Abnormal: 0") s.confirmOutputContains(t, statusCmd, 0, - "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 38 B / 18 B") + "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 38 B / 18 B / 0 B") internal.CreateTestFile(t, file2, "less data") exitCode, _ := s.runBinary(t, "retry", "--name", newName, "-a") @@ -1695,7 +1705,7 @@ Local Path Status Size Attempts Date Error`+"\n"+ s.confirmOutputContains(t, statusCmd, 0, "Uploaded: 0; Replaced: 1; Skipped: 2; Failed: 0; Missing: 0; Abnormal: 0") s.confirmOutputContains(t, statusCmd, 0, - "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded): 29 B / 9 B") + "Num files: 3; Symlinks: 0; Hardlinks: 0; Size (total/recently uploaded/recently removed): 29 B / 9 B / 0 B") }) }) @@ -1951,6 +1961,8 @@ func TestRemove(t *testing.T) { return } + remotePath = filepath.Join(remotePath, "test_remove") + schedulerDeployment := os.Getenv("IBACKUP_TEST_SCHEDULER") if schedulerDeployment == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_SCHEDULER not set", func() {}) @@ -2003,6 +2015,8 @@ func TestRemove(t *testing.T) { err = os.Link(file1, linkPath) So(err, ShouldBeNil) + remoteLink := filepath.Join(remotePath, "link") + err = os.Symlink(file2, symPath) So(err, ShouldBeNil) @@ -2131,7 +2145,7 @@ func TestRemove(t *testing.T) { s.removePath(t, setName, linkPath, 1) - _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() + _, err = exec.Command("ils", remoteLink).CombinedOutput() So(err, ShouldNotBeNil) _, err = exec.Command("ils", remoteInode).CombinedOutput() @@ -2156,7 +2170,7 @@ func TestRemove(t *testing.T) { s.removePath(t, setName, linkPath, 1) - _, err = exec.Command("ils", filepath.Join(remotePath, "link")).CombinedOutput() + _, err = exec.Command("ils", remoteLink).CombinedOutput() So(err, ShouldNotBeNil) _, err = exec.Command("ils", remoteInode).CombinedOutput() diff --git a/put/baton.go b/put/baton.go index 3c8f74f..f872f77 100644 --- a/put/baton.go +++ b/put/baton.go @@ -33,6 +33,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "sync" "time" @@ -521,8 +522,7 @@ func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path strin MetaKeyRequester: strings.Join(requesters, ","), } - if metaToRemove[MetaKeySets] == newMeta[MetaKeySets] && - metaToRemove[MetaKeyRequester] == newMeta[MetaKeyRequester] { + if reflect.DeepEqual(metaToRemove, newMeta) { return nil } diff --git a/server/server.go b/server/server.go index 6126a10..86e376b 100644 --- a/server/server.go +++ b/server/server.go @@ -206,6 +206,9 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC return nil } +// handleRemoveRequests removes objects belonging to the provided reserveGroup +// inside removeQueue from iRODS and data base. This function should be called +// inside a go routine, so the user API request is not locked. func (s *Server) handleRemoveRequests(reserveGroup string) { baton, err := put.GetBatonHandlerWithMetaClient() if err != nil { @@ -221,11 +224,10 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { } err = s.removeRequestFromIRODSandDB(&removeReq, baton) - if err != nil { - beenReleased := s.setErrorOrReleaseItem(item, removeReq, err) - if beenReleased { - continue - } + + beenReleased := s.handleErrorOrReleaseItem(item, removeReq, err) + if beenReleased { + continue } errr := s.removeQueue.Remove(context.Background(), removeReq.key()) @@ -243,29 +245,36 @@ func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq, baton *put.Ba return s.removeFileFromIRODSandDB(removeReq, baton) } -func (s *Server) setErrorOrReleaseItem(item *queue.Item, removeReq removeReq, err error) bool { - if uint8(item.Stats().Releases) < jobRetries { - item.SetData(removeReq) +// handleErrorOrReleaseItem returns immediately if there was no error. Otherwise +// it releases the item with updated data from a queue, provided it has attempts +// left, or sets the error on the set. Returns whether the item was released. +func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq removeReq, err error) bool { + if err == nil { + return false + } - err = s.removeQueue.SetDelay(removeReq.key(), retryDelay) - if err != nil { - s.Logger.Printf("%s", err.Error()) + if uint8(item.Stats().Releases) >= jobRetries { //nolint:gosec + errs := s.db.SetError(removeReq.set.ID(), "Error when removing: "+err.Error()) + if errs != nil { + s.Logger.Printf("Could not put error on set due to: %s\nError was: %s\n", errs.Error(), err.Error()) } - err = s.removeQueue.Release(context.Background(), removeReq.key()) - if err != nil { - s.Logger.Printf("%s", err.Error()) - } + return false + } - return true + item.SetData(removeReq) + + err = s.removeQueue.SetDelay(removeReq.key(), retryDelay) + if err != nil { + s.Logger.Printf("%s", err.Error()) } - errs := s.db.SetError(removeReq.set.ID(), "Error when removing: "+err.Error()) - if errs != nil { - s.Logger.Printf("Could not put error on set due to: %s\nError was: %s\n", errs.Error(), err.Error()) + err = s.removeQueue.Release(context.Background(), removeReq.key()) + if err != nil { + s.Logger.Printf("%s", err.Error()) } - return false + return true } // reserveRemoveRequest reserves an item from the given reserve group from the @@ -282,12 +291,12 @@ func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeR s.Logger.Printf("%s", err.Error()) } - removeReq, ok := item.Data().(removeReq) + remReq, ok := item.Data().(removeReq) if !ok { s.Logger.Printf("Invalid data type in remove queue") } - return item, removeReq, nil + return item, remReq, nil } // rac is our queue's ready added callback which will get all ready put Requests diff --git a/server/setdb.go b/server/setdb.go index 221001d..064b9a4 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -101,14 +101,15 @@ const ( // EndPointAuthRemoveDirs = gas.EndPointAuth + removeDirsPath - ErrNoAuth = gas.Error("auth must be enabled") - ErrNoSetDBDirFound = gas.Error("set database directory not found") - ErrNoRequester = gas.Error("requester not supplied") - ErrBadRequester = gas.Error("you are not the set requester") - ErrNotAdmin = gas.Error("you are not the server admin") - ErrBadSet = gas.Error("set with that id does not exist") - ErrInvalidInput = gas.Error("invalid input") - ErrInternal = gas.Error("internal server error") + ErrNoAuth = gas.Error("auth must be enabled") + ErrNoSetDBDirFound = gas.Error("set database directory not found") + ErrNoRequester = gas.Error("requester not supplied") + ErrBadRequester = gas.Error("you are not the set requester") + ErrNotAdmin = gas.Error("you are not the server admin") + ErrBadSet = gas.Error("set with that id does not exist") + ErrInvalidInput = gas.Error("invalid input") + ErrInternal = gas.Error("internal server error") + ErrElementNotInSlice = gas.Error("element not in slice") paramRequester = "requester" paramSetID = "id" @@ -426,7 +427,7 @@ func (s *Server) removeFilesAndDirs(set *set.Set, filePaths, dirPaths []string) go s.handleRemoveRequests(reserveGroup) - return s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) + return s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) //nolint:gosec } type removeReq struct { @@ -556,7 +557,7 @@ func getNamesFromSets(sets []*set.Set) []string { func removeElementFromSlice(slice []string, element string) ([]string, error) { index := slices.Index(slice, element) if index < 0 { - return nil, fmt.Errorf("Element %s not in slice", element) + return nil, fmt.Errorf("%w: %s", ErrElementNotInSlice, element) } slice[index] = slice[len(slice)-1] @@ -602,7 +603,8 @@ func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup return filepaths, err } -func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, reserveGroup string, paths []string) ([]string, []*queue.ItemDef, error) { +func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, + reserveGroup string, paths []string) ([]string, []*queue.ItemDef, error) { var filepaths []string defs := make([]*queue.ItemDef, len(paths)) @@ -646,12 +648,7 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton if !removeReq.isRemovedFromIRODS { err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) if err != nil { - entry.LastError = err.Error() - - erru := s.db.UploadEntry(removeReq.set.ID(), removeReq.path, entry) - if erru != nil { - s.Logger.Printf("%s", erru.Error()) - } + s.setErrorOnEntry(entry, removeReq.set.ID(), removeReq.path, err) return err } @@ -667,6 +664,15 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) } +func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) { + entry.LastError = err.Error() + + erru := s.db.UploadEntry(sid, path, entry) + if erru != nil { + s.Logger.Printf("%s", erru.Error()) + } +} + func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { transformer, err := removeReq.set.MakeTransformer() if err != nil { @@ -690,44 +696,6 @@ func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) } -// func (s *Server) removeFromIRODSandDB(userSet *set.Set, dirpaths, filepaths []string) error { -// err := s.removePathsFromIRODS(userSet, dirpaths, filepaths) -// if err != nil { -// return err -// } - -// err = s.removePathsFromDB(userSet, dirpaths, filepaths) -// if err != nil { -// return err -// } - -// return s.db.SetNewCounts(userSet.ID()) - -// } - -// func (s *Server) removePathsFromIRODS(set *set.Set, dirpaths, filepaths []string) error { -// baton, transformer, err := s.getBatonAndTransformer(set) -// if err != nil { -// return err -// } - -// err = s.removeFilesFromIRODS(set, filepaths, baton, transformer) -// if err != nil { -// return err -// } - -// return s.removeDirsFromIRODS(set, dirpaths, baton, transformer) -// } - -// func (s *Server) removePathsFromDB(set *set.Set, dirpaths, filepaths []string) error { -// err := s.db.RemoveDirEntries(set.ID(), dirpaths) -// if err != nil { -// return err -// } - -// return s.db.RemoveFileEntries(set.ID(), filepaths) -// } - // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id // from the URL parameter if Requester matches logged-in username. func (s *Server) bindPathsAndValidateSet(c *gin.Context) (string, []string, bool) { diff --git a/set/db.go b/set/db.go index e6306b7..bba37ef 100644 --- a/set/db.go +++ b/set/db.go @@ -1025,7 +1025,7 @@ func (d *DBRO) GetFileEntries(setID string) ([]*Entry, error) { func (d *DBRO) GetFileEntryForSet(setID, filePath string) (*Entry, error) { var entry *Entry - if err := d.db.Update(func(tx *bolt.Tx) error { + if err := d.db.View(func(tx *bolt.Tx) error { var err error entry, _, err = d.getEntry(tx, setID, filePath) diff --git a/set/set.go b/set/set.go index 23f3eaa..c1cefdc 100644 --- a/set/set.go +++ b/set/set.go @@ -461,7 +461,7 @@ func (s *Set) entryStatusToSetCounts(entry *Entry) { //nolint:gocyclo } } -func (s *Set) removedEntryStatusToSetCounts(entry *Entry) { //nolint:gocyclo +func (s *Set) removedEntryStatusToSetCounts(entry *Entry) { switch entry.Status { //nolint:exhaustive case Uploaded: s.Uploaded-- From 2b954c61396c8b41c9a1f884fdfd21688c3a033a Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 12 Feb 2025 14:51:06 +0000 Subject: [PATCH 23/35] Refactor classification of a path to use db instead of file system --- cmd/remove.go | 19 ++++++------------- main_test.go | 25 +++++++++++++++++++++++-- server/client.go | 21 ++++----------------- server/setdb.go | 24 ++---------------------- set/db.go | 39 +++++++++++++++++++++++++++++---------- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/cmd/remove.go b/cmd/remove.go index b5d91bc..9e9d591 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -74,8 +74,7 @@ var removeCmd = &cobra.Command{ die("exactly one of --items or --path must be provided") } - var files []string - var dirs []string + var paths []string client, err := newServerClient(serverURL, serverCert) if err != nil { @@ -83,8 +82,7 @@ var removeCmd = &cobra.Command{ } if removeItems != "" { - filesAndDirs := readPaths(removeItems, fofnLineSplitter(removeNull)) - files, dirs = categorisePaths(filesAndDirs, files, dirs) + paths = append(paths, readPaths(removeItems, fofnLineSplitter(removeNull))...) } if removePath != "" { @@ -93,15 +91,10 @@ var removeCmd = &cobra.Command{ die("%s", err.Error()) } - //TODO if dir does not exist then we will add it as a file - if pathIsDir(removePath) { - dirs = append(dirs, removePath) - } else { - files = append(files, removePath) - } + paths = append(paths, removePath) } - remove(client, removeUser, removeName, files, dirs) + remove(client, removeUser, removeName, paths) }, } @@ -125,7 +118,7 @@ func init() { } // remove does the main job of sending the set, files and dirs to the server. -func remove(client *server.Client, user, name string, files, dirs []string) { +func remove(client *server.Client, user, name string, paths []string) { sets := getSetByName(client, user, name) if len(sets) == 0 { warn("No backup sets found with name %s", name) @@ -133,7 +126,7 @@ func remove(client *server.Client, user, name string, files, dirs []string) { return } - err := client.RemoveFiles(sets[0].ID(), files, dirs) + err := client.RemoveFilesAndDirs(sets[0].ID(), paths) if err != nil { die("%s", err.Error()) } diff --git a/main_test.go b/main_test.go index cafb243..2feffd2 100644 --- a/main_test.go +++ b/main_test.go @@ -2082,6 +2082,22 @@ func TestRemove(t *testing.T) { 0, dir1+" => ") }) + Convey("Remove removes the dir from the set even if it no longer exists", func() { + err = os.RemoveAll(dir1) + So(err, ShouldBeNil) + + s.removePath(t, setName, dir1, 2) + + s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + 0, dir2) + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1+"/") + + s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + 0, dir1+" => ") + }) + Convey("Remove removes an empty dir from the set", func() { s.removePath(t, setName, dir2, 1) @@ -2199,7 +2215,7 @@ func TestRemove(t *testing.T) { Convey("Remove returns an error if you try to remove just the file", func() { s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", file5}, - 1, fmt.Sprintf("%s is not part of the backup set [%s]", file5, setName)) + 1, fmt.Sprintf("path(s) do not belong to the backup set : [%s] [%s]", file5, setName)) }) Convey("Remove ignores the file if you remove the directory", func() { @@ -2211,12 +2227,17 @@ func TestRemove(t *testing.T) { Convey("And a new directory", func() { dir3 := filepath.Join(path, "path/to/new/dir/") + Convey("Remove returns an error if you try to remove the directory that doesn't exist", func() { + s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", dir3}, + 1, fmt.Sprintf("path(s) do not belong to the backup set : [%s] [%s]", dir3, setName)) + }) + err = os.MkdirAll(dir3, 0755) So(err, ShouldBeNil) Convey("Remove returns an error if you try to remove the directory", func() { s.confirmOutputContains(t, []string{"remove", "--name", setName, "--path", dir3}, - 1, fmt.Sprintf("%s is not part of the backup set [%s]", dir3, setName)) + 1, fmt.Sprintf("path(s) do not belong to the backup set : [%s] [%s]", dir3, setName)) }) }) diff --git a/server/client.go b/server/client.go index 2926d45..0e2c7e4 100644 --- a/server/client.go +++ b/server/client.go @@ -48,9 +48,6 @@ const ( numHandlers = 2 millisecondsInSecond = 1000 - fileKeyForJSON = "files" - dirKeyForJSON = "dirs" - ErrKilledDueToStuck = gas.Error(put.ErrStuckTimeout) ) @@ -618,18 +615,8 @@ func (c *Client) RetryFailedSetUploads(id string) (int, error) { return retried, err } -func (c *Client) RemoveFiles(setID string, files, dirs []string) error { - return c.putThing(EndPointAuthRemovePaths+"/"+setID, mapToBytes(files, dirs)) +// RemoveFilesAndDirs removes the given paths from the backup set with the given +// ID. +func (c *Client) RemoveFilesAndDirs(setID string, paths []string) error { + return c.putThing(EndPointAuthRemovePaths+"/"+setID, stringsToBytes(paths)) } - -func mapToBytes(files, dirs []string) map[string][][]byte { - newMap := make(map[string][][]byte) - newMap[fileKeyForJSON] = stringsToBytes(files) - newMap[dirKeyForJSON] = stringsToBytes(dirs) - - return newMap -} - -// func (c *Client) RemoveDirs(setID string, dirs []string) error { -// return c.putThing(EndPointAuthRemoveDirs+"/"+setID, stringsToBytes(dirs)) -// } diff --git a/server/setdb.go b/server/setdb.go index 064b9a4..cec8bc4 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -383,14 +383,14 @@ func (s *Server) putFiles(c *gin.Context) { } func (s *Server) removePaths(c *gin.Context) { - sid, filePaths, dirPaths, ok := s.parseRemoveParamsAndValidateSet(c) + sid, paths, ok := s.bindPathsAndValidateSet(c) if !ok { return } set := s.db.GetByID(sid) - err := s.db.ValidateFileAndDirPaths(set, filePaths, dirPaths) + filePaths, dirPaths, err := s.db.ValidateFileAndDirPaths(set, paths) if err != nil { c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck @@ -715,26 +715,6 @@ func (s *Server) bindPathsAndValidateSet(c *gin.Context) (string, []string, bool return set.ID(), bytesToStrings(bpaths), true } -// parseRemoveParamsAndValidateSet gets the file paths and dir paths out of the -// JSON body, and the set id from the URL parameter if Requester matches -// logged-in username. -func (s *Server) parseRemoveParamsAndValidateSet(c *gin.Context) (string, []string, []string, bool) { - bmap := make(map[string][][]byte) - - if err := c.BindJSON(&bmap); err != nil { - c.AbortWithError(http.StatusBadRequest, err) //nolint:errcheck - - return "", nil, nil, false - } - - set, ok := s.validateSet(c) - if !ok { - return "", nil, nil, false - } - - return set.ID(), bytesToStrings(bmap[fileKeyForJSON]), bytesToStrings(bmap[dirKeyForJSON]), true -} - // validateSet gets the id parameter from the given context and checks a // corresponding set exists and the logged-in user is the same as the set's // Requester. If so, returns the set and true. If not, Aborts with an error diff --git a/set/db.go b/set/db.go index bba37ef..fc15ee2 100644 --- a/set/db.go +++ b/set/db.go @@ -62,6 +62,7 @@ const ( ErrInvalidEntry = "invalid set entry" ErrInvalidTransformerPath = "invalid transformer path concatenation" ErrNoAddDuringDiscovery = "can't add set while set is being discovered" + ErrPathNotInSet = "path(s) do not belong to the backup set" setsBucket = "sets" userToSetBucket = "userLookup" @@ -264,26 +265,35 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } -func (d *DB) ValidateFileAndDirPaths(set *Set, filePaths, dirPaths []string) error { - err := d.validateFilePaths(set, filePaths) +func (d *DB) ValidateFileAndDirPaths(set *Set, paths []string) ([]string, []string, error) { + filePaths, notFilePaths, err := d.validateFilePaths(set, paths) if err != nil { - return err + return nil, nil, err + } + + dirPaths, invalidPaths, err := d.validateDirPaths(set, notFilePaths) + if err != nil { + return nil, nil, err + } + + if len(invalidPaths) > 0 { + err = Error{Msg: fmt.Sprintf("%s : %v", ErrPathNotInSet, invalidPaths), id: set.Name} } - return d.validateDirPaths(set, dirPaths) + return filePaths, dirPaths, err } -func (d *DB) validateFilePaths(set *Set, paths []string) error { +func (d *DB) validateFilePaths(set *Set, paths []string) ([]string, []string, error) { return d.validatePaths(set, fileBucket, discoveredBucket, paths) } -func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) error { +func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) ([]string, []string, error) { entriesMap := make(map[string]bool) for _, bucket := range []string{bucket1, bucket2} { entries, err := d.getEntries(set.ID(), bucket) if err != nil { - return err + return nil, nil, err } for _, entry := range entries { @@ -291,16 +301,25 @@ func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) er } } + var ( //nolint:prealloc + validPaths []string + invalidPaths []string + ) + for _, path := range paths { if _, ok := entriesMap[path]; !ok { - return Error{path + " is not part of the backup set", set.Name} + invalidPaths = append(invalidPaths, path) + + continue } + + validPaths = append(validPaths, path) } - return nil + return validPaths, invalidPaths, nil } -func (d *DB) validateDirPaths(set *Set, paths []string) error { +func (d *DB) validateDirPaths(set *Set, paths []string) ([]string, []string, error) { return d.validatePaths(set, dirBucket, discoveredBucket, paths) } From 621627aff4c76203f9e3a7a6e7023a2190a2d331 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 12 Feb 2025 15:33:09 +0000 Subject: [PATCH 24/35] Modify code for pull request --- cmd/filestatus.go | 3 - cmd/remove.go | 12 +-- put/baton.go | 4 + server/server.go | 45 +++++----- server/setdb.go | 218 +++++++++++++++++++++++----------------------- set/db.go | 42 ++++++--- set/set.go | 10 ++- set/set_test.go | 4 +- 8 files changed, 179 insertions(+), 159 deletions(-) diff --git a/cmd/filestatus.go b/cmd/filestatus.go index 170fdbc..9e909c7 100644 --- a/cmd/filestatus.go +++ b/cmd/filestatus.go @@ -60,13 +60,11 @@ func init() { func fileSummary(dbPath, filePath string, useIrods bool) error { db, err := set.NewRO(dbPath) if err != nil { - println("fail 1") return err } sets, err := db.GetAll() if err != nil { - println("fail 2") return err } @@ -77,7 +75,6 @@ func fileSummary(dbPath, filePath string, useIrods bool) error { } if err := fsg.printFileStatuses(sets); err != nil { - println("fail 3") return err } diff --git a/cmd/remove.go b/cmd/remove.go index 9e9d591..1c841e1 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -44,12 +44,12 @@ var removeNull bool // removeCmd represents the add command. var removeCmd = &cobra.Command{ Use: "remove", - Short: "Remove files from backed up set", - Long: `Remove files from backed up set. + Short: "Remove objects from backed up set", + Long: `Remove objects from backed up set. - Remove files from a backed up set by providing the files or directories to be - removed. This will remove files from the set and from iRODS if it is not found - in any other sets. + Remove objects from a backed up set by providing the files and/or directories + to be removed. This will remove objects from the set and from iRODS if it is + not found in any other sets. You also need to supply the ibackup server's URL in the form domain:port (using the IBACKUP_SERVER_URL environment variable, or overriding that with the --url @@ -104,7 +104,7 @@ func init() { // flags specific to this sub-command removeCmd.Flags().StringVar(&removeUser, "user", currentUsername(), "pretend to be this user (only works if you started the server)") - removeCmd.Flags().StringVarP(&removeName, "name", "n", "", "remove files from the set with this name") + removeCmd.Flags().StringVarP(&removeName, "name", "n", "", "remove objects from the set with this name") removeCmd.Flags().StringVarP(&removeItems, "items", "i", "", "path to file with one absolute local directory or file path per line") removeCmd.Flags().StringVarP(&removePath, "path", "p", "", diff --git a/put/baton.go b/put/baton.go index f872f77..143d2af 100644 --- a/put/baton.go +++ b/put/baton.go @@ -594,6 +594,7 @@ func (b *Baton) queryIRODSMeta(transformer PathTransformer, meta map[string]stri return items, err } +// RemoveDirFromIRODS removes the given directory from iRODS given it is empty. func (b *Baton) RemoveDirFromIRODS(path string) error { it := &ex.RodsItem{ IPath: path, @@ -608,6 +609,7 @@ func (b *Baton) RemoveDirFromIRODS(path string) error { return err } +// RemoveMeta removes the given metadata from a given object in iRODS. func (b *Baton) RemoveMeta(path string, meta map[string]string) error { it := remotePathToRodsItem(path) it.IAVUs = metaToAVUs(meta) @@ -621,6 +623,7 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { return err } +// GetMeta gets all the metadata for the given object in iRODS. func (b *Baton) GetMeta(path string) (map[string]string, error) { it, err := b.metaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ IPath: filepath.Dir(path), @@ -638,6 +641,7 @@ func remotePathToRodsItem(path string) *ex.RodsItem { } } +// AddMeta adds the given metadata to a given object in iRODS. func (b *Baton) AddMeta(path string, meta map[string]string) error { it := remotePathToRodsItem(path) it.IAVUs = metaToAVUs(meta) diff --git a/server/server.go b/server/server.go index 86e376b..7dfe425 100644 --- a/server/server.go +++ b/server/server.go @@ -144,7 +144,6 @@ func New(conf Config) (*Server, error) { s.clientQueue.SetTTRCallback(s.clientTTRC) s.Server.Router().Use(gas.IncludeAbortErrorsInBody) - // s.removeQueue.SetReadyAddedCallback(s.removeCallback) s.monitor = NewMonitor(func(given *set.Set) { if err := s.discoverSet(given); err != nil { @@ -237,6 +236,28 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { } } +// reserveRemoveRequest reserves an item from the given reserve group from the +// remove queue and converts it to a removeRequest. Returns nil and no error if +// the queue is empty. +func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeReq, error) { + item, err := s.removeQueue.Reserve(reserveGroup, retryDelay+2*time.Second) + if err != nil { + qerr, ok := err.(queue.Error) //nolint:errorlint + if ok && errors.Is(qerr.Err, queue.ErrNothingReady) { + return nil, removeReq{}, err + } + + s.Logger.Printf("%s", err.Error()) + } + + remReq, ok := item.Data().(removeReq) + if !ok { + s.Logger.Printf("Invalid data type in remove queue") + } + + return item, remReq, nil +} + func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { if removeReq.isDir { return s.removeDirFromIRODSandDB(removeReq, baton) @@ -277,28 +298,6 @@ func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq removeReq, return true } -// reserveRemoveRequest reserves an item from the given reserve group from the -// remove queue and converts it to a removeRequest. Returns nil and no error if -// the queue is empty. -func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeReq, error) { - item, err := s.removeQueue.Reserve(reserveGroup, retryDelay+2*time.Second) - if err != nil { - qerr, ok := err.(queue.Error) //nolint:errorlint - if ok && errors.Is(qerr.Err, queue.ErrNothingReady) { - return nil, removeReq{}, err - } - - s.Logger.Printf("%s", err.Error()) - } - - remReq, ok := item.Data().(removeReq) - if !ok { - s.Logger.Printf("Invalid data type in remove queue") - } - - return item, remReq, nil -} - // rac is our queue's ready added callback which will get all ready put Requests // and ensure there are enough put jobs added to wr. // diff --git a/server/setdb.go b/server/setdb.go index cec8bc4..245ebb7 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -58,7 +58,6 @@ const ( fileStatusPath = "/file_status" fileRetryPath = "/retry" removePathsPath = "/remove_paths" - removeDirsPath = "/remove_dirs" // EndPointAuthSet is the endpoint for getting and setting sets. EndPointAuthSet = gas.EndPointAuth + setPath @@ -95,12 +94,9 @@ const ( // EndPointAuthRetryEntries is the endpoint for retrying file uploads. EndPointAuthRetryEntries = gas.EndPointAuth + fileRetryPath - // + // EndPointAuthRemovePaths is the endpoint for removing objects from sets. EndPointAuthRemovePaths = gas.EndPointAuth + removePathsPath - // - EndPointAuthRemoveDirs = gas.EndPointAuth + removeDirsPath - ErrNoAuth = gas.Error("auth must be enabled") ErrNoSetDBDirFound = gas.Error("set database directory not found") ErrNoRequester = gas.Error("requester not supplied") @@ -430,6 +426,18 @@ func (s *Server) removeFilesAndDirs(set *set.Set, filePaths, dirPaths []string) return s.db.UpdateSetTotalToRemove(set.ID(), uint64(len(filePaths)+len(dirPaths)+len(dirFilePaths))) //nolint:gosec } +func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGroup string) error { + if len(paths) == 0 { + return nil + } + + defs := makeItemsDefsFromFilePaths(set, reserveGroup, paths) + + _, _, err := s.removeQueue.AddMany(context.Background(), defs) + + return err +} + type removeReq struct { path string set *set.Set @@ -442,23 +450,64 @@ func (rq removeReq) key() string { return strings.Join([]string{rq.set.ID(), rq.path}, ":") } -func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGroup string) error { +func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []string) []*queue.ItemDef { + defs := make([]*queue.ItemDef, len(paths)) + + for i, path := range paths { + rq := removeReq{path: path, set: set, isDir: false} + + defs[i] = &queue.ItemDef{ + Key: rq.key(), + Data: rq, + TTR: ttr, + ReserveGroup: reserveGroup, + } + } + + return defs +} + +func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup string) ([]string, error) { if len(paths) == 0 { - return nil + return []string{}, nil } - defs := makeItemsDefsFromFilePaths(set, reserveGroup, paths) + filepaths, dirDefs, err := s.makeItemsDefsAndFilePathsFromDirPaths(set, reserveGroup, paths) + if err != nil { + return nil, err + } - _, _, err := s.removeQueue.AddMany(context.Background(), defs) + fileDefs := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) - return err + _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) + if err != nil { + return nil, err + } + + _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) + + return filepaths, err } -func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []string) []*queue.ItemDef { +func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, + reserveGroup string, paths []string) ([]string, []*queue.ItemDef, error) { + var filepaths []string + defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { - rq := removeReq{path: path, set: set, isDir: false} + rq := removeReq{path: path, set: set, isDir: true} + + dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) + if err != nil { + return nil, nil, err + } + + if len(dirFilepaths) == 0 { + rq.isDirEmpty = true + } + + filepaths = append(filepaths, dirFilepaths...) defs[i] = &queue.ItemDef{ Key: rq.key(), @@ -468,7 +517,37 @@ func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []strin } } - return defs + return filepaths, defs, nil +} + +func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { + entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) + if err != nil { + return err + } + + transformer, err := removeReq.set.MakeTransformer() + if err != nil { + return err + } + + if !removeReq.isRemovedFromIRODS { + err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) + if err != nil { + s.setErrorOnEntry(entry, removeReq.set.ID(), removeReq.path, err) + + return err + } + + removeReq.isRemovedFromIRODS = true + } + + err = s.db.RemoveFileEntry(removeReq.set.ID(), removeReq.path) + if err != nil { + return err + } + + return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) } func (s *Server) removeFileFromIRODS(set *set.Set, path string, @@ -566,134 +645,51 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { return slice[:len(slice)-1], nil } -func (s *Server) removeDirFromIRODS(path string, baton *put.Baton, - transformer put.PathTransformer) error { - rpath, err := transformer(path) - if err != nil { - return err - } - - err = baton.RemoveDirFromIRODS(rpath) - if err != nil { - return err - } - - return nil -} - -func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup string) ([]string, error) { - if len(paths) == 0 { - return []string{}, nil - } - - filepaths, dirDefs, err := s.makeItemsDefsAndFilePathsFromDirPaths(set, reserveGroup, paths) - if err != nil { - return nil, err - } - - fileDefs := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) - - _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) - if err != nil { - return nil, err - } - - _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) - - return filepaths, err -} - -func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, - reserveGroup string, paths []string) ([]string, []*queue.ItemDef, error) { - var filepaths []string - - defs := make([]*queue.ItemDef, len(paths)) - - for i, path := range paths { - rq := removeReq{path: path, set: set, isDir: true} - - dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) - if err != nil { - return nil, nil, err - } - - if len(dirFilepaths) == 0 { - rq.isDirEmpty = true - } - - filepaths = append(filepaths, dirFilepaths...) +func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) { + entry.LastError = err.Error() - defs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, - ReserveGroup: reserveGroup, - } + erru := s.db.UploadEntry(sid, path, entry) + if erru != nil { + s.Logger.Printf("%s", erru.Error()) } - - return filepaths, defs, nil } -func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { - entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) - if err != nil { - return err - } - +func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { transformer, err := removeReq.set.MakeTransformer() if err != nil { return err } - if !removeReq.isRemovedFromIRODS { - err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) + if !removeReq.isDirEmpty && !removeReq.isRemovedFromIRODS { + err = s.removeDirFromIRODS(removeReq.path, baton, transformer) if err != nil { - s.setErrorOnEntry(entry, removeReq.set.ID(), removeReq.path, err) - return err } removeReq.isRemovedFromIRODS = true } - err = s.db.RemoveFileEntries(removeReq.set.ID(), removeReq.path) + err = s.db.RemoveDirEntry(removeReq.set.ID(), removeReq.path) if err != nil { return err } - return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) -} - -func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) { - entry.LastError = err.Error() - - erru := s.db.UploadEntry(sid, path, entry) - if erru != nil { - s.Logger.Printf("%s", erru.Error()) - } + return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) } -func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { - transformer, err := removeReq.set.MakeTransformer() +func (s *Server) removeDirFromIRODS(path string, baton *put.Baton, + transformer put.PathTransformer) error { + rpath, err := transformer(path) if err != nil { return err } - if !removeReq.isDirEmpty && !removeReq.isRemovedFromIRODS { - err = s.removeDirFromIRODS(removeReq.path, baton, transformer) - if err != nil { - return err - } - - removeReq.isRemovedFromIRODS = true - } - - err = s.db.RemoveDirEntries(removeReq.set.ID(), removeReq.path) + err = baton.RemoveDirFromIRODS(rpath) if err != nil { return err } - return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) + return nil } // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id diff --git a/set/db.go b/set/db.go index fc15ee2..13aa9ed 100644 --- a/set/db.go +++ b/set/db.go @@ -265,6 +265,9 @@ func (d *DB) encodeToBytes(thing interface{}) []byte { return encoded } +// ValidateFileAndDirPaths returns an error if any provided path is not in the +// given set. Also classifies the valid paths into a slice of filepaths or +// dirpaths. func (d *DB) ValidateFileAndDirPaths(set *Set, paths []string) ([]string, []string, error) { filePaths, notFilePaths, err := d.validateFilePaths(set, paths) if err != nil { @@ -287,6 +290,9 @@ func (d *DB) validateFilePaths(set *Set, paths []string) ([]string, []string, er return d.validatePaths(set, fileBucket, discoveredBucket, paths) } +// validatePaths checks if the provided paths are in atleast one of the given +// buckets for the set. Returns a slice of all valid paths and a slice of all +// invalid paths. func (d *DB) validatePaths(set *Set, bucket1, bucket2 string, paths []string) ([]string, []string, error) { entriesMap := make(map[string]bool) @@ -323,19 +329,19 @@ func (d *DB) validateDirPaths(set *Set, paths []string) ([]string, []string, err return d.validatePaths(set, dirBucket, discoveredBucket, paths) } -// RemoveFileEntries removes the provided files from a given set. -func (d *DB) RemoveFileEntries(setID string, path string) error { - err := d.removeEntries(setID, path, fileBucket) +// RemoveFileEntry removes the provided file from a given set. +func (d *DB) RemoveFileEntry(setID string, path string) error { + err := d.removeEntry(setID, path, fileBucket) if err != nil { return err } - return d.removeEntries(setID, path, discoveredBucket) + return d.removeEntry(setID, path, discoveredBucket) } -// removeEntries removes the entries with the provided entry keys from a given +// removeEntry removes the entry with the provided entry key from a given // bucket of a given set. -func (d *DB) removeEntries(setID string, entryKey string, bucketName string) error { +func (d *DB) removeEntry(setID string, entryKey string, bucketName string) error { return d.db.Update(func(tx *bolt.Tx) error { subBucketName := []byte(bucketName + separator + setID) setsBucket := tx.Bucket([]byte(setsBucket)) @@ -351,9 +357,9 @@ func (d *DB) removeEntries(setID string, entryKey string, bucketName string) err }) } -// RemoveDirEntries removes the provided directories from a given set. -func (d *DB) RemoveDirEntries(setID string, path string) error { - return d.removeEntries(setID, path, dirBucket) +// RemoveDirEntry removes the provided directory from a given set. +func (d *DB) RemoveDirEntry(setID string, path string) error { + return d.removeEntry(setID, path, dirBucket) } func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) { @@ -1188,6 +1194,8 @@ func (d *DB) SetError(setID, errMsg string) error { }) } +// UpdateBasedOnRemovedEntry updates set counts based on the given entry that's +// been removed. func (d *DB) UpdateBasedOnRemovedEntry(setID string, entry *Entry) error { return d.updateSetProperties(setID, func(got *Set) { got.SizeRemoved += entry.Size @@ -1199,19 +1207,31 @@ func (d *DB) UpdateBasedOnRemovedEntry(setID string, entry *Entry) error { }) } +// UpdateSetTotalToRemove sets num of objects to be removed to provided value +// and resets num of objects removed if the previous removal was successful +// otherwise just increases number to be removed with provided value. func (d *DB) UpdateSetTotalToRemove(setID string, num uint64) error { return d.updateSetProperties(setID, func(got *Set) { - got.NumObjectsToBeRemoved = num - got.NumObjectsRemoved = 0 + if got.NumObjectsToBeRemoved == got.NumObjectsRemoved { + got.NumObjectsToBeRemoved = num + got.NumObjectsRemoved = 0 + + return + } + + got.NumObjectsToBeRemoved += num }) } +// IncrementSetTotalRemoved increments the number of objects removed for the +// set. func (d *DB) IncrementSetTotalRemoved(setID string) error { return d.updateSetProperties(setID, func(got *Set) { got.NumObjectsRemoved++ }) } +// ResetRemoveSize resets the size removed for the set. func (d *DB) ResetRemoveSize(setID string) error { return d.updateSetProperties(setID, func(got *Set) { got.SizeRemoved = 0 diff --git a/set/set.go b/set/set.go index c1cefdc..86dbade 100644 --- a/set/set.go +++ b/set/set.go @@ -201,8 +201,12 @@ type Set struct { // remove. This is a read-only value. SizeRemoved uint64 + // NumObjectsToBeRemoved provides the number of objects to be removed in the + // current remove process. NumObjectsToBeRemoved uint64 + // numObjectsRemoved provides the number of objects already removed in the + // current remove process. NumObjectsRemoved uint64 // Error holds any error that applies to the whole set, such as an issue @@ -548,7 +552,7 @@ func (s *Set) UpdateBasedOnEntry(entry *Entry, getFileEntries func(string) ([]*E s.adjustBasedOnEntry(entry) - err := s.FixCounts(entry, getFileEntries) + err := s.fixCounts(entry, getFileEntries) if err != nil { return err } @@ -583,10 +587,10 @@ func (s *Set) checkIfComplete() { s.Uploaded, s.Replaced, s.Skipped, s.Failed, s.Missing, s.Abnormal, s.UploadedSize())) } -// FixCounts resets the set counts to 0 and goes through all the entries for +// fixCounts resets the set counts to 0 and goes through all the entries for // the set in the db to recaluclate them. The supplied entry should be one you // newly updated and that wasn't in the db before the transaction we're in. -func (s *Set) FixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, error)) error { +func (s *Set) fixCounts(entry *Entry, getFileEntries func(string) ([]*Entry, error)) error { if s.countsValid() { return nil } diff --git a/set/set_test.go b/set/set_test.go index 77e1d7c..22d5ae6 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -358,7 +358,7 @@ func TestSetDB(t *testing.T) { So(err, ShouldBeNil) Convey("Then remove files and dirs from the sets", func() { - err = db.removeEntries(set.ID(), "/a/b.txt", fileBucket) + err = db.removeEntry(set.ID(), "/a/b.txt", fileBucket) So(err, ShouldBeNil) fEntries, errg := db.GetFileEntries(set.ID()) @@ -367,7 +367,7 @@ func TestSetDB(t *testing.T) { So(fEntries[0], ShouldResemble, &Entry{Path: "/c/d.txt"}) So(fEntries[1], ShouldResemble, &Entry{Path: "/e/f.txt"}) - err = db.RemoveDirEntries(set.ID(), "/g/h") + err = db.RemoveDirEntry(set.ID(), "/g/h") So(err, ShouldBeNil) dEntries, errg := db.GetDirEntries(set.ID()) From 4b87f5b0c18b56cb9d79bafffa0545cf1caca91b Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Thu, 13 Feb 2025 14:05:37 +0000 Subject: [PATCH 25/35] Small PR fixes --- cmd/remove.go | 5 ----- cmd/status.go | 2 +- main_test.go | 7 +++++++ server/server.go | 2 +- server/setdb.go | 7 ++----- set/db.go | 2 ++ 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/remove.go b/cmd/remove.go index 1c841e1..9bc7a12 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -120,11 +120,6 @@ func init() { // remove does the main job of sending the set, files and dirs to the server. func remove(client *server.Client, user, name string, paths []string) { sets := getSetByName(client, user, name) - if len(sets) == 0 { - warn("No backup sets found with name %s", name) - - return - } err := client.RemoveFilesAndDirs(sets[0].ID(), paths) if err != nil { diff --git a/cmd/status.go b/cmd/status.go index ba1e391..45880bc 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -263,7 +263,7 @@ func displayQueueStatus(qs *server.QStatus) { func getSetByName(client *server.Client, user, name string) []*set.Set { got, err := client.GetSetByName(user, name) if err != nil { - die(err.Error()) + die("%s [%s]", err.Error(), name) } return []*set.Set{got} diff --git a/main_test.go b/main_test.go index 2feffd2..d1c2e56 100644 --- a/main_test.go +++ b/main_test.go @@ -1984,6 +1984,13 @@ func TestRemove(t *testing.T) { path := t.TempDir() transformer := "prefix=" + path + ":" + remotePath + Convey("And an invalid set name, remove returns an error", func() { + invalidSetName := "invalid_set" + + s.confirmOutputContains(t, []string{"remove", "--name", invalidSetName, "--path", path}, + 1, fmt.Sprintf("set with that id does not exist [%s]", invalidSetName)) + }) + Convey("And an added set with files and folders", func() { dir := t.TempDir() diff --git a/server/server.go b/server/server.go index 7dfe425..b303af0 100644 --- a/server/server.go +++ b/server/server.go @@ -274,7 +274,7 @@ func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq removeReq, return false } - if uint8(item.Stats().Releases) >= jobRetries { //nolint:gosec + if item.Stats().Releases >= uint32(jobRetries) { errs := s.db.SetError(removeReq.set.ID(), "Error when removing: "+err.Error()) if errs != nil { s.Logger.Printf("Could not put error on set due to: %s\nError was: %s\n", errs.Error(), err.Error()) diff --git a/server/setdb.go b/server/setdb.go index 245ebb7..77d1b5c 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -469,7 +469,7 @@ func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []strin func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup string) ([]string, error) { if len(paths) == 0 { - return []string{}, nil + return nil, nil } filepaths, dirDefs, err := s.makeItemsDefsAndFilePathsFromDirPaths(set, reserveGroup, paths) @@ -639,10 +639,7 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { return nil, fmt.Errorf("%w: %s", ErrElementNotInSlice, element) } - slice[index] = slice[len(slice)-1] - slice[len(slice)-1] = "" - - return slice[:len(slice)-1], nil + return slices.Delete(slice, index, index+1), nil } func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) { diff --git a/set/db.go b/set/db.go index 13aa9ed..00eaf43 100644 --- a/set/db.go +++ b/set/db.go @@ -362,6 +362,8 @@ func (d *DB) RemoveDirEntry(setID string, path string) error { return d.removeEntry(setID, path, dirBucket) } +// GetFilesInDir returns all file paths from inside the given directory (and all +// nested inside) for the given set using the db. func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) { var filepaths []string From 596919fd711cf3c39f5bcbcfd912f13fac6322b9 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 14 Feb 2025 15:16:03 +0000 Subject: [PATCH 26/35] Change implementation to carry out iRODS commands through a handler on the server --- cmd/server.go | 11 +++-- main_test.go | 13 +++-- put/baton.go | 83 ++++++-------------------------- put/mock.go | 49 +++++++++++++++++-- put/put.go | 13 ++++- put/put_test.go | 44 ++++++++++++++++- put/remove.go | 109 ++++++++++++++++++++++++++++++++++++++++++ server/server.go | 20 ++++---- server/server_test.go | 15 ++++-- server/setdb.go | 30 +++--------- 10 files changed, 260 insertions(+), 127 deletions(-) create mode 100644 put/remove.go diff --git a/cmd/server.go b/cmd/server.go index 52ed396..dcdecb5 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -193,11 +193,17 @@ database that you've made, to investigate. die("slack_debounce period must be positive, not: %d", serverSlackDebouncePeriod) } + handler, errb := put.GetBatonHandlerWithMetaClient() + if errb != nil { + die("failed to get baton handler: %s", errb) + } + conf := server.Config{ HTTPLogger: logWriter, Slacker: slacker, SlackMessageDebounce: time.Duration(serverSlackDebouncePeriod) * time.Second, StillRunningMsgFreq: stillRunningMsgFreq, + StorageHandler: handler, } s, err := server.New(conf) @@ -252,11 +258,6 @@ database that you've made, to investigate. die("remote backup path defined when no local backup path provided") } - handler, errb := put.GetBatonHandler() - if errb != nil { - die("failed to get baton handler: %s", errb) - } - s.EnableRemoteDBBackups(serverRemoteBackupPath, handler) } diff --git a/main_test.go b/main_test.go index d1c2e56..9eee7c5 100644 --- a/main_test.go +++ b/main_test.go @@ -1970,6 +1970,8 @@ func TestRemove(t *testing.T) { return } + resetIRODS() + dir := t.TempDir() s := new(TestServer) s.prepareFilePaths(dir) @@ -2056,18 +2058,15 @@ func TestRemove(t *testing.T) { "(total/recently uploaded/recently removed): 30 B / 40 B / 10 B\n"+ "Uploaded: 5; Replaced: 0; Skipped: 0; Failed: 0; Missing: 0; Abnormal: 0") - Convey("Remove again will remove another file", func() { - exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", file2) + Convey("Remove again will remove another object", func() { + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--path", dir1) So(exitCode, ShouldEqual, 0) - s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, - 0, "Removal status: 0 / 1 objects removed") - - s.waitForStatus(setName, "Removal status: 1 / 1 objects removed", 5*time.Second) + s.waitForStatus(setName, "Removal status: 2 / 2 objects removed", 5*time.Second) s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, - 0, file2) + 0, file3) s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, 0, "Num files: 4; Symlinks: 1; Hardlinks: 1; Size "+ diff --git a/put/baton.go b/put/baton.go index 143d2af..40f5913 100644 --- a/put/baton.go +++ b/put/baton.go @@ -33,7 +33,6 @@ import ( "fmt" "os" "path/filepath" - "reflect" "strings" "sync" "time" @@ -503,64 +502,7 @@ func metaToAVUs(meta map[string]string) []ex.AVU { return avus } -// RemovePathFromSetInIRODS removes the given path from iRODS if the path is not -// associated with any other sets. Otherwise it updates the iRODS metadata for -// the path to not include the given set. -func (b *Baton) RemovePathFromSetInIRODS(transformer PathTransformer, path string, - sets, requesters []string, meta map[string]string) error { - if len(sets) == 0 { - return b.handleHardlinkAndRemoveFromIRODS(path, transformer, meta) - } - - metaToRemove := map[string]string{ - MetaKeySets: meta[MetaKeySets], - MetaKeyRequester: meta[MetaKeyRequester], - } - - newMeta := map[string]string{ - MetaKeySets: strings.Join(sets, ","), - MetaKeyRequester: strings.Join(requesters, ","), - } - - if reflect.DeepEqual(metaToRemove, newMeta) { - return nil - } - - err := b.RemoveMeta(path, metaToRemove) - if err != nil { - return err - } - - return b.AddMeta(path, newMeta) -} - -// handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the -// path is found to be a hardlink, it checks if there are other hardlinks to the -// same file, if not, it removes the file. -func (b *Baton) handleHardlinkAndRemoveFromIRODS(path string, transformer PathTransformer, - meta map[string]string) error { - err := b.removeFileFromIRODS(path) - if err != nil { - return err - } - - if meta[MetaKeyHardlink] == "" { - return nil - } - - items, err := b.queryIRODSMeta(transformer, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) - if err != nil { - return err - } - - if len(items) != 0 { - return nil - } - - return b.removeFileFromIRODS(meta[MetaKeyRemoteHardlink]) -} - -func (b *Baton) removeFileFromIRODS(path string) error { +func (b *Baton) removeFile(path string) error { it := remotePathToRodsItem(path) err := timeoutOp(func() error { @@ -572,30 +514,33 @@ func (b *Baton) removeFileFromIRODS(path string) error { return err } -func (b *Baton) queryIRODSMeta(transformer PathTransformer, meta map[string]string) ([]ex.RodsItem, error) { - path, err := transformer("/") - if err != nil { - return nil, err - } - +func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) { it := &ex.RodsItem{ - IPath: path, + IPath: dirToSearch, IAVUs: metaToAVUs(meta), } var items []ex.RodsItem + var err error + err = timeoutOp(func() error { items, err = b.metaClient.MetaQuery(ex.Args{Object: true}, *it) return err - }, "remove meta error: "+path) + }, "remove meta error: "+dirToSearch) + + paths := make([]string, len(items)) + + for i, item := range items { + paths[i] = item.IPath + } - return items, err + return paths, err } // RemoveDirFromIRODS removes the given directory from iRODS given it is empty. -func (b *Baton) RemoveDirFromIRODS(path string) error { +func (b *Baton) RemoveDir(path string) error { it := &ex.RodsItem{ IPath: path, } diff --git a/put/mock.go b/put/mock.go index afb9cab..fad14e8 100644 --- a/put/mock.go +++ b/put/mock.go @@ -29,6 +29,7 @@ import ( "io" "maps" "os" + "strings" "sync" "time" ) @@ -231,14 +232,12 @@ func (l *LocalHandler) AddMeta(path string, meta map[string]string) error { // GetMeta gets the metadata stored for the given path (returns an empty map if // path is not known about or has no metadata). -// -// (Currently this is just for testing and not part of the Handler interface.) -func (l *LocalHandler) GetMeta(path string) map[string]string { +func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { l.mu.Lock() defer l.mu.Unlock() if l.meta == nil { - return make(map[string]string) + return make(map[string]string), nil } currentMeta, exists := l.meta[path] @@ -252,5 +251,45 @@ func (l *LocalHandler) GetMeta(path string) map[string]string { meta[k] = v } - return meta + return meta, nil +} + +func (l *LocalHandler) RemoveDir(path string) error { + return os.Remove(path) +} + +func (l *LocalHandler) removeFile(path string) error { + delete(l.meta, path) + + return os.Remove(path) +} + +func (l *LocalHandler) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) { + var objects []string + + for path, pathMeta := range l.meta { + if !strings.HasPrefix(path, dirToSearch) { + continue + } + + if doesMetaContainMeta(pathMeta, meta) { + objects = append(objects, path) + } + } + + return objects, nil +} + +func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { + valid := true + + for k, v := range targetMeta { + if sourceMeta[k] != v { + valid = false + + break + } + } + + return valid } diff --git a/put/put.go b/put/put.go index b43cda2..d9cb459 100644 --- a/put/put.go +++ b/put/put.go @@ -23,7 +23,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -// package put is used to put files in iRODS. +// package put is used to interact with iRODS. package put @@ -118,6 +118,17 @@ type Handler interface { // Cleanup stops any connections created earlier and does any other cleanup // needed. Cleanup() error + + GetMeta(path string) (map[string]string, error) + + // RemoveDir deletes a given empty folder + RemoveDir(path string) error + + // removeFile deletes a given file + removeFile(path string) error + + // queryMeta return paths to all objects with given metadata + queryMeta(dirToSearch string, meta map[string]string) ([]string, error) } // FileReadTester is a function that attempts to open and read the given path, diff --git a/put/put_test.go b/put/put_test.go index 1c68e97..6dfa875 100644 --- a/put/put_test.go +++ b/put/put_test.go @@ -121,7 +121,8 @@ func TestPutMock(t *testing.T) { } } - metadata := lh.GetMeta(requests[0].Remote) + metadata, errm := lh.GetMeta(requests[0].Remote) + So(errm, ShouldBeNil) So(metadata, ShouldResemble, requests[0].Meta.LocalMeta) for request := range srCh { @@ -168,6 +169,47 @@ func TestPutMock(t *testing.T) { So(err, ShouldBeNil) So(lh.cleaned, ShouldBeTrue) }) + + Convey("RemoveFile removes a file", func() { + filePath := requests[0].Remote + + err = lh.removeFile(filePath) + So(err, ShouldBeNil) + + _, err = os.Stat(filePath) + So(err, ShouldNotBeNil) + + So(lh.meta[filePath], ShouldBeNil) + + Convey("RemoveDir removes an empty directory", func() { + dirPath := filepath.Dir(filePath) + + err = lh.RemoveDir(dirPath) + So(err, ShouldBeNil) + + _, err = os.Stat(dirPath) + So(err, ShouldNotBeNil) + }) + }) + + Convey("queryMeta returns all paths with matching metadata", func() { + paths, errq := lh.queryMeta("", map[string]string{"a": "1"}) + So(errq, ShouldBeNil) + + So(len(paths), ShouldEqual, 5) + + paths, err = lh.queryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester}) + So(err, ShouldBeNil) + + So(len(paths), ShouldEqual, 1) + + Convey("queryMeta only returns paths in the provided scope", func() { + paths, errq := lh.queryMeta(expectedCollections[1], map[string]string{"a": "1"}) + So(errq, ShouldBeNil) + + So(len(paths), ShouldEqual, 2) + }) + }) }) Convey("Put() fails if the local files don't exist", func() { diff --git a/put/remove.go b/put/remove.go new file mode 100644 index 0000000..0761538 --- /dev/null +++ b/put/remove.go @@ -0,0 +1,109 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Author: Rosie Kern + * Author: Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package put + +import ( + "reflect" + "strings" +) + +// RemovePathFromSetInIRODS removes the given path from iRODS if the path is not +// associated with any other sets. Otherwise it updates the iRODS metadata for +// the path to not include the given set. +func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path string, + sets, requesters []string, meta map[string]string) error { + if len(sets) == 0 { + return handleHardlinkAndRemoveFromIRODS(handler, path, transformer, meta) + } + + metaToRemove := map[string]string{ + MetaKeySets: meta[MetaKeySets], + MetaKeyRequester: meta[MetaKeyRequester], + } + + newMeta := map[string]string{ + MetaKeySets: strings.Join(sets, ","), + MetaKeyRequester: strings.Join(requesters, ","), + } + + if reflect.DeepEqual(metaToRemove, newMeta) { + return nil + } + + err := handler.RemoveMeta(path, metaToRemove) + if err != nil { + return err + } + + return handler.AddMeta(path, newMeta) +} + +// handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the +// path is found to be a hardlink, it checks if there are other hardlinks to the +// same file, if not, it removes the file. +func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer PathTransformer, + meta map[string]string) error { + err := handler.removeFile(path) + if err != nil { + return err + } + + if meta[MetaKeyHardlink] == "" { + return nil + } + + dirToSearch, err := transformer("/") + if err != nil { + return err + } + + items, err := handler.queryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) + if err != nil { + return err + } + + if len(items) != 0 { + return nil + } + + return handler.removeFile(meta[MetaKeyRemoteHardlink]) +} + +// RemoveDirFromIRODS removes the remote path of a given directory from iRODS. +func RemoveDirFromIRODS(handler Handler, path string, transformer PathTransformer) error { + rpath, err := transformer(path) + if err != nil { + return err + } + + err = handler.RemoveDir(rpath) + if err != nil { + return err + } + + return nil +} diff --git a/server/server.go b/server/server.go index b303af0..64599e9 100644 --- a/server/server.go +++ b/server/server.go @@ -87,6 +87,9 @@ type Config struct { // messages. Default value means send unlimited messages, which will likely // result in slack restricting messages itself. SlackMessageDebounce time.Duration + + // StorageHandler is used to interact with the storage system, e.g. iRODS. + StorageHandler put.Handler } // Server is used to start a web server that provides a REST API to the setdb @@ -111,6 +114,7 @@ type Server struct { stillRunningMsgFreq time.Duration serverAliveCh chan bool uploadTracker *uploadTracker + storageHandler put.Handler mapMu sync.RWMutex creatingCollections map[string]bool @@ -139,6 +143,7 @@ func New(conf Config) (*Server, error) { uploadTracker: newUploadTracker(conf.Slacker, conf.SlackMessageDebounce), iRODSTracker: newiRODSTracker(conf.Slacker, conf.SlackMessageDebounce), clientQueue: queue.New(context.Background(), "client"), + storageHandler: conf.StorageHandler, } s.clientQueue.SetTTRCallback(s.clientTTRC) @@ -209,20 +214,13 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC // inside removeQueue from iRODS and data base. This function should be called // inside a go routine, so the user API request is not locked. func (s *Server) handleRemoveRequests(reserveGroup string) { - baton, err := put.GetBatonHandlerWithMetaClient() - if err != nil { - s.Logger.Printf("%s", err.Error()) - - return - } - for { item, removeReq, err := s.reserveRemoveRequest(reserveGroup) if err != nil { break } - err = s.removeRequestFromIRODSandDB(&removeReq, baton) + err = s.removeRequestFromIRODSandDB(&removeReq) beenReleased := s.handleErrorOrReleaseItem(item, removeReq, err) if beenReleased { @@ -258,12 +256,12 @@ func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeR return item, remReq, nil } -func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { +func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq) error { if removeReq.isDir { - return s.removeDirFromIRODSandDB(removeReq, baton) + return s.removeDirFromIRODSandDB(removeReq) } - return s.removeFileFromIRODSandDB(removeReq, baton) + return s.removeFileFromIRODSandDB(removeReq) } // handleErrorOrReleaseItem returns immediately if there was no error. Otherwise diff --git a/server/server_test.go b/server/server_test.go index f848c35..75c38a2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1878,7 +1878,8 @@ func TestServer(t *testing.T) { expectedSetSize += r.UploadedSize() if r.Symlink != "" { - remoteMeta := handler.GetMeta(remote) + remoteMeta, errr := handler.GetMeta(remote) + So(errr, ShouldBeNil) So(remoteMeta, ShouldNotBeNil) So(remoteMeta[put.MetaKeySymlink], ShouldEqual, entries[3].Dest) } @@ -1981,7 +1982,8 @@ func TestServer(t *testing.T) { So(err, ShouldBeNil) if n == 3 { - remoteMeta := handler.GetMeta(remote) + remoteMeta, errr := handler.GetMeta(remote) + So(errr, ShouldBeNil) So(remoteMeta, ShouldNotBeNil) So(remoteMeta[put.MetaKeySymlink], ShouldEqual, entries[2].Dest) @@ -2873,7 +2875,8 @@ func TestServer(t *testing.T) { remote1, errg := transformer(path2) So(errg, ShouldBeNil) - remoteMeta := handler.GetMeta(remote1) + remoteMeta, errr := handler.GetMeta(remote1) + So(errr, ShouldBeNil) So(remoteMeta, ShouldNotBeNil) So(remoteMeta[put.MetaKeyHardlink], ShouldEqual, path2) So(remoteMeta[put.MetaKeyRemoteHardlink], ShouldEqual, inodeFile) @@ -2885,7 +2888,8 @@ func TestServer(t *testing.T) { remote2, errg := transformer(path3) So(errg, ShouldBeNil) - remoteMeta = handler.GetMeta(remote2) + remoteMeta, err = handler.GetMeta(remote2) + So(err, ShouldBeNil) So(remoteMeta, ShouldNotBeNil) So(remoteMeta[put.MetaKeyHardlink], ShouldEqual, path3) So(remoteMeta[put.MetaKeyRemoteHardlink], ShouldEqual, inodeFile) @@ -2898,7 +2902,8 @@ func TestServer(t *testing.T) { So(errs, ShouldBeNil) So(info.Size(), ShouldEqual, 1) - inodeMeta := handler.GetMeta(inodeFile) + inodeMeta, errm := handler.GetMeta(inodeFile) + So(errm, ShouldBeNil) So(inodeMeta, ShouldNotBeNil) So(inodeMeta[put.MetaKeyHardlink], ShouldEqual, path1) So(inodeMeta[put.MetaKeyRemoteHardlink], ShouldBeBlank) diff --git a/server/setdb.go b/server/setdb.go index 77d1b5c..e83c6ab 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -520,7 +520,7 @@ func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, return filepaths, defs, nil } -func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { +func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq) error { entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) if err != nil { return err @@ -532,7 +532,7 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton } if !removeReq.isRemovedFromIRODS { - err = s.removeFileFromIRODS(removeReq.set, removeReq.path, baton, transformer) + err = s.removeFileFromIRODS(removeReq.set, removeReq.path, transformer) if err != nil { s.setErrorOnEntry(entry, removeReq.set.ID(), removeReq.path, err) @@ -550,14 +550,13 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq, baton *put.Baton return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) } -func (s *Server) removeFileFromIRODS(set *set.Set, path string, - baton *put.Baton, transformer put.PathTransformer) error { +func (s *Server) removeFileFromIRODS(set *set.Set, path string, transformer put.PathTransformer) error { rpath, err := transformer(path) if err != nil { return err } - remoteMeta, err := baton.GetMeta(rpath) + remoteMeta, err := s.storageHandler.GetMeta(rpath) if err != nil { return err } @@ -567,7 +566,7 @@ func (s *Server) removeFileFromIRODS(set *set.Set, path string, return err } - return baton.RemovePathFromSetInIRODS(transformer, rpath, sets, requesters, remoteMeta) + return put.RemovePathFromSetInIRODS(s.storageHandler, transformer, rpath, sets, requesters, remoteMeta) } func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { @@ -651,14 +650,14 @@ func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) } } -func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) error { +func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq) error { transformer, err := removeReq.set.MakeTransformer() if err != nil { return err } if !removeReq.isDirEmpty && !removeReq.isRemovedFromIRODS { - err = s.removeDirFromIRODS(removeReq.path, baton, transformer) + err = put.RemoveDirFromIRODS(s.storageHandler, removeReq.path, transformer) if err != nil { return err } @@ -674,21 +673,6 @@ func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq, baton *put.Baton) return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) } -func (s *Server) removeDirFromIRODS(path string, baton *put.Baton, - transformer put.PathTransformer) error { - rpath, err := transformer(path) - if err != nil { - return err - } - - err = baton.RemoveDirFromIRODS(rpath) - if err != nil { - return err - } - - return nil -} - // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id // from the URL parameter if Requester matches logged-in username. func (s *Server) bindPathsAndValidateSet(c *gin.Context) (string, []string, bool) { From 540463dc57aeb4f8066e789c676ed3e35479fe51 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 14 Feb 2025 16:49:06 +0000 Subject: [PATCH 27/35] Store removeReq in db incase server shutdown WIP --- main_test.go | 17 ++++++++++++-- server/server.go | 21 +++++++++++------ server/setdb.go | 46 +++++++++++++++++++++++++++++------- set/db.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 19 deletions(-) diff --git a/main_test.go b/main_test.go index 9eee7c5..8928fc3 100644 --- a/main_test.go +++ b/main_test.go @@ -1953,7 +1953,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - Convey("Given a server", t, func() { + FocusConvey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1993,7 +1993,7 @@ func TestRemove(t *testing.T) { 1, fmt.Sprintf("set with that id does not exist [%s]", invalidSetName)) }) - Convey("And an added set with files and folders", func() { + FocusConvey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -2147,6 +2147,19 @@ func TestRemove(t *testing.T) { 0, dir1+" => ") }) + FocusConvey("if the server dies during removal, the removal will continue upon server startup", func() { + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--items", tempTestFileOfPaths.Name()) + + So(exitCode, ShouldEqual, 0) + + err = s.Shutdown() + So(err, ShouldBeNil) + + s.startServer() + + s.waitForStatus(setName, "Removal status: 8 / 8 objects removed", 5*time.Second) + }) + Convey("Remove removes the provided file from iRODS", func() { output, erro := exec.Command("ils", remotePath).CombinedOutput() So(erro, ShouldBeNil) diff --git a/server/server.go b/server/server.go index 64599e9..8c0ab8f 100644 --- a/server/server.go +++ b/server/server.go @@ -220,6 +220,8 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { break } + fmt.Println(removeReq, removeReq.set) + err = s.removeRequestFromIRODSandDB(&removeReq) beenReleased := s.handleErrorOrReleaseItem(item, removeReq, err) @@ -227,8 +229,13 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { continue } - errr := s.removeQueue.Remove(context.Background(), removeReq.key()) - if errr != nil { + err = s.db.DeleteRemoveEntry(removeReq.key()) + if err != nil { + s.Logger.Printf("%s", err.Error()) + } + + err = s.removeQueue.Remove(context.Background(), removeReq.key()) + if err != nil { s.Logger.Printf("%s", err.Error()) } } @@ -237,18 +244,18 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { // reserveRemoveRequest reserves an item from the given reserve group from the // remove queue and converts it to a removeRequest. Returns nil and no error if // the queue is empty. -func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeReq, error) { +func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, RemoveReq, error) { item, err := s.removeQueue.Reserve(reserveGroup, retryDelay+2*time.Second) if err != nil { qerr, ok := err.(queue.Error) //nolint:errorlint if ok && errors.Is(qerr.Err, queue.ErrNothingReady) { - return nil, removeReq{}, err + return nil, RemoveReq{}, err } s.Logger.Printf("%s", err.Error()) } - remReq, ok := item.Data().(removeReq) + remReq, ok := item.Data().(RemoveReq) if !ok { s.Logger.Printf("Invalid data type in remove queue") } @@ -256,7 +263,7 @@ func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, removeR return item, remReq, nil } -func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq) error { +func (s *Server) removeRequestFromIRODSandDB(removeReq *RemoveReq) error { if removeReq.isDir { return s.removeDirFromIRODSandDB(removeReq) } @@ -267,7 +274,7 @@ func (s *Server) removeRequestFromIRODSandDB(removeReq *removeReq) error { // handleErrorOrReleaseItem returns immediately if there was no error. Otherwise // it releases the item with updated data from a queue, provided it has attempts // left, or sets the error on the set. Returns whether the item was released. -func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq removeReq, err error) bool { +func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq RemoveReq, err error) bool { if err == nil { return false } diff --git a/server/setdb.go b/server/setdb.go index e83c6ab..0ba5b7b 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -433,12 +433,17 @@ func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGrou defs := makeItemsDefsFromFilePaths(set, reserveGroup, paths) - _, _, err := s.removeQueue.AddMany(context.Background(), defs) + err := s.db.SetRemoveEntries(defs) + if err != nil { + return err + } + + _, _, err = s.removeQueue.AddMany(context.Background(), defs) return err } -type removeReq struct { +type RemoveReq struct { path string set *set.Set isDir bool @@ -446,7 +451,7 @@ type removeReq struct { isRemovedFromIRODS bool } -func (rq removeReq) key() string { +func (rq RemoveReq) key() string { return strings.Join([]string{rq.set.ID(), rq.path}, ":") } @@ -454,7 +459,7 @@ func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []strin defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { - rq := removeReq{path: path, set: set, isDir: false} + rq := RemoveReq{path: path, set: set, isDir: false} defs[i] = &queue.ItemDef{ Key: rq.key(), @@ -479,12 +484,14 @@ func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup fileDefs := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) - _, _, err = s.removeQueue.AddMany(context.Background(), fileDefs) + defs := append(fileDefs, dirDefs...) //nolint:gocritic + + err = s.db.SetRemoveEntries(defs) if err != nil { return nil, err } - _, _, err = s.removeQueue.AddMany(context.Background(), dirDefs) + _, _, err = s.removeQueue.AddMany(context.Background(), defs) return filepaths, err } @@ -496,7 +503,7 @@ func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { - rq := removeReq{path: path, set: set, isDir: true} + rq := RemoveReq{path: path, set: set, isDir: true} dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) if err != nil { @@ -520,7 +527,7 @@ func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, return filepaths, defs, nil } -func (s *Server) removeFileFromIRODSandDB(removeReq *removeReq) error { +func (s *Server) removeFileFromIRODSandDB(removeReq *RemoveReq) error { entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) if err != nil { return err @@ -650,7 +657,7 @@ func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) } } -func (s *Server) removeDirFromIRODSandDB(removeReq *removeReq) error { +func (s *Server) removeDirFromIRODSandDB(removeReq *RemoveReq) error { transformer, err := removeReq.set.MakeTransformer() if err != nil { return err @@ -1238,11 +1245,32 @@ func (s *Server) recoverQueue() error { } } + err = s.recoverRemoveQueue() + if err != nil { + return err + } + s.sendSlackMessage(slack.Success, "recovery completed") return nil } +func (s *Server) recoverRemoveQueue() error { + defs, err := s.db.GetRemoveEntries() + if err != nil { + return err + } + + _, _, err = s.removeQueue.AddMany(context.Background(), defs) + if err != nil { + return err + } + + go s.handleRemoveRequests(set.RemoveReserveGroup) + + return nil +} + // recoverSet checks the status of the given Set and recovers its state as // appropriate: discover it if it was previously in the middle of being // discovered; adds its remaining upload requests if it was previously in the diff --git a/set/db.go b/set/db.go index 00eaf43..d1e52ba 100644 --- a/set/db.go +++ b/set/db.go @@ -37,6 +37,7 @@ import ( "syscall" "time" + "github.com/VertebrateResequencing/wr/queue" "github.com/gammazero/workerpool" "github.com/ugorji/go/codec" "github.com/wtsi-hgi/ibackup/put" @@ -73,6 +74,7 @@ const ( fileBucket = subBucketPrefix + "files" dirBucket = subBucketPrefix + "dirs" discoveredBucket = subBucketPrefix + "discovered" + removeBucket = "remove" failedBucket = "failed" dbOpenMode = 0600 separator = ":!:" @@ -85,6 +87,8 @@ const ( // workerPoolSizeFiles is the max number of concurrent file stats we'll do // during discovery. workerPoolSizeFiles = 16 + + RemoveReserveGroup = "removeRecovery" ) // DBRO is the read-only component of the DB struct. @@ -178,7 +182,7 @@ func initDB(path string) (*bolt.DB, error) { } err = boltDB.Update(func(tx *bolt.Tx) error { - for _, bucket := range [...]string{setsBucket, failedBucket, inodeBucket, + for _, bucket := range [...]string{setsBucket, failedBucket, inodeBucket, removeBucket, userToSetBucket, userToSetBucket, transformerToIDBucket, transformerFromIDBucket} { if _, errc := tx.CreateBucketIfNotExists([]byte(bucket)); errc != nil { return errc @@ -422,6 +426,51 @@ func (d *DB) setEntries(setID string, dirents []*Dirent, bucketName string) erro }) } +// TODO: we cant use itemDef cuz it doesnt encode the data. we cant use removereq cuz it cant be imported. + +func (d *DB) SetRemoveEntries(entries []*queue.ItemDef) error { + return d.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(removeBucket)) + + for _, entry := range entries { + err := b.Put([]byte(entry.Key), d.encodeToBytes(entry.Data)) + if err != nil { + return err + } + } + + return nil + }) +} + +func (d *DB) DeleteRemoveEntry(key string) error { + return d.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(removeBucket)) + + return b.Delete([]byte(key)) + }) +} + +func (d *DB) GetRemoveEntries() ([]*queue.ItemDef, error) { + var entries []*queue.ItemDef + + err := d.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(removeBucket)) + + return b.ForEach(func(k, v []byte) error { + def := d.decodeRemoveReq(v) + + def.ReserveGroup = RemoveReserveGroup + + entries = append(entries, def) + + return nil + }) + }) + + return entries, err +} + // getAndDeleteExistingEntries gets existing entries in the given sub bucket // of the setsBucket, then deletes and recreates the sub bucket. Returns the // empty sub bucket and any old values. @@ -1152,6 +1201,16 @@ func (d *DBRO) decodeEntry(v []byte) *Entry { return entry } +func (d *DBRO) decodeRemoveReq(v []byte) *queue.ItemDef { + dec := codec.NewDecoderBytes(v, d.ch) + + var def *queue.ItemDef + + dec.MustDecode(&def) + + return def +} + // GetFailedEntries returns up to 10 of the file entries for the given set (both // SetFileEntries and SetDiscoveredEntries) that have a failed status. Also // returns the number of failed entries that were not returned. From 345cfbb65387b2cf58e6a520eca9ed5439ab75ea Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Mon, 17 Feb 2025 13:25:29 +0000 Subject: [PATCH 28/35] Allow removal to recover if server goes down --- main_test.go | 26 +++++++++++-- server/server.go | 34 +++++++++++++---- server/setdb.go | 97 +++++++++++++++++++++++++++++------------------- set/db.go | 39 ++++++++++--------- 4 files changed, 128 insertions(+), 68 deletions(-) diff --git a/main_test.go b/main_test.go index 8928fc3..c29b225 100644 --- a/main_test.go +++ b/main_test.go @@ -1953,7 +1953,7 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { - FocusConvey("Given a server", t, func() { + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { SkipConvey("skipping iRODS backup test since IBACKUP_TEST_COLLECTION not set", func() {}) @@ -1993,7 +1993,7 @@ func TestRemove(t *testing.T) { 1, fmt.Sprintf("set with that id does not exist [%s]", invalidSetName)) }) - FocusConvey("And an added set with files and folders", func() { + Convey("And an added set with files and folders", func() { dir := t.TempDir() linkPath := filepath.Join(path, "link") @@ -2147,8 +2147,26 @@ func TestRemove(t *testing.T) { 0, dir1+" => ") }) - FocusConvey("if the server dies during removal, the removal will continue upon server startup", func() { - exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--items", tempTestFileOfPaths.Name()) + Convey("if the server dies during removal, the removal will continue upon server startup", func() { + tempTestFileOfPathsToRemove1, errt := os.CreateTemp(dir, "testFileSet") + So(errt, ShouldBeNil) + + _, err = io.WriteString(tempTestFileOfPathsToRemove1, + fmt.Sprintf("%s\n%s", file1, dir1)) + So(err, ShouldBeNil) + + tempTestFileOfPathsToRemove2, errt := os.CreateTemp(dir, "testFileSet") + So(errt, ShouldBeNil) + + _, err = io.WriteString(tempTestFileOfPathsToRemove2, + fmt.Sprintf("%s\n%s\n%s\n%s\n%s", file2, file4, dir2, linkPath, symPath)) + So(err, ShouldBeNil) + + exitCode, _ := s.runBinary(t, "remove", "--name", setName, "--items", tempTestFileOfPathsToRemove1.Name()) + + So(exitCode, ShouldEqual, 0) + + exitCode, _ = s.runBinary(t, "remove", "--name", setName, "--items", tempTestFileOfPathsToRemove2.Name()) So(exitCode, ShouldEqual, 0) diff --git a/server/server.go b/server/server.go index 8c0ab8f..f19bd4f 100644 --- a/server/server.go +++ b/server/server.go @@ -40,6 +40,7 @@ import ( jqs "github.com/VertebrateResequencing/wr/jobqueue/scheduler" "github.com/VertebrateResequencing/wr/queue" "github.com/gammazero/workerpool" + "github.com/goccy/go-json" "github.com/inconshreveable/log15" gas "github.com/wtsi-hgi/go-authserver" "github.com/wtsi-hgi/ibackup/put" @@ -49,7 +50,8 @@ import ( ) const ( - ErrNoLogger = gas.Error("a http logger must be configured") + ErrNoLogger = gas.Error("a http logger must be configured") + ErrFailedByteConversion = gas.Error("could not convert data type to []byte") // workerPoolSizeDir is the max number of directory walks we'll do // concurrently during discovery; each of those walks in turn operate on 16 @@ -220,8 +222,6 @@ func (s *Server) handleRemoveRequests(reserveGroup string) { break } - fmt.Println(removeReq, removeReq.set) - err = s.removeRequestFromIRODSandDB(&removeReq) beenReleased := s.handleErrorOrReleaseItem(item, removeReq, err) @@ -255,16 +255,25 @@ func (s *Server) reserveRemoveRequest(reserveGroup string) (*queue.Item, RemoveR s.Logger.Printf("%s", err.Error()) } - remReq, ok := item.Data().(RemoveReq) + var remReq RemoveReq + + data, ok := item.Data().([]byte) if !ok { s.Logger.Printf("Invalid data type in remove queue") + + return nil, RemoveReq{}, ErrFailedByteConversion + } + + err = json.Unmarshal(data, &remReq) + if err != nil { + s.Logger.Printf("%s", err.Error()) } - return item, remReq, nil + return item, remReq, err } func (s *Server) removeRequestFromIRODSandDB(removeReq *RemoveReq) error { - if removeReq.isDir { + if removeReq.IsDir { return s.removeDirFromIRODSandDB(removeReq) } @@ -280,7 +289,7 @@ func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq RemoveReq, } if item.Stats().Releases >= uint32(jobRetries) { - errs := s.db.SetError(removeReq.set.ID(), "Error when removing: "+err.Error()) + errs := s.db.SetError(removeReq.Set.ID(), "Error when removing: "+err.Error()) if errs != nil { s.Logger.Printf("Could not put error on set due to: %s\nError was: %s\n", errs.Error(), err.Error()) } @@ -288,7 +297,7 @@ func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq RemoveReq, return false } - item.SetData(removeReq) + s.setRemoveReqOnItemDef(item, removeReq) err = s.removeQueue.SetDelay(removeReq.key(), retryDelay) if err != nil { @@ -303,6 +312,15 @@ func (s *Server) handleErrorOrReleaseItem(item *queue.Item, removeReq RemoveReq, return true } +func (s *Server) setRemoveReqOnItemDef(item *queue.Item, removeReq RemoveReq) { + rqStr, err := json.Marshal(removeReq) + if err != nil { + s.Logger.Printf("%s", err.Error()) + } + + item.SetData(rqStr) +} + // rac is our queue's ready added callback which will get all ready put Requests // and ensure there are enough put jobs added to wr. // diff --git a/server/setdb.go b/server/setdb.go index 0ba5b7b..9a76d27 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -27,6 +27,7 @@ package server import ( "context" + "encoding/json" "errors" "fmt" "math" @@ -431,9 +432,12 @@ func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGrou return nil } - defs := makeItemsDefsFromFilePaths(set, reserveGroup, paths) + defs, err := makeItemsDefsFromFilePaths(set, reserveGroup, paths) + if err != nil { + return err + } - err := s.db.SetRemoveEntries(defs) + err = s.db.SetRemoveEntries(defs) if err != nil { return err } @@ -444,32 +448,46 @@ func (s *Server) submitFilesForRemoval(set *set.Set, paths []string, reserveGrou } type RemoveReq struct { - path string - set *set.Set - isDir bool - isDirEmpty bool - isRemovedFromIRODS bool + Path string + Set *set.Set + IsDir bool + IsDirEmpty bool + IsRemovedFromIRODS bool } func (rq RemoveReq) key() string { - return strings.Join([]string{rq.set.ID(), rq.path}, ":") + return strings.Join([]string{rq.Set.ID(), rq.Path}, ":") } -func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []string) []*queue.ItemDef { +func makeItemsDefsFromFilePaths(set *set.Set, reserveGroup string, paths []string) ([]*queue.ItemDef, error) { defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { - rq := RemoveReq{path: path, set: set, isDir: false} + rq := RemoveReq{Path: path, Set: set, IsDir: false} - defs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, - ReserveGroup: reserveGroup, + def, err := buildRemoveItemDef(rq, reserveGroup) + if err != nil { + return nil, err } + + defs[i] = def } - return defs + return defs, nil +} + +func buildRemoveItemDef(rq RemoveReq, reserveGroup string) (*queue.ItemDef, error) { + rqStr, err := json.Marshal(rq) + if err != nil { + return nil, err + } + + return &queue.ItemDef{ + Key: rq.key(), + Data: rqStr, + TTR: ttr, + ReserveGroup: reserveGroup, + }, nil } func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup string) ([]string, error) { @@ -482,7 +500,10 @@ func (s *Server) submitDirsForRemoval(set *set.Set, paths []string, reserveGroup return nil, err } - fileDefs := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) + fileDefs, err := makeItemsDefsFromFilePaths(set, reserveGroup, filepaths) + if err != nil { + return nil, err + } defs := append(fileDefs, dirDefs...) //nolint:gocritic @@ -503,7 +524,7 @@ func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, defs := make([]*queue.ItemDef, len(paths)) for i, path := range paths { - rq := RemoveReq{path: path, set: set, isDir: true} + rq := RemoveReq{Path: path, Set: set, IsDir: true} dirFilepaths, err := s.db.GetFilesInDir(set.ID(), path) if err != nil { @@ -511,50 +532,50 @@ func (s *Server) makeItemsDefsAndFilePathsFromDirPaths(set *set.Set, } if len(dirFilepaths) == 0 { - rq.isDirEmpty = true + rq.IsDirEmpty = true } filepaths = append(filepaths, dirFilepaths...) - defs[i] = &queue.ItemDef{ - Key: rq.key(), - Data: rq, - TTR: ttr, - ReserveGroup: reserveGroup, + def, err := buildRemoveItemDef(rq, reserveGroup) + if err != nil { + return nil, nil, err } + + defs[i] = def } return filepaths, defs, nil } func (s *Server) removeFileFromIRODSandDB(removeReq *RemoveReq) error { - entry, err := s.db.GetFileEntryForSet(removeReq.set.ID(), removeReq.path) + entry, err := s.db.GetFileEntryForSet(removeReq.Set.ID(), removeReq.Path) if err != nil { return err } - transformer, err := removeReq.set.MakeTransformer() + transformer, err := removeReq.Set.MakeTransformer() if err != nil { return err } - if !removeReq.isRemovedFromIRODS { - err = s.removeFileFromIRODS(removeReq.set, removeReq.path, transformer) + if !removeReq.IsRemovedFromIRODS { + err = s.removeFileFromIRODS(removeReq.Set, removeReq.Path, transformer) if err != nil { - s.setErrorOnEntry(entry, removeReq.set.ID(), removeReq.path, err) + s.setErrorOnEntry(entry, removeReq.Set.ID(), removeReq.Path, err) return err } - removeReq.isRemovedFromIRODS = true + removeReq.IsRemovedFromIRODS = true } - err = s.db.RemoveFileEntry(removeReq.set.ID(), removeReq.path) + err = s.db.RemoveFileEntry(removeReq.Set.ID(), removeReq.Path) if err != nil { return err } - return s.db.UpdateBasedOnRemovedEntry(removeReq.set.ID(), entry) + return s.db.UpdateBasedOnRemovedEntry(removeReq.Set.ID(), entry) } func (s *Server) removeFileFromIRODS(set *set.Set, path string, transformer put.PathTransformer) error { @@ -658,26 +679,26 @@ func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) } func (s *Server) removeDirFromIRODSandDB(removeReq *RemoveReq) error { - transformer, err := removeReq.set.MakeTransformer() + transformer, err := removeReq.Set.MakeTransformer() if err != nil { return err } - if !removeReq.isDirEmpty && !removeReq.isRemovedFromIRODS { - err = put.RemoveDirFromIRODS(s.storageHandler, removeReq.path, transformer) + if !removeReq.IsDirEmpty && !removeReq.IsRemovedFromIRODS { + err = put.RemoveDirFromIRODS(s.storageHandler, removeReq.Path, transformer) if err != nil { return err } - removeReq.isRemovedFromIRODS = true + removeReq.IsRemovedFromIRODS = true } - err = s.db.RemoveDirEntry(removeReq.set.ID(), removeReq.path) + err = s.db.RemoveDirEntry(removeReq.Set.ID(), removeReq.Path) if err != nil { return err } - return s.db.IncrementSetTotalRemoved(removeReq.set.ID()) + return s.db.IncrementSetTotalRemoved(removeReq.Set.ID()) } // bindPathsAndValidateSet gets the paths out of the JSON body, and the set id diff --git a/set/db.go b/set/db.go index d1e52ba..d704ca4 100644 --- a/set/db.go +++ b/set/db.go @@ -426,14 +426,14 @@ func (d *DB) setEntries(setID string, dirents []*Dirent, bucketName string) erro }) } -// TODO: we cant use itemDef cuz it doesnt encode the data. we cant use removereq cuz it cant be imported. - +// SetRemoveEntries writes a list of itemDefs containing remove requests into +// the database. func (d *DB) SetRemoveEntries(entries []*queue.ItemDef) error { return d.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(removeBucket)) for _, entry := range entries { - err := b.Put([]byte(entry.Key), d.encodeToBytes(entry.Data)) + err := b.Put([]byte(entry.Key), d.encodeToBytes(entry)) if err != nil { return err } @@ -443,6 +443,7 @@ func (d *DB) SetRemoveEntries(entries []*queue.ItemDef) error { }) } +// DeleteRemoveEntry removes the itemDef for a remove request from the database. func (d *DB) DeleteRemoveEntry(key string) error { return d.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(removeBucket)) @@ -451,24 +452,36 @@ func (d *DB) DeleteRemoveEntry(key string) error { }) } +// GetRemoveEntries returns all objects from the remove bucket in the database. +// It redefines the reserve group on each object to be the same. func (d *DB) GetRemoveEntries() ([]*queue.ItemDef, error) { - var entries []*queue.ItemDef + var defs []*queue.ItemDef err := d.db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(removeBucket)) - return b.ForEach(func(k, v []byte) error { - def := d.decodeRemoveReq(v) + return b.ForEach(func(_, v []byte) error { + def := d.decodeItemDef(v) def.ReserveGroup = RemoveReserveGroup - entries = append(entries, def) + defs = append(defs, def) return nil }) }) - return entries, err + return defs, err +} + +func (d *DBRO) decodeItemDef(v []byte) *queue.ItemDef { + dec := codec.NewDecoderBytes(v, d.ch) + + var def *queue.ItemDef + + dec.MustDecode(&def) + + return def } // getAndDeleteExistingEntries gets existing entries in the given sub bucket @@ -1201,16 +1214,6 @@ func (d *DBRO) decodeEntry(v []byte) *Entry { return entry } -func (d *DBRO) decodeRemoveReq(v []byte) *queue.ItemDef { - dec := codec.NewDecoderBytes(v, d.ch) - - var def *queue.ItemDef - - dec.MustDecode(&def) - - return def -} - // GetFailedEntries returns up to 10 of the file entries for the given set (both // SetFileEntries and SetDiscoveredEntries) that have a failed status. Also // returns the number of failed entries that were not returned. From 9ae8992753ace965fab7d79a9d619171be209500 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 18 Feb 2025 11:50:40 +0000 Subject: [PATCH 29/35] Small PR fixes, and optimisation of GetFilesInDir --- put/baton.go | 6 +++--- put/mock.go | 4 ++-- put/put.go | 8 ++++---- put/put_test.go | 8 ++++---- put/remove.go | 6 +++--- server/server.go | 2 +- set/db.go | 38 ++++++++++++++++++++++++++++---------- set/set_test.go | 13 +++++++++++++ 8 files changed, 58 insertions(+), 27 deletions(-) diff --git a/put/baton.go b/put/baton.go index 40f5913..9c088af 100644 --- a/put/baton.go +++ b/put/baton.go @@ -502,7 +502,7 @@ func metaToAVUs(meta map[string]string) []ex.AVU { return avus } -func (b *Baton) removeFile(path string) error { +func (b *Baton) RemoveFile(path string) error { it := remotePathToRodsItem(path) err := timeoutOp(func() error { @@ -514,7 +514,7 @@ func (b *Baton) removeFile(path string) error { return err } -func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) { +func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { it := &ex.RodsItem{ IPath: dirToSearch, IAVUs: metaToAVUs(meta), @@ -539,7 +539,7 @@ func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string, return paths, err } -// RemoveDirFromIRODS removes the given directory from iRODS given it is empty. +// RemoveDir removes the given directory from iRODS given it is empty. func (b *Baton) RemoveDir(path string) error { it := &ex.RodsItem{ IPath: path, diff --git a/put/mock.go b/put/mock.go index fad14e8..c24f648 100644 --- a/put/mock.go +++ b/put/mock.go @@ -258,13 +258,13 @@ func (l *LocalHandler) RemoveDir(path string) error { return os.Remove(path) } -func (l *LocalHandler) removeFile(path string) error { +func (l *LocalHandler) RemoveFile(path string) error { delete(l.meta, path) return os.Remove(path) } -func (l *LocalHandler) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) { +func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { var objects []string for path, pathMeta := range l.meta { diff --git a/put/put.go b/put/put.go index d9cb459..6589728 100644 --- a/put/put.go +++ b/put/put.go @@ -124,11 +124,11 @@ type Handler interface { // RemoveDir deletes a given empty folder RemoveDir(path string) error - // removeFile deletes a given file - removeFile(path string) error + // RemoveFile deletes a given file + RemoveFile(path string) error - // queryMeta return paths to all objects with given metadata - queryMeta(dirToSearch string, meta map[string]string) ([]string, error) + // QueryMeta return paths to all objects with given metadata + QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) } // FileReadTester is a function that attempts to open and read the given path, diff --git a/put/put_test.go b/put/put_test.go index 6dfa875..102c1a9 100644 --- a/put/put_test.go +++ b/put/put_test.go @@ -173,7 +173,7 @@ func TestPutMock(t *testing.T) { Convey("RemoveFile removes a file", func() { filePath := requests[0].Remote - err = lh.removeFile(filePath) + err = lh.RemoveFile(filePath) So(err, ShouldBeNil) _, err = os.Stat(filePath) @@ -193,18 +193,18 @@ func TestPutMock(t *testing.T) { }) Convey("queryMeta returns all paths with matching metadata", func() { - paths, errq := lh.queryMeta("", map[string]string{"a": "1"}) + paths, errq := lh.QueryMeta("", map[string]string{"a": "1"}) So(errq, ShouldBeNil) So(len(paths), ShouldEqual, 5) - paths, err = lh.queryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester}) + paths, err = lh.QueryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester}) So(err, ShouldBeNil) So(len(paths), ShouldEqual, 1) Convey("queryMeta only returns paths in the provided scope", func() { - paths, errq := lh.queryMeta(expectedCollections[1], map[string]string{"a": "1"}) + paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"}) So(errq, ShouldBeNil) So(len(paths), ShouldEqual, 2) diff --git a/put/remove.go b/put/remove.go index 0761538..79f9f11 100644 --- a/put/remove.go +++ b/put/remove.go @@ -67,7 +67,7 @@ func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path // same file, if not, it removes the file. func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer PathTransformer, meta map[string]string) error { - err := handler.removeFile(path) + err := handler.RemoveFile(path) if err != nil { return err } @@ -81,7 +81,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return err } - items, err := handler.queryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) + items, err := handler.QueryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) if err != nil { return err } @@ -90,7 +90,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return nil } - return handler.removeFile(meta[MetaKeyRemoteHardlink]) + return handler.RemoveFile(meta[MetaKeyRemoteHardlink]) } // RemoveDirFromIRODS removes the remote path of a given directory from iRODS. diff --git a/server/server.go b/server/server.go index f19bd4f..f9f2cde 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,7 @@ package server import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -40,7 +41,6 @@ import ( jqs "github.com/VertebrateResequencing/wr/jobqueue/scheduler" "github.com/VertebrateResequencing/wr/queue" "github.com/gammazero/workerpool" - "github.com/goccy/go-json" "github.com/inconshreveable/log15" gas "github.com/wtsi-hgi/go-authserver" "github.com/wtsi-hgi/ibackup/put" diff --git a/set/db.go b/set/db.go index d704ca4..9e7a083 100644 --- a/set/db.go +++ b/set/db.go @@ -368,23 +368,41 @@ func (d *DB) RemoveDirEntry(setID string, path string) error { // GetFilesInDir returns all file paths from inside the given directory (and all // nested inside) for the given set using the db. -func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) { - var filepaths []string - - entries, err := d.getEntries(setID, discoveredBucket) +func (d *DBRO) GetFilesInDir(setID string, dirpath string) ([]string, error) { + filepaths, err := d.getPathsWithPrefix(setID, discoveredBucket, dirpath) if err != nil { return nil, err } - for _, entry := range entries { - path := entry.Path + return filepaths, nil +} + +// getPathsWithPrefix returns all the filepaths for the given set from the given sub +// bucket prefix, that have the given prefix. +func (d *DBRO) getPathsWithPrefix(setID, bucketName, prefix string) ([]string, error) { + var entries []string - if strings.HasPrefix(path, dirpath) { - filepaths = append(filepaths, path) + err := d.db.View(func(tx *bolt.Tx) error { + subBucketName := []byte(bucketName + separator + setID) + setsBucket := tx.Bucket([]byte(setsBucket)) + + entriesBucket := setsBucket.Bucket(subBucketName) + if entriesBucket == nil { + return nil } - } - return filepaths, nil + c := entriesBucket.Cursor() + + prefix := []byte(prefix) + + for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() { + entries = append(entries, string(k)) + } + + return nil + }) + + return entries, err } // SetFileEntries sets the file paths for the given backup set. Only supply diff --git a/set/set_test.go b/set/set_test.go index 22d5ae6..70e37b9 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -357,6 +357,19 @@ func TestSetDB(t *testing.T) { err = db.SetFileEntries(set2.ID(), []string{"/a/b.txt", "/c/k.txt"}) So(err, ShouldBeNil) + Convey("You can get all paths containing a prefix", func() { + err = db.SetFileEntries(set2.ID(), []string{"/a/a/j.txt", "/a/b/c/k.txt", + "/a/b/c/l.txt", "/a/b/d/m.txt", "/c/n.txt"}) + So(err, ShouldBeNil) + + files, errp := db.getPathsWithPrefix(set2.ID(), fileBucket, "/a/b/c/") + So(errp, ShouldBeNil) + + So(len(files), ShouldEqual, 2) + So(files, ShouldContain, "/a/b/c/k.txt") + So(files, ShouldContain, "/a/b/c/l.txt") + }) + Convey("Then remove files and dirs from the sets", func() { err = db.removeEntry(set.ID(), "/a/b.txt", fileBucket) So(err, ShouldBeNil) From 7d3b3893b1ca221f84602ddecbe319ccc81bf348 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Tue, 18 Feb 2025 15:04:21 +0000 Subject: [PATCH 30/35] Move removal functionality from put pkg to remove pkg --- cmd/remove.go | 6 +- cmd/server.go | 4 +- main_test.go | 4 +- put/baton.go | 139 +++++++-------------------------- put/baton_test.go | 4 +- put/mock.go | 57 ++------------ put/put.go | 11 +-- put/put_test.go | 80 +++++-------------- remove/baton.go | 137 ++++++++++++++++++++++++++++++++ remove/mock.go | 89 +++++++++++++++++++++ {put => remove}/remove.go | 39 +++++++--- remove/remove_test.go | 159 ++++++++++++++++++++++++++++++++++++++ server/server.go | 17 ++-- server/setdb.go | 5 +- 14 files changed, 493 insertions(+), 258 deletions(-) create mode 100644 remove/baton.go create mode 100644 remove/mock.go rename {put => remove}/remove.go (71%) create mode 100644 remove/remove_test.go diff --git a/cmd/remove.go b/cmd/remove.go index 9bc7a12..7b26046 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -94,7 +94,7 @@ var removeCmd = &cobra.Command{ paths = append(paths, removePath) } - remove(client, removeUser, removeName, paths) + handleRemove(client, removeUser, removeName, paths) }, } @@ -117,8 +117,8 @@ func init() { } } -// remove does the main job of sending the set, files and dirs to the server. -func remove(client *server.Client, user, name string, paths []string) { +// handleRemove does the main job of sending the set, files and dirs to the server. +func handleRemove(client *server.Client, user, name string, paths []string) { sets := getSetByName(client, user, name) err := client.RemoveFilesAndDirs(sets[0].ID(), paths) diff --git a/cmd/server.go b/cmd/server.go index dcdecb5..5ed797d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -39,7 +39,7 @@ import ( "github.com/inconshreveable/log15" "github.com/spf13/cobra" gas "github.com/wtsi-hgi/go-authserver" - "github.com/wtsi-hgi/ibackup/put" + "github.com/wtsi-hgi/ibackup/remove" "github.com/wtsi-hgi/ibackup/server" "github.com/wtsi-hgi/ibackup/set" "github.com/wtsi-hgi/ibackup/slack" @@ -193,7 +193,7 @@ database that you've made, to investigate. die("slack_debounce period must be positive, not: %d", serverSlackDebouncePeriod) } - handler, errb := put.GetBatonHandlerWithMetaClient() + handler, errb := remove.GetBatonHandlerWithMetaClient() if errb != nil { die("failed to get baton handler: %s", errb) } diff --git a/main_test.go b/main_test.go index c29b225..3700bec 100644 --- a/main_test.go +++ b/main_test.go @@ -1953,6 +1953,8 @@ func confirmFileContents(file, expectedContents string) { } func TestRemove(t *testing.T) { + resetIRODS() + Convey("Given a server", t, func() { remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") if remotePath == "" { @@ -1970,8 +1972,6 @@ func TestRemove(t *testing.T) { return } - resetIRODS() - dir := t.TempDir() s := new(TestServer) s.prepareFilePaths(dir) diff --git a/put/baton.go b/put/baton.go index 9c088af..63ed556 100644 --- a/put/baton.go +++ b/put/baton.go @@ -30,7 +30,6 @@ package put import ( "context" "errors" - "fmt" "os" "path/filepath" "strings" @@ -71,9 +70,9 @@ type Baton struct { collCh chan string collErrCh chan error collMu sync.Mutex - putMetaPool *ex.ClientPool + PutMetaPool *ex.ClientPool putClient *ex.Client - metaClient *ex.Client + MetaClient *ex.Client } // GetBatonHandler returns a Handler that uses Baton to interact with iRODS. If @@ -86,34 +85,6 @@ func GetBatonHandler() (*Baton, error) { return &Baton{}, err } -// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact -// with iRODS and contains a meta client for interacting with metadata. If you -// don't have baton-do in your PATH, you'll get an error. -func GetBatonHandlerWithMetaClient() (*Baton, error) { - setupExtendoLogger() - - _, err := ex.FindBaton() - if err != nil { - return nil, err - } - - params := ex.DefaultClientPoolParams - params.MaxSize = 1 - pool := ex.NewClientPool(params, "") - - metaClient, err := pool.Get() - if err != nil { - return nil, fmt.Errorf("failed to get metaClient: %w", err) - } - - baton := &Baton{ - putMetaPool: pool, - metaClient: metaClient, - } - - return baton, nil -} - // setupExtendoLogger sets up a STDERR logger that the extendo library will use. // (We don't actually care about what it might log, but extendo doesn't work // without this.) @@ -238,7 +209,7 @@ func (b *Baton) getClientsFromPoolConcurrently(pool *ex.ClientPool, numClients u } func (b *Baton) ensureCollection(clientIndex int, ri ex.RodsItem) error { - err := timeoutOp(func() error { + err := TimeoutOp(func() error { _, errl := b.collClients[clientIndex].ListItem(ex.Args{}, ri) return errl @@ -254,9 +225,9 @@ func (b *Baton) ensureCollection(clientIndex int, ri ex.RodsItem) error { return b.createCollectionWithTimeoutAndRetries(clientIndex, ri) } -// timeoutOp carries out op, returning any error from it. Has a 10s timeout on +// TimeoutOp carries out op, returning any error from it. Has a 10s timeout on // running op, and will return a timeout error instead if exceeded. -func timeoutOp(op retry.Operation, path string) error { +func TimeoutOp(op retry.Operation, path string) error { errCh := make(chan error, 1) go func() { @@ -316,14 +287,14 @@ func (b *Baton) doWithTimeoutAndRetries(op retry.Operation, clientIndex int, pat // makes a new client on timeout or error. func (b *Baton) timeoutOpAndMakeNewClientOnError(op retry.Operation, clientIndex int, path string) retry.Operation { return func() error { - err := timeoutOp(op, path) + err := TimeoutOp(op, path) if err != nil { pool := ex.NewClientPool(ex.DefaultClientPoolParams, "") client, errp := pool.Get() if errp == nil { go func(oldClient *ex.Client) { - timeoutOp(func() error { //nolint:errcheck + TimeoutOp(func() error { //nolint:errcheck oldClient.StopIgnoreError() return nil @@ -344,7 +315,7 @@ func (b *Baton) getClientByIndex(clientIndex int) *ex.Client { case putClientIndex: return b.putClient case metaClientIndex: - return b.metaClient + return b.MetaClient default: return b.collClients[clientIndex] } @@ -355,7 +326,7 @@ func (b *Baton) setClientByIndex(clientIndex int, client *ex.Client) { case putClientIndex: b.putClient = client case metaClientIndex: - b.metaClient = client + b.MetaClient = client default: b.collClients[clientIndex] = client } @@ -382,9 +353,9 @@ func (b *Baton) CollectionsDone() error { return err } - b.putMetaPool = pool + b.PutMetaPool = pool b.putClient = <-clientCh - b.metaClient = <-clientCh + b.MetaClient = <-clientCh return nil } @@ -393,7 +364,7 @@ func (b *Baton) CollectionsDone() error { // errors. func (b *Baton) closeConnections(clients []*ex.Client) { for _, client := range clients { - timeoutOp(func() error { //nolint:errcheck + TimeoutOp(func() error { //nolint:errcheck client.StopIgnoreError() return nil @@ -405,9 +376,9 @@ func (b *Baton) closeConnections(clients []*ex.Client) { func (b *Baton) Stat(request *Request) (*ObjectInfo, error) { var it ex.RodsItem - err := timeoutOp(func() error { + err := TimeoutOp(func() error { var errl error - it, errl = b.metaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem(request)) + it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem(request)) return errl }, "stat failed: "+request.Remote) @@ -485,12 +456,12 @@ func (b *Baton) Put(request *Request) error { // AVUs. func requestToRodsItemWithAVUs(request *Request) *ex.RodsItem { item := requestToRodsItem(request) - item.IAVUs = metaToAVUs(request.Meta.Metadata()) + item.IAVUs = MetaToAVUs(request.Meta.Metadata()) return item } -func metaToAVUs(meta map[string]string) []ex.AVU { +func MetaToAVUs(meta map[string]string) []ex.AVU { avus := make([]ex.AVU, len(meta)) i := 0 @@ -502,65 +473,13 @@ func metaToAVUs(meta map[string]string) []ex.AVU { return avus } -func (b *Baton) RemoveFile(path string) error { - it := remotePathToRodsItem(path) - - err := timeoutOp(func() error { - _, errl := b.metaClient.RemObj(ex.Args{}, *it) - - return errl - }, "remove meta error: "+path) - - return err -} - -func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { - it := &ex.RodsItem{ - IPath: dirToSearch, - IAVUs: metaToAVUs(meta), - } - - var items []ex.RodsItem - - var err error - - err = timeoutOp(func() error { - items, err = b.metaClient.MetaQuery(ex.Args{Object: true}, *it) - - return err - }, "remove meta error: "+dirToSearch) - - paths := make([]string, len(items)) - - for i, item := range items { - paths[i] = item.IPath - } - - return paths, err -} - -// RemoveDir removes the given directory from iRODS given it is empty. -func (b *Baton) RemoveDir(path string) error { - it := &ex.RodsItem{ - IPath: path, - } - - err := timeoutOp(func() error { - _, errl := b.metaClient.RemDir(ex.Args{}, *it) - - return errl - }, "remove meta error: "+path) - - return err -} - // RemoveMeta removes the given metadata from a given object in iRODS. func (b *Baton) RemoveMeta(path string, meta map[string]string) error { - it := remotePathToRodsItem(path) - it.IAVUs = metaToAVUs(meta) + it := RemotePathToRodsItem(path) + it.IAVUs = MetaToAVUs(meta) - err := timeoutOp(func() error { - _, errl := b.metaClient.MetaRem(ex.Args{}, *it) + err := TimeoutOp(func() error { + _, errl := b.MetaClient.MetaRem(ex.Args{}, *it) return errl }, "remove meta error: "+path) @@ -570,7 +489,7 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { // GetMeta gets all the metadata for the given object in iRODS. func (b *Baton) GetMeta(path string) (map[string]string, error) { - it, err := b.metaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ + it, err := b.MetaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ IPath: filepath.Dir(path), IName: filepath.Base(path), }) @@ -578,8 +497,8 @@ func (b *Baton) GetMeta(path string) (map[string]string, error) { return rodsItemToMeta(it), err } -// remotePathToRodsItem converts a path in to an extendo RodsItem. -func remotePathToRodsItem(path string) *ex.RodsItem { +// RemotePathToRodsItem converts a path in to an extendo RodsItem. +func RemotePathToRodsItem(path string) *ex.RodsItem { return &ex.RodsItem{ IPath: filepath.Dir(path), IName: filepath.Base(path), @@ -588,11 +507,11 @@ func remotePathToRodsItem(path string) *ex.RodsItem { // AddMeta adds the given metadata to a given object in iRODS. func (b *Baton) AddMeta(path string, meta map[string]string) error { - it := remotePathToRodsItem(path) - it.IAVUs = metaToAVUs(meta) + it := RemotePathToRodsItem(path) + it.IAVUs = MetaToAVUs(meta) - err := timeoutOp(func() error { - _, errl := b.metaClient.MetaAdd(ex.Args{}, *it) + err := TimeoutOp(func() error { + _, errl := b.MetaClient.MetaAdd(ex.Args{}, *it) return errl }, "add meta error: "+path) @@ -602,9 +521,9 @@ func (b *Baton) AddMeta(path string, meta map[string]string) error { // Cleanup stops our clients and closes our client pool. func (b *Baton) Cleanup() error { - b.closeConnections(append(b.collClients, b.putClient, b.metaClient)) + b.closeConnections(append(b.collClients, b.putClient, b.MetaClient)) - b.putMetaPool.Close() + b.PutMetaPool.Close() b.collMu.Lock() defer b.collMu.Unlock() diff --git a/put/baton_test.go b/put/baton_test.go index 0cb1124..e6e23ec 100644 --- a/put/baton_test.go +++ b/put/baton_test.go @@ -158,9 +158,9 @@ func TestPutBaton(t *testing.T) { err = p.Cleanup() So(err, ShouldBeNil) - So(h.putMetaPool.IsOpen(), ShouldBeFalse) + So(h.PutMetaPool.IsOpen(), ShouldBeFalse) So(h.putClient.IsRunning(), ShouldBeFalse) - So(h.metaClient.IsRunning(), ShouldBeFalse) + So(h.MetaClient.IsRunning(), ShouldBeFalse) So(h.collClients, ShouldBeNil) }) }) diff --git a/put/mock.go b/put/mock.go index c24f648..ab1416f 100644 --- a/put/mock.go +++ b/put/mock.go @@ -29,7 +29,6 @@ import ( "io" "maps" "os" - "strings" "sync" "time" ) @@ -45,7 +44,7 @@ type LocalHandler struct { connected bool cleaned bool collections []string - meta map[string]map[string]string + Meta map[string]map[string]string statFail string putFail string putSlow string @@ -59,7 +58,7 @@ type LocalHandler struct { // Remote for any Put()s. For use during tests. func GetLocalHandler() *LocalHandler { return &LocalHandler{ - meta: make(map[string]map[string]string), + Meta: make(map[string]map[string]string), } } @@ -112,7 +111,7 @@ func (l *LocalHandler) Stat(request *Request) (*ObjectInfo, error) { l.mu.RLock() defer l.mu.RUnlock() - meta, exists := l.meta[request.Remote] + meta, exists := l.Meta[request.Remote] if !exists { meta = make(map[string]string) } else { @@ -191,7 +190,7 @@ func (l *LocalHandler) RemoveMeta(path string, meta map[string]string) error { l.mu.Lock() defer l.mu.Unlock() - pathMeta, exists := l.meta[path] + pathMeta, exists := l.Meta[path] if !exists { return nil } @@ -213,10 +212,10 @@ func (l *LocalHandler) AddMeta(path string, meta map[string]string) error { l.mu.Lock() defer l.mu.Unlock() - pathMeta, exists := l.meta[path] + pathMeta, exists := l.Meta[path] if !exists { pathMeta = make(map[string]string) - l.meta[path] = pathMeta + l.Meta[path] = pathMeta } for key, val := range meta { @@ -236,11 +235,11 @@ func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { l.mu.Lock() defer l.mu.Unlock() - if l.meta == nil { + if l.Meta == nil { return make(map[string]string), nil } - currentMeta, exists := l.meta[path] + currentMeta, exists := l.Meta[path] if !exists { currentMeta = make(map[string]string) } @@ -253,43 +252,3 @@ func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { return meta, nil } - -func (l *LocalHandler) RemoveDir(path string) error { - return os.Remove(path) -} - -func (l *LocalHandler) RemoveFile(path string) error { - delete(l.meta, path) - - return os.Remove(path) -} - -func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { - var objects []string - - for path, pathMeta := range l.meta { - if !strings.HasPrefix(path, dirToSearch) { - continue - } - - if doesMetaContainMeta(pathMeta, meta) { - objects = append(objects, path) - } - } - - return objects, nil -} - -func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { - valid := true - - for k, v := range targetMeta { - if sourceMeta[k] != v { - valid = false - - break - } - } - - return valid -} diff --git a/put/put.go b/put/put.go index 6589728..d5f0f8a 100644 --- a/put/put.go +++ b/put/put.go @@ -23,7 +23,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -// package put is used to interact with iRODS. +// package put is used to put files in iRODS. package put @@ -120,15 +120,6 @@ type Handler interface { Cleanup() error GetMeta(path string) (map[string]string, error) - - // RemoveDir deletes a given empty folder - RemoveDir(path string) error - - // RemoveFile deletes a given file - RemoveFile(path string) error - - // QueryMeta return paths to all objects with given metadata - QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) } // FileReadTester is a function that attempts to open and read the given path, diff --git a/put/put_test.go b/put/put_test.go index 102c1a9..c42e30b 100644 --- a/put/put_test.go +++ b/put/put_test.go @@ -79,8 +79,8 @@ func TestPutMock(t *testing.T) { So(err, ShouldBeNil) lh.mu.RLock() - So(lh.meta[request.Remote], ShouldResemble, request.Meta.LocalMeta) - checkAddedMeta(lh.meta[request.Remote]) + So(lh.Meta[request.Remote], ShouldResemble, request.Meta.LocalMeta) + checkAddedMeta(lh.Meta[request.Remote]) lh.mu.RUnlock() } @@ -111,10 +111,10 @@ func TestPutMock(t *testing.T) { case RequestStatusReplaced: replaced++ lh.mu.RLock() - So(lh.meta[request.Remote], ShouldResemble, requests[0].Meta.LocalMeta) - So(lh.meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") - So(lh.meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB") - date = lh.meta[request.Remote][MetaKeyDate] + So(lh.Meta[request.Remote], ShouldResemble, requests[0].Meta.LocalMeta) + So(lh.Meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") + So(lh.Meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB") + date = lh.Meta[request.Remote][MetaKeyDate] lh.mu.RUnlock() default: other++ @@ -157,9 +157,9 @@ func TestPutMock(t *testing.T) { request = <-srCh So(request.Status, ShouldEqual, RequestStatusUnmodified) lh.mu.RLock() - So(lh.meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") - So(lh.meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB,setC") - So(lh.meta[request.Remote][MetaKeyDate], ShouldEqual, date) + So(lh.Meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") + So(lh.Meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB,setC") + So(lh.Meta[request.Remote][MetaKeyDate], ShouldEqual, date) lh.mu.RUnlock() }) }) @@ -170,46 +170,6 @@ func TestPutMock(t *testing.T) { So(lh.cleaned, ShouldBeTrue) }) - Convey("RemoveFile removes a file", func() { - filePath := requests[0].Remote - - err = lh.RemoveFile(filePath) - So(err, ShouldBeNil) - - _, err = os.Stat(filePath) - So(err, ShouldNotBeNil) - - So(lh.meta[filePath], ShouldBeNil) - - Convey("RemoveDir removes an empty directory", func() { - dirPath := filepath.Dir(filePath) - - err = lh.RemoveDir(dirPath) - So(err, ShouldBeNil) - - _, err = os.Stat(dirPath) - So(err, ShouldNotBeNil) - }) - }) - - Convey("queryMeta returns all paths with matching metadata", func() { - paths, errq := lh.QueryMeta("", map[string]string{"a": "1"}) - So(errq, ShouldBeNil) - - So(len(paths), ShouldEqual, 5) - - paths, err = lh.QueryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester}) - So(err, ShouldBeNil) - - So(len(paths), ShouldEqual, 1) - - Convey("queryMeta only returns paths in the provided scope", func() { - paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"}) - So(errq, ShouldBeNil) - - So(len(paths), ShouldEqual, 2) - }) - }) }) Convey("Put() fails if the local files don't exist", func() { @@ -287,15 +247,15 @@ func TestPutMock(t *testing.T) { So(errs, ShouldBeNil) So(info.Size(), ShouldEqual, 2) - So(lh.meta[requests[2].Remote][setMetaKey], ShouldEqual, "a") - So(lh.meta[requests[2].Hardlink][setMetaKey], ShouldNotBeBlank) - So(lh.meta[requests[2].Remote][MetaKeyRemoteHardlink], ShouldEqual, requests[2].Hardlink) - So(lh.meta[requests[2].Remote][MetaKeyHardlink], ShouldEqual, requests[2].Local) + So(lh.Meta[requests[2].Remote][setMetaKey], ShouldEqual, "a") + So(lh.Meta[requests[2].Hardlink][setMetaKey], ShouldNotBeBlank) + So(lh.Meta[requests[2].Remote][MetaKeyRemoteHardlink], ShouldEqual, requests[2].Hardlink) + So(lh.Meta[requests[2].Remote][MetaKeyHardlink], ShouldEqual, requests[2].Local) - So(lh.meta[requests[4].Remote][setMetaKey], ShouldEqual, "b") - So(lh.meta[requests[4].Hardlink][setMetaKey], ShouldNotBeBlank) - So(lh.meta[requests[4].Remote][MetaKeyRemoteHardlink], ShouldEqual, requests[2].Hardlink) - So(lh.meta[requests[4].Remote][MetaKeyHardlink], ShouldEqual, requests[4].Local) + So(lh.Meta[requests[4].Remote][setMetaKey], ShouldEqual, "b") + So(lh.Meta[requests[4].Hardlink][setMetaKey], ShouldNotBeBlank) + So(lh.Meta[requests[4].Remote][MetaKeyRemoteHardlink], ShouldEqual, requests[2].Hardlink) + So(lh.Meta[requests[4].Remote][MetaKeyHardlink], ShouldEqual, requests[4].Local) Convey("re-uploading an unmodified hardlink does not replace remote files", func() { hardlinkMTime := info.ModTime() @@ -463,9 +423,9 @@ func TestPutMock(t *testing.T) { So(statusCounts[RequestStatusUploaded], ShouldEqual, 1) So(statusCounts[RequestStatusUnmodified], ShouldEqual, 0) - So(lh.meta[remotePath][MetaKeySets], ShouldEqual, "aSet,bSet,cSet") - So(lh.meta[remotePath]["aKey"], ShouldEqual, "cValue") - So(lh.meta[remotePath]["bKey"], ShouldEqual, "yetAnotherValue") + So(lh.Meta[remotePath][MetaKeySets], ShouldEqual, "aSet,bSet,cSet") + So(lh.Meta[remotePath]["aKey"], ShouldEqual, "cValue") + So(lh.Meta[remotePath]["bKey"], ShouldEqual, "yetAnotherValue") }) }) diff --git a/remove/baton.go b/remove/baton.go new file mode 100644 index 0000000..3772a88 --- /dev/null +++ b/remove/baton.go @@ -0,0 +1,137 @@ +/******************************************************************************* + * Copyright (c) 2022, 2023 Genome Research Ltd. + * + * Author: Sendu Bala + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +// this file implements a Handler using baton, via extendo. + +package remove + +import ( + "fmt" + "os" + + "github.com/rs/zerolog" + "github.com/wtsi-hgi/ibackup/put" + ex "github.com/wtsi-npg/extendo/v2" + logs "github.com/wtsi-npg/logshim" + "github.com/wtsi-npg/logshim-zerolog/zlog" +) + +// Baton is a Handler that uses Baton (via extendo) to interact with iRODS. +type Baton struct { + put.Baton +} + +const ( + extendoLogLevel = logs.ErrorLevel +) + +// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact +// with iRODS and contains a meta client for interacting with metadata. If you +// don't have baton-do in your PATH, you'll get an error. +func GetBatonHandlerWithMetaClient() (*Baton, error) { + setupExtendoLogger() + + _, err := ex.FindBaton() + if err != nil { + return nil, err + } + + params := ex.DefaultClientPoolParams + params.MaxSize = 1 + pool := ex.NewClientPool(params, "") + + metaClient, err := pool.Get() + if err != nil { + return nil, fmt.Errorf("failed to get metaClient: %w", err) + } + + baton := &Baton{ + Baton: put.Baton{ + PutMetaPool: pool, + MetaClient: metaClient, + }, + } + + return baton, nil +} + +// setupExtendoLogger sets up a STDERR logger that the extendo library will use. +// (We don't actually care about what it might log, but extendo doesn't work +// without this.) +func setupExtendoLogger() { + logs.InstallLogger(zlog.New(zerolog.SyncWriter(os.Stderr), extendoLogLevel)) +} + +func (b *Baton) RemoveFile(path string) error { + it := put.RemotePathToRodsItem(path) + + err := put.TimeoutOp(func() error { + _, errl := b.MetaClient.RemObj(ex.Args{}, *it) + + return errl + }, "remove file error: "+path) + + return err +} + +// RemoveDir removes the given directory from iRODS given it is empty. +func (b *Baton) RemoveDir(path string) error { + it := &ex.RodsItem{ + IPath: path, + } + + err := put.TimeoutOp(func() error { + _, errl := b.MetaClient.RemDir(ex.Args{}, *it) + + return errl + }, "remove meta error: "+path) + + return err +} + +func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { + it := &ex.RodsItem{ + IPath: dirToSearch, + IAVUs: put.MetaToAVUs(meta), + } + + var items []ex.RodsItem + + var err error + + err = put.TimeoutOp(func() error { + items, err = b.MetaClient.MetaQuery(ex.Args{Object: true}, *it) + + return err + }, "query meta error: "+dirToSearch) + + paths := make([]string, len(items)) + + for i, item := range items { + paths[i] = item.IPath + } + + return paths, err +} diff --git a/remove/mock.go b/remove/mock.go new file mode 100644 index 0000000..73c9891 --- /dev/null +++ b/remove/mock.go @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Author: Rosie Kern + * Author: Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package remove + +import ( + "os" + "strings" + + "github.com/wtsi-hgi/ibackup/put" +) + +type LocalHandler struct { + put.LocalHandler +} + +// GetLocalHandler returns a Handler that doesn't actually interact with iRODS, +// but instead simply treats "Remote" as local paths and copies from Local to +// Remote for any Put()s. For use during tests. +func GetLocalHandler() *LocalHandler { + return &LocalHandler{ + LocalHandler: put.LocalHandler{ + Meta: make(map[string]map[string]string), + }, + } +} + +func (l *LocalHandler) RemoveDir(path string) error { + return os.Remove(path) +} + +func (l *LocalHandler) RemoveFile(path string) error { + delete(l.Meta, path) + + return os.Remove(path) +} + +func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { + var objects []string + + for path, pathMeta := range l.Meta { + if !strings.HasPrefix(path, dirToSearch) { + continue + } + + if doesMetaContainMeta(pathMeta, meta) { + objects = append(objects, path) + } + } + + return objects, nil +} + +func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { + valid := true + + for k, v := range targetMeta { + if sourceMeta[k] != v { + valid = false + + break + } + } + + return valid +} diff --git a/put/remove.go b/remove/remove.go similarity index 71% rename from put/remove.go rename to remove/remove.go index 79f9f11..8589f49 100644 --- a/put/remove.go +++ b/remove/remove.go @@ -24,30 +24,47 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -package put +// package remove is used to interact with iRODS. + +package remove import ( "reflect" "strings" + + "github.com/wtsi-hgi/ibackup/put" ) +type RemoveHandler interface { + put.Handler + + // RemoveDir deletes a given empty folder + RemoveDir(path string) error + + // RemoveFile deletes a given file + RemoveFile(path string) error + + // QueryMeta return paths to all objects with given metadata + QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) +} + // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not // associated with any other sets. Otherwise it updates the iRODS metadata for // the path to not include the given set. -func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path string, +func RemovePathFromSetInIRODS(handler RemoveHandler, transformer put.PathTransformer, path string, sets, requesters []string, meta map[string]string) error { if len(sets) == 0 { return handleHardlinkAndRemoveFromIRODS(handler, path, transformer, meta) } metaToRemove := map[string]string{ - MetaKeySets: meta[MetaKeySets], - MetaKeyRequester: meta[MetaKeyRequester], + put.MetaKeySets: meta[put.MetaKeySets], + put.MetaKeyRequester: meta[put.MetaKeyRequester], } newMeta := map[string]string{ - MetaKeySets: strings.Join(sets, ","), - MetaKeyRequester: strings.Join(requesters, ","), + put.MetaKeySets: strings.Join(sets, ","), + put.MetaKeyRequester: strings.Join(requesters, ","), } if reflect.DeepEqual(metaToRemove, newMeta) { @@ -65,14 +82,14 @@ func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path // handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the // path is found to be a hardlink, it checks if there are other hardlinks to the // same file, if not, it removes the file. -func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer PathTransformer, +func handleHardlinkAndRemoveFromIRODS(handler RemoveHandler, path string, transformer put.PathTransformer, meta map[string]string) error { err := handler.RemoveFile(path) if err != nil { return err } - if meta[MetaKeyHardlink] == "" { + if meta[put.MetaKeyHardlink] == "" { return nil } @@ -81,7 +98,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return err } - items, err := handler.QueryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]}) + items, err := handler.QueryMeta(dirToSearch, map[string]string{put.MetaKeyRemoteHardlink: meta[put.MetaKeyRemoteHardlink]}) if err != nil { return err } @@ -90,11 +107,11 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return nil } - return handler.RemoveFile(meta[MetaKeyRemoteHardlink]) + return handler.RemoveFile(meta[put.MetaKeyRemoteHardlink]) } // RemoveDirFromIRODS removes the remote path of a given directory from iRODS. -func RemoveDirFromIRODS(handler Handler, path string, transformer PathTransformer) error { +func RemoveDirFromIRODS(handler RemoveHandler, path string, transformer put.PathTransformer) error { rpath, err := transformer(path) if err != nil { return err diff --git a/remove/remove_test.go b/remove/remove_test.go new file mode 100644 index 0000000..56a3542 --- /dev/null +++ b/remove/remove_test.go @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Author: Rosie Kern + * Author: Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package remove + +import ( + "os" + "path/filepath" + "strings" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "github.com/wtsi-hgi/ibackup/internal" + "github.com/wtsi-hgi/ibackup/put" +) + +const userPerms = 0700 + +func TestRemoveMock(t *testing.T) { + Convey("Given a mock RemoveHandler", t, func() { + requests, expectedCollections := makeMockRequests(t) + + lh := GetLocalHandler() + + requests[0].Requester = "John" + + for _, request := range requests { + uploadRequest(t, lh, request) + } + + Convey("RemoveFile removes a file", func() { + filePath := requests[0].Remote + + err := lh.RemoveFile(filePath) + So(err, ShouldBeNil) + + _, err = os.Stat(filePath) + So(err, ShouldNotBeNil) + + So(lh.Meta[filePath], ShouldBeNil) + + Convey("RemoveDir removes an empty directory", func() { + dirPath := filepath.Dir(filePath) + + err = lh.RemoveDir(dirPath) + So(err, ShouldBeNil) + + _, err = os.Stat(dirPath) + So(err, ShouldNotBeNil) + }) + }) + + Convey("queryMeta returns all paths with matching metadata", func() { + paths, errq := lh.QueryMeta("", map[string]string{"a": "1"}) + So(errq, ShouldBeNil) + + So(len(paths), ShouldEqual, 5) + + paths, err := lh.QueryMeta("", map[string]string{put.MetaKeyRequester: requests[0].Requester}) + So(err, ShouldBeNil) + + So(len(paths), ShouldEqual, 1) + + Convey("queryMeta only returns paths in the provided scope", func() { + paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"}) + So(errq, ShouldBeNil) + + So(len(paths), ShouldEqual, 2) + }) + }) + }) +} + +func uploadRequest(t *testing.T, lh *LocalHandler, request *put.Request) { + err := os.MkdirAll(filepath.Dir(request.Remote), userPerms) + So(err, ShouldBeNil) + + err = lh.Put(request) + So(err, ShouldBeNil) + + request.Meta.LocalMeta[put.MetaKeyRequester] = request.Requester + err = lh.AddMeta(request.Remote, request.Meta.LocalMeta) + So(err, ShouldBeNil) +} + +// makeMockRequests creates some local directories and files, and returns +// requests that all share the same metadata, with remotes pointing to another +// local temp directory (but the remote sub-directories are not created). +// Also returns the execpted remote directories that would have to be created. +func makeMockRequests(t *testing.T) ([]*put.Request, []string) { + t.Helper() + + sourceDir := t.TempDir() + destDir := t.TempDir() + + requests := makeTestRequests(t, sourceDir, destDir) + + return requests, []string{ + filepath.Join(destDir, "a", "b", "c"), + filepath.Join(destDir, "a", "b", "d", "e"), + } +} + +func makeTestRequests(t *testing.T, sourceDir, destDir string) []*put.Request { + t.Helper() + + sourcePaths := []string{ + filepath.Join(sourceDir, "a", "b", "c", "file.1"), + filepath.Join(sourceDir, "a", "b", "file.2"), + filepath.Join(sourceDir, "a", "b", "d", "file.3"), + filepath.Join(sourceDir, "a", "b", "d", "e", "file.4"), + filepath.Join(sourceDir, "a", "b", "d", "e", "file.5"), + } + + requests := make([]*put.Request, len(sourcePaths)) + localMeta := map[string]string{"a": "1", "b": "2"} + + for i, path := range sourcePaths { + dir := filepath.Dir(path) + + err := os.MkdirAll(dir, userPerms) + if err != nil { + t.Fatal(err) + } + + internal.CreateTestFile(t, path, "1\n") + + requests[i] = &put.Request{ + Local: path, + Remote: strings.Replace(path, sourceDir, destDir, 1), + Meta: &put.Meta{LocalMeta: localMeta}, + } + } + + return requests +} diff --git a/server/server.go b/server/server.go index f9f2cde..c19f5e9 100644 --- a/server/server.go +++ b/server/server.go @@ -44,6 +44,7 @@ import ( "github.com/inconshreveable/log15" gas "github.com/wtsi-hgi/go-authserver" "github.com/wtsi-hgi/ibackup/put" + "github.com/wtsi-hgi/ibackup/remove" "github.com/wtsi-hgi/ibackup/set" "github.com/wtsi-hgi/ibackup/slack" "github.com/wtsi-ssg/wrstat/v6/scheduler" @@ -91,7 +92,7 @@ type Config struct { SlackMessageDebounce time.Duration // StorageHandler is used to interact with the storage system, e.g. iRODS. - StorageHandler put.Handler + StorageHandler remove.RemoveHandler } // Server is used to start a web server that provides a REST API to the setdb @@ -116,7 +117,7 @@ type Server struct { stillRunningMsgFreq time.Duration serverAliveCh chan bool uploadTracker *uploadTracker - storageHandler put.Handler + storageHandler remove.RemoveHandler mapMu sync.RWMutex creatingCollections map[string]bool @@ -152,17 +153,19 @@ func New(conf Config) (*Server, error) { s.Server.Router().Use(gas.IncludeAbortErrorsInBody) - s.monitor = NewMonitor(func(given *set.Set) { - if err := s.discoverSet(given); err != nil { - s.Logger.Printf("error discovering set during monitoring: %s", err) - } - }) + s.monitor = NewMonitor(s.monitorCB) s.SetStopCallBack(s.stop) return s, nil } +func (s *Server) monitorCB(given *set.Set) { + if err := s.discoverSet(given); err != nil { + s.Logger.Printf("error discovering set during monitoring: %s", err) + } +} + // Start logs to slack that the server has been started and then calls // gas.Server.Start(). func (s *Server) Start(addr, certFile, keyFile string) error { diff --git a/server/setdb.go b/server/setdb.go index 9a76d27..9305641 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -42,6 +42,7 @@ import ( gas "github.com/wtsi-hgi/go-authserver" "github.com/wtsi-hgi/grand" "github.com/wtsi-hgi/ibackup/put" + "github.com/wtsi-hgi/ibackup/remove" "github.com/wtsi-hgi/ibackup/set" "github.com/wtsi-hgi/ibackup/slack" ) @@ -594,7 +595,7 @@ func (s *Server) removeFileFromIRODS(set *set.Set, path string, transformer put. return err } - return put.RemovePathFromSetInIRODS(s.storageHandler, transformer, rpath, sets, requesters, remoteMeta) + return remove.RemovePathFromSetInIRODS(s.storageHandler, transformer, rpath, sets, requesters, remoteMeta) } func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { @@ -685,7 +686,7 @@ func (s *Server) removeDirFromIRODSandDB(removeReq *RemoveReq) error { } if !removeReq.IsDirEmpty && !removeReq.IsRemovedFromIRODS { - err = put.RemoveDirFromIRODS(s.storageHandler, removeReq.Path, transformer) + err = remove.RemoveDirFromIRODS(s.storageHandler, removeReq.Path, transformer) if err != nil { return err } From a4224f3b12047fdabf74a4827f88d74479bb8c52 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 19 Feb 2025 16:05:40 +0000 Subject: [PATCH 31/35] Add new baton pkg and make remove.handler use baton.GetBatonHandler() WIP --- {put => baton}/baton.go | 162 +++++++++++++++++++++------- cmd/filestatus.go | 3 +- cmd/put.go | 3 +- cmd/server.go | 4 +- remove/mock.go => internal/error.go | 65 +++-------- internal/file.go | 4 +- {put => internal}/mock.go | 113 ++++++++++++------- put/baton_test.go | 22 ++-- put/info.go | 4 +- put/put.go | 39 ++----- put/put_test.go | 40 +++---- put/request.go | 19 ++-- remove/baton.go | 137 ----------------------- remove/remove.go | 19 +++- remove/remove_test.go | 6 +- server/server.go | 4 +- server/server_test.go | 21 ++-- set/set_test.go | 5 +- 18 files changed, 310 insertions(+), 360 deletions(-) rename {put => baton}/baton.go (77%) rename remove/mock.go => internal/error.go (54%) rename {put => internal}/mock.go (74%) delete mode 100644 remove/baton.go diff --git a/put/baton.go b/baton/baton.go similarity index 77% rename from put/baton.go rename to baton/baton.go index 63ed556..8ccbf61 100644 --- a/put/baton.go +++ b/baton/baton.go @@ -25,7 +25,7 @@ // this file implements a Handler using baton, via extendo. -package put +package baton import ( "context" @@ -37,6 +37,7 @@ import ( "time" "github.com/rs/zerolog" + "github.com/wtsi-hgi/ibackup/internal" ex "github.com/wtsi-npg/extendo/v2" logs "github.com/wtsi-npg/logshim" "github.com/wtsi-npg/logshim-zerolog/zlog" @@ -48,18 +49,19 @@ import ( const ( ErrOperationTimeout = "iRODS operation timed out" - extendoLogLevel = logs.ErrorLevel - numCollClients = workerPoolSizeCollections - numPutMetaClients = 2 - collClientMaxIndex = numCollClients - 1 - putClientIndex = collClientMaxIndex + 1 - metaClientIndex = putClientIndex + 1 - extendoNotExist = "does not exist" - operationMinBackoff = 5 * time.Second - operationMaxBackoff = 30 * time.Second - operationBackoffFactor = 1.1 - operationTimeout = 15 * time.Second - operationRetries = 6 + extendoLogLevel = logs.ErrorLevel + workerPoolSizeCollections = 2 + numCollClients = workerPoolSizeCollections + numPutMetaClients = 2 + collClientMaxIndex = numCollClients - 1 + putClientIndex = collClientMaxIndex + 1 + metaClientIndex = putClientIndex + 1 + extendoNotExist = "does not exist" + operationMinBackoff = 5 * time.Second + operationMaxBackoff = 30 * time.Second + operationBackoffFactor = 1.1 + operationTimeout = 15 * time.Second + operationRetries = 6 ) // Baton is a Handler that uses Baton (via extendo) to interact with iRODS. @@ -173,14 +175,14 @@ func (b *Baton) connect(numClients uint8) (*ex.ClientPool, chan *ex.Client, erro params.MaxSize = numClients pool := ex.NewClientPool(params, "") - clientCh, err := b.getClientsFromPoolConcurrently(pool, numClients) + clientCh, err := b.GetClientsFromPoolConcurrently(pool, numClients) return pool, clientCh, err } -// getClientsFromPoolConcurrently gets numClients clients from the pool +// GetClientsFromPoolConcurrently gets numClients clients from the pool // concurrently. -func (b *Baton) getClientsFromPoolConcurrently(pool *ex.ClientPool, numClients uint8) (chan *ex.Client, error) { +func (b *Baton) GetClientsFromPoolConcurrently(pool *ex.ClientPool, numClients uint8) (chan *ex.Client, error) { clientCh := make(chan *ex.Client, numClients) errCh := make(chan error, numClients) @@ -218,7 +220,7 @@ func (b *Baton) ensureCollection(clientIndex int, ri ex.RodsItem) error { return nil } - if errors.Is(err, Error{ErrOperationTimeout, ri.IPath}) { + if errors.Is(err, internal.Error{ErrOperationTimeout, ri.IPath}) { return err } @@ -242,7 +244,7 @@ func TimeoutOp(op retry.Operation, path string) error { case err = <-errCh: timer.Stop() case <-timer.C: - err = Error{ErrOperationTimeout, path} + err = internal.Error{ErrOperationTimeout, path} } return err @@ -372,42 +374,42 @@ func (b *Baton) closeConnections(clients []*ex.Client) { } } +// TODO only take remote + // Stat gets mtime and metadata info for the request Remote object. -func (b *Baton) Stat(request *Request) (*ObjectInfo, error) { +func (b *Baton) Stat(local, remote string) (bool, map[string]string, error) { var it ex.RodsItem err := TimeoutOp(func() error { var errl error - it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem(request)) + it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem(local, remote)) return errl - }, "stat failed: "+request.Remote) + }, "stat failed: "+remote) if err != nil { if strings.Contains(err.Error(), extendoNotExist) { - return &ObjectInfo{Exists: false}, nil + return false, map[string]string{}, nil } - return nil, err + return false, nil, err } - return &ObjectInfo{Exists: true, Meta: rodsItemToMeta(it)}, nil + return true, RodsItemToMeta(it), nil } // requestToRodsItem converts a Request in to an extendo RodsItem without AVUs. -func requestToRodsItem(request *Request) *ex.RodsItem { - local := request.LocalDataPath() - +func requestToRodsItem(local, remote string) *ex.RodsItem { return &ex.RodsItem{ IDirectory: filepath.Dir(local), IFile: filepath.Base(local), - IPath: filepath.Dir(request.Remote), - IName: filepath.Base(request.Remote), + IPath: filepath.Dir(remote), + IName: filepath.Base(remote), } } -// rodsItemToMeta pulls out the AVUs from a RodsItem and returns them as a map. -func rodsItemToMeta(it ex.RodsItem) map[string]string { +// RodsItemToMeta pulls out the AVUs from a RodsItem and returns them as a map. +func RodsItemToMeta(it ex.RodsItem) map[string]string { meta := make(map[string]string, len(it.IAVUs)) for _, iavu := range it.IAVUs { @@ -420,8 +422,8 @@ func rodsItemToMeta(it ex.RodsItem) map[string]string { // Put uploads request Local to the Remote object, overwriting it if it already // exists. It calculates and stores the md5 checksum remotely, comparing to the // local checksum. -func (b *Baton) Put(request *Request) error { - item := requestToRodsItemWithAVUs(request) +func (b *Baton) Put(local, remote string, meta map[string]string) error { + item := requestToRodsItemWithAVUs(local, remote, meta) // iRODS treats /dev/null specially, so unless that changes we have to check // for it and create a temporary empty file in its place. @@ -454,9 +456,9 @@ func (b *Baton) Put(request *Request) error { // requestToRodsItemWithAVUs converts a Request in to an extendo RodsItem with // AVUs. -func requestToRodsItemWithAVUs(request *Request) *ex.RodsItem { - item := requestToRodsItem(request) - item.IAVUs = MetaToAVUs(request.Meta.Metadata()) +func requestToRodsItemWithAVUs(local, remote string, meta map[string]string) *ex.RodsItem { + item := requestToRodsItem(local, remote) + item.IAVUs = MetaToAVUs(meta) return item } @@ -494,7 +496,7 @@ func (b *Baton) GetMeta(path string) (map[string]string, error) { IName: filepath.Base(path), }) - return rodsItemToMeta(it), err + return RodsItemToMeta(it), err } // RemotePathToRodsItem converts a path in to an extendo RodsItem. @@ -537,3 +539,89 @@ func (b *Baton) Cleanup() error { return nil } + +// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact +// with iRODS and contains a meta client for interacting with metadata. If you +// don't have baton-do in your PATH, you'll get an error. +// func GetBatonHandlerWithMetaClient() (*Baton, error) { +// setupExtendoLogger() + +// _, err := ex.FindBaton() +// if err != nil { +// return nil, err +// } + +// params := ex.DefaultClientPoolParams +// params.MaxSize = 1 +// pool := ex.NewClientPool(params, "") + +// metaClient, err := pool.Get() +// if err != nil { +// return nil, fmt.Errorf("failed to get metaClient: %w", err) +// } + +// baton := &Baton{ +// Baton: put.Baton{ +// PutMetaPool: pool, +// MetaClient: metaClient, +// }, +// } + +// return baton, nil +// } + +func (b *Baton) RemoveFile(path string) error { + it := RemotePathToRodsItem(path) + + err := TimeoutOp(func() error { + _, errl := b.MetaClient.RemObj(ex.Args{}, *it) + + return errl + }, "remove file error: "+path) + + return err +} + +// RemoveDir removes the given directory from iRODS given it is empty. +func (b *Baton) RemoveDir(path string) error { + it := &ex.RodsItem{ + IPath: path, + } + + err := TimeoutOp(func() error { + _, errl := b.MetaClient.RemDir(ex.Args{}, *it) + + return errl + }, "remove meta error: "+path) + + return err +} + +func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { + it := &ex.RodsItem{ + IPath: dirToSearch, + IAVUs: MetaToAVUs(meta), + } + + var items []ex.RodsItem + + var err error + + err = TimeoutOp(func() error { + items, err = b.MetaClient.MetaQuery(ex.Args{Object: true}, *it) + + return err + }, "query meta error: "+dirToSearch) + + paths := make([]string, len(items)) + + for i, item := range items { + paths[i] = item.IPath + } + + return paths, err +} + +func (b *Baton) AllClientsStopped() bool { + return !b.PutMetaPool.IsOpen() && !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && b.collClients == nil +} diff --git a/cmd/filestatus.go b/cmd/filestatus.go index 9e909c7..99057c2 100644 --- a/cmd/filestatus.go +++ b/cmd/filestatus.go @@ -11,6 +11,7 @@ import ( "github.com/dustin/go-humanize" //nolint:misspell "github.com/spf13/cobra" + "github.com/wtsi-hgi/ibackup/baton" "github.com/wtsi-hgi/ibackup/put" "github.com/wtsi-hgi/ibackup/set" "github.com/wtsi-npg/extendo/v2" @@ -107,7 +108,7 @@ func newFSG(db *set.DBRO, filePath string, useIRods bool) *fileStatusGetter { return fsg } - put.GetBatonHandler() //nolint:errcheck + baton.GetBatonHandler() //nolint:errcheck if client, err := extendo.FindAndStart("--unbuffered", "--no-error"); err != nil { warn("error occurred invoking baton; disabling irods mode: %s", err) diff --git a/cmd/put.go b/cmd/put.go index d12f76a..7cfd34d 100644 --- a/cmd/put.go +++ b/cmd/put.go @@ -37,6 +37,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/wtsi-hgi/ibackup/baton" "github.com/wtsi-hgi/ibackup/put" "github.com/wtsi-hgi/ibackup/server" ) @@ -247,7 +248,7 @@ func handlePut(client *server.Client, requests []*put.Request) (chan *put.Reques } func getPutter(requests []*put.Request) (*put.Putter, func()) { - handler, err := put.GetBatonHandler() + handler, err := baton.GetBatonHandler() if err != nil { die("%s", err) } diff --git a/cmd/server.go b/cmd/server.go index 5ed797d..bf46480 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -39,7 +39,7 @@ import ( "github.com/inconshreveable/log15" "github.com/spf13/cobra" gas "github.com/wtsi-hgi/go-authserver" - "github.com/wtsi-hgi/ibackup/remove" + "github.com/wtsi-hgi/ibackup/baton" "github.com/wtsi-hgi/ibackup/server" "github.com/wtsi-hgi/ibackup/set" "github.com/wtsi-hgi/ibackup/slack" @@ -193,7 +193,7 @@ database that you've made, to investigate. die("slack_debounce period must be positive, not: %d", serverSlackDebouncePeriod) } - handler, errb := remove.GetBatonHandlerWithMetaClient() + handler, errb := baton.GetBatonHandler() if errb != nil { die("failed to get baton handler: %s", errb) } diff --git a/remove/mock.go b/internal/error.go similarity index 54% rename from remove/mock.go rename to internal/error.go index 73c9891..91b3139 100644 --- a/remove/mock.go +++ b/internal/error.go @@ -24,66 +24,31 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -package remove +package internal import ( - "os" - "strings" - - "github.com/wtsi-hgi/ibackup/put" + "errors" + "fmt" ) -type LocalHandler struct { - put.LocalHandler +type Error struct { + Msg string + Path string } -// GetLocalHandler returns a Handler that doesn't actually interact with iRODS, -// but instead simply treats "Remote" as local paths and copies from Local to -// Remote for any Put()s. For use during tests. -func GetLocalHandler() *LocalHandler { - return &LocalHandler{ - LocalHandler: put.LocalHandler{ - Meta: make(map[string]map[string]string), - }, +func (e Error) Error() string { + if e.Path != "" { + return fmt.Sprintf("%s [%s]", e.Msg, e.Path) } -} -func (l *LocalHandler) RemoveDir(path string) error { - return os.Remove(path) + return e.Msg } -func (l *LocalHandler) RemoveFile(path string) error { - delete(l.Meta, path) - - return os.Remove(path) -} - -func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { - var objects []string - - for path, pathMeta := range l.Meta { - if !strings.HasPrefix(path, dirToSearch) { - continue - } - - if doesMetaContainMeta(pathMeta, meta) { - objects = append(objects, path) - } - } - - return objects, nil -} - -func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { - valid := true - - for k, v := range targetMeta { - if sourceMeta[k] != v { - valid = false - - break - } +func (e Error) Is(err error) bool { + var putErr *Error + if errors.As(err, &putErr) { + return putErr.Msg == e.Msg } - return valid + return false } diff --git a/internal/file.go b/internal/file.go index e9aa458..997ec12 100644 --- a/internal/file.go +++ b/internal/file.go @@ -42,7 +42,7 @@ import ( const ( fileCheckFrequency = 10 * time.Millisecond retryTimeout = 5 * time.Second - userPerms = 0700 + UserPerms = 0700 ) var ErrFileUnchanged = errors.New("file did not change") @@ -123,7 +123,7 @@ func CreateTestFile(t *testing.T, path, contents string) { dir := filepath.Dir(path) - err := os.MkdirAll(dir, userPerms) + err := os.MkdirAll(dir, UserPerms) if err != nil { t.Fatalf("mkdir failed: %s", err) } diff --git a/put/mock.go b/internal/mock.go similarity index 74% rename from put/mock.go rename to internal/mock.go index ab1416f..5d10563 100644 --- a/put/mock.go +++ b/internal/mock.go @@ -1,7 +1,8 @@ /******************************************************************************* - * Copyright (c) 2022, 2023 Genome Research Ltd. + * Copyright (c) 2025 Genome Research Ltd. * - * Author: Sendu Bala + * Author: Rosie Kern + * Author: Iaroslav Popov * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -23,17 +24,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -package put +package internal import ( "io" "maps" "os" + "strings" "sync" "time" ) -const userPerms = 0700 const ErrMockStatFail = "stat fail" const ErrMockPutFail = "put fail" const ErrMockMetaFail = "meta fail" @@ -41,16 +42,16 @@ const ErrMockMetaFail = "meta fail" // LocalHandler satisfies the Handler interface, treating "Remote" as local // paths and moving from Local to Remote for the Put(). type LocalHandler struct { - connected bool - cleaned bool - collections []string + Connected bool + Cleaned bool + Collections []string Meta map[string]map[string]string statFail string putFail string putSlow string putDur time.Duration metaFail string - mu sync.RWMutex + Mu sync.RWMutex } // GetLocalHandler returns a Handler that doesn't actually interact with iRODS, @@ -64,24 +65,24 @@ func GetLocalHandler() *LocalHandler { // Cleanup just records this was called. func (l *LocalHandler) Cleanup() error { - l.cleaned = true + l.Cleaned = true return nil } // EnsureCollection creates the given dir locally and records that we did this. func (l *LocalHandler) EnsureCollection(dir string) error { - l.mu.Lock() - defer l.mu.Unlock() + l.Mu.Lock() + defer l.Mu.Unlock() - l.collections = append(l.collections, dir) + l.Collections = append(l.Collections, dir) - return os.MkdirAll(dir, userPerms) + return os.MkdirAll(dir, UserPerms) } // CollectionsDone says we connected and prepares us for metadata handling. func (l *LocalHandler) CollectionsDone() error { - l.connected = true + l.Connected = true return nil } @@ -94,31 +95,31 @@ func (l *LocalHandler) MakeStatFail(remote string) { // Stat returns info about the Remote file, which is a local file on disk. // Returns an error if statFail == Remote. -func (l *LocalHandler) Stat(request *Request) (*ObjectInfo, error) { - if l.statFail == request.Remote { - return nil, Error{ErrMockStatFail, ""} +func (l *LocalHandler) Stat(_, remote string) (bool, map[string]string, error) { + if l.statFail == remote { + return false, nil, Error{ErrMockStatFail, ""} } - _, err := os.Stat(request.Remote) + _, err := os.Stat(remote) if os.IsNotExist(err) { - return &ObjectInfo{Exists: false}, nil + return false, map[string]string{}, nil } if err != nil { - return nil, err + return false, nil, err } - l.mu.RLock() - defer l.mu.RUnlock() + l.Mu.RLock() + defer l.Mu.RUnlock() - meta, exists := l.Meta[request.Remote] + meta, exists := l.Meta[remote] if !exists { meta = make(map[string]string) } else { meta = maps.Clone(meta) } - return &ObjectInfo{Exists: true, Meta: meta}, nil + return true, meta, nil } // MakePutFail will result in any subsequent Put()s for a Request with the @@ -129,22 +130,22 @@ func (l *LocalHandler) MakePutFail(remote string) { // MakePutSlow will result in any subsequent Put()s for a Request with the // given local path taking the given amount of time. -func (l *LocalHandler) MakePutSlow(local string, dur time.Duration) { - l.putSlow = local +func (l *LocalHandler) MakePutSlow(remote string, dur time.Duration) { + l.putSlow = remote l.putDur = dur } // Put just copies from Local to Remote. Returns an error if putFail == Remote. -func (l *LocalHandler) Put(request *Request) error { - if l.putFail == request.Remote { +func (l *LocalHandler) Put(local, remote string, meta map[string]string) error { + if l.putFail == remote { return Error{ErrMockPutFail, ""} } - if l.putSlow == request.Local { + if l.putSlow == remote { <-time.After(l.putDur) } - return copyFile(request.LocalDataPath(), request.Remote) + return copyFile(local, remote) } // copyFile copies source to dest. @@ -187,8 +188,8 @@ func (l *LocalHandler) RemoveMeta(path string, meta map[string]string) error { return Error{ErrMockMetaFail, ""} } - l.mu.Lock() - defer l.mu.Unlock() + l.Mu.Lock() + defer l.Mu.Unlock() pathMeta, exists := l.Meta[path] if !exists { @@ -209,8 +210,8 @@ func (l *LocalHandler) AddMeta(path string, meta map[string]string) error { return Error{ErrMockMetaFail, ""} } - l.mu.Lock() - defer l.mu.Unlock() + l.Mu.Lock() + defer l.Mu.Unlock() pathMeta, exists := l.Meta[path] if !exists { @@ -232,8 +233,8 @@ func (l *LocalHandler) AddMeta(path string, meta map[string]string) error { // GetMeta gets the metadata stored for the given path (returns an empty map if // path is not known about or has no metadata). func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { - l.mu.Lock() - defer l.mu.Unlock() + l.Mu.Lock() + defer l.Mu.Unlock() if l.Meta == nil { return make(map[string]string), nil @@ -252,3 +253,43 @@ func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { return meta, nil } + +func (l *LocalHandler) RemoveDir(path string) error { + return os.Remove(path) +} + +func (l *LocalHandler) RemoveFile(path string) error { + delete(l.Meta, path) + + return os.Remove(path) +} + +func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { + var objects []string + + for path, pathMeta := range l.Meta { + if !strings.HasPrefix(path, dirToSearch) { + continue + } + + if doesMetaContainMeta(pathMeta, meta) { + objects = append(objects, path) + } + } + + return objects, nil +} + +func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { + valid := true + + for k, v := range targetMeta { + if sourceMeta[k] != v { + valid = false + + break + } + } + + return valid +} diff --git a/put/baton_test.go b/put/baton_test.go index e6e23ec..8d3106c 100644 --- a/put/baton_test.go +++ b/put/baton_test.go @@ -37,12 +37,13 @@ import ( "time" . "github.com/smartystreets/goconvey/convey" + "github.com/wtsi-hgi/ibackup/baton" "github.com/wtsi-hgi/ibackup/internal" ex "github.com/wtsi-npg/extendo/v2" ) func TestPutBaton(t *testing.T) { - h, errgbh := GetBatonHandler() + h, errgbh := baton.GetBatonHandler() if errgbh != nil { t.Logf("GetBatonHandler error: %s", errgbh) SkipConvey("Skipping baton tests since couldn't find baton", t, func() {}) @@ -66,7 +67,7 @@ func TestPutBaton(t *testing.T) { Convey("CreateCollections() creates the needed collections", func() { testPool := ex.NewClientPool(ex.DefaultClientPoolParams, "") - testClientCh, err := h.getClientsFromPoolConcurrently(testPool, 1) + testClientCh, err := h.GetClientsFromPoolConcurrently(testPool, 1) So(err, ShouldBeNil) testClient := <-testClientCh defer testClient.StopIgnoreError() @@ -158,10 +159,7 @@ func TestPutBaton(t *testing.T) { err = p.Cleanup() So(err, ShouldBeNil) - So(h.PutMetaPool.IsOpen(), ShouldBeFalse) - So(h.putClient.IsRunning(), ShouldBeFalse) - So(h.MetaClient.IsRunning(), ShouldBeFalse) - So(h.collClients, ShouldBeNil) + So(h.AllClientsStopped(), ShouldBeTrue) }) }) @@ -216,7 +214,7 @@ func TestPutBaton(t *testing.T) { it, err = getItemWithBaton(testClient, requests[2].Remote) So(err, ShouldBeNil) So(it.ISize, ShouldEqual, 0) - meta := rodsItemToMeta(it) + meta := baton.RodsItemToMeta(it) So(meta[MetaKeyRemoteHardlink], ShouldEqual, requests[2].Hardlink) So(meta[MetaKeyHardlink], ShouldEqual, requests[2].Local) @@ -413,10 +411,10 @@ func getObjectMetadataWithBaton(client *ex.Client, path string) map[string]strin it, err := getItemWithBaton(client, path) So(err, ShouldBeNil) - return rodsItemToMeta(it) + return baton.RodsItemToMeta(it) } -func testPreparePutFile(t *testing.T, h *Baton, basename, rootCollection string) (string, *Putter) { +func testPreparePutFile(t *testing.T, h *baton.Baton, basename, rootCollection string) (string, *Putter) { t.Helper() path, sourceDir := testCreateLocalFile(t, basename) @@ -441,7 +439,7 @@ func testCreateLocalFile(t *testing.T, basename string) (string, string) { return path, sourceDir } -func testPreparePutter(t *testing.T, h *Baton, req *Request, rootCollection string) *Putter { +func testPreparePutter(t *testing.T, h *baton.Baton, req *Request, rootCollection string) *Putter { t.Helper() p, err := New(h, []*Request{req}) @@ -456,11 +454,11 @@ func testPreparePutter(t *testing.T, h *Baton, req *Request, rootCollection stri return p } -func testDeleteCollection(t *testing.T, h *Baton, collection string) { +func testDeleteCollection(t *testing.T, h *baton.Baton, collection string) { t.Helper() testPool := ex.NewClientPool(ex.DefaultClientPoolParams, "") - testClientCh, err := h.getClientsFromPoolConcurrently(testPool, 1) + testClientCh, err := h.GetClientsFromPoolConcurrently(testPool, 1) So(err, ShouldBeNil) testClient := <-testClientCh diff --git a/put/info.go b/put/info.go index c445fdf..f99d015 100644 --- a/put/info.go +++ b/put/info.go @@ -31,6 +31,8 @@ import ( "strconv" "syscall" "time" + + "github.com/wtsi-hgi/ibackup/internal" ) const ErrStatFailed = "stat of local path returned strange results" @@ -75,7 +77,7 @@ func Stat(localPath string) (*ObjectInfo, error) { func getUserAndGroupFromFileInfo(fi os.FileInfo, localPath string) (string, string, error) { stat, ok := fi.Sys().(*syscall.Stat_t) if !ok { - return "", "", Error{ErrStatFailed, localPath} + return "", "", internal.Error{ErrStatFailed, localPath} } u, err := user.LookupId(strconv.Itoa(int(stat.Uid))) diff --git a/put/put.go b/put/put.go index d5f0f8a..bb36229 100644 --- a/put/put.go +++ b/put/put.go @@ -29,8 +29,6 @@ package put import ( "context" - "errors" - "fmt" "os" "os/exec" "path/filepath" @@ -41,30 +39,9 @@ import ( "github.com/gammazero/workerpool" "github.com/hashicorp/go-multierror" + "github.com/wtsi-hgi/ibackup/internal" ) -type Error struct { - msg string - path string -} - -func (e Error) Error() string { - if e.path != "" { - return fmt.Sprintf("%s [%s]", e.msg, e.path) - } - - return e.msg -} - -func (e Error) Is(err error) bool { - var putErr *Error - if errors.As(err, &putErr) { - return putErr.msg == e.msg - } - - return false -} - const ( ErrLocalNotAbs = "local path could not be made absolute" ErrRemoteNotAbs = "remote path not absolute" @@ -78,9 +55,9 @@ const ( // during Put(). workerPoolSizeStats = 16 - // workerPoolSizeCollections is the max number of concurrent collection + // WorkerPoolSizeCollections is the max number of concurrent collection // creations we'll do during CreateCollections(). - workerPoolSizeCollections = 2 + WorkerPoolSizeCollections = 2 ) // Handler is something that knows how to communicate with iRODS and carry out @@ -100,12 +77,12 @@ type Handler interface { // Stat checks if the Request's Remote object exists. If it does, records // its metadata in the returned ObjectInfo. Returns an error if there was a // problem finding out information (but not if the object does not exist). - Stat(request *Request) (*ObjectInfo, error) + Stat(local, remote string) (bool, map[string]string, error) // Put uploads the Request's Local file to the Remote location, overwriting // any existing object, and ensuring that a locally calculated and remotely // calculated md5 checksum match. - Put(request *Request) error + Put(local, remote string, meta map[string]string) error // RemoveMeta deletes the given metadata from the given object. RemoveMeta(path string, meta map[string]string) error @@ -132,7 +109,7 @@ type FileReadTester func(ctx context.Context, path string) error func headRead(ctx context.Context, path string) error { out, err := exec.CommandContext(ctx, "head", "-c", "1", path).CombinedOutput() if err != nil && len(out) > 0 { - err = Error{msg: string(out)} + err = internal.Error{Msg: string(out)} } return err @@ -236,7 +213,7 @@ func (p *Putter) Cleanup() error { // wrapped in to one. func (p *Putter) CreateCollections() error { dirs := p.getUniqueRequestLeafCollections() - pool := workerpool.New(workerPoolSizeCollections) + pool := workerpool.New(WorkerPoolSizeCollections) errCh := make(chan error, len(dirs)) for _, dir := range dirs { @@ -610,7 +587,7 @@ func (p *Putter) testRead(request *Request) error { go func() { select { case <-timer.C: - errCh <- Error{ErrReadTimeout, request.Local} + errCh <- internal.Error{ErrReadTimeout, request.Local} case err := <-readCh: timer.Stop() errCh <- err diff --git a/put/put_test.go b/put/put_test.go index c42e30b..99642e7 100644 --- a/put/put_test.go +++ b/put/put_test.go @@ -41,7 +41,7 @@ func TestPutMock(t *testing.T) { Convey("Given Requests and a mock Handler, you can make a new Putter", t, func() { requests, expectedCollections := makeMockRequests(t) - lh := GetLocalHandler() + lh := internal.GetLocalHandler() p, err := New(lh, requests) So(err, ShouldBeNil) @@ -50,16 +50,16 @@ func TestPutMock(t *testing.T) { Convey("CreateCollections() creates the minimal number of collections", func() { err = p.CreateCollections() So(err, ShouldBeNil) - So(lh.connected, ShouldBeTrue) + So(lh.Connected, ShouldBeTrue) for _, request := range requests { _, err = os.Stat(filepath.Dir(request.Remote)) So(err, ShouldBeNil) } - sort.Strings(lh.collections) + sort.Strings(lh.Collections) - So(lh.collections, ShouldResemble, expectedCollections) + So(lh.Collections, ShouldResemble, expectedCollections) Convey("Put() then puts the files, and adds the metadata", func() { requests[0].Requester = "John" @@ -78,10 +78,10 @@ func TestPutMock(t *testing.T) { _, err = os.Stat(request.Remote) So(err, ShouldBeNil) - lh.mu.RLock() + lh.Mu.RLock() So(lh.Meta[request.Remote], ShouldResemble, request.Meta.LocalMeta) checkAddedMeta(lh.Meta[request.Remote]) - lh.mu.RUnlock() + lh.Mu.RUnlock() } skipped := 0 @@ -110,12 +110,12 @@ func TestPutMock(t *testing.T) { switch request.Status { case RequestStatusReplaced: replaced++ - lh.mu.RLock() + lh.Mu.RLock() So(lh.Meta[request.Remote], ShouldResemble, requests[0].Meta.LocalMeta) So(lh.Meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") So(lh.Meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB") date = lh.Meta[request.Remote][MetaKeyDate] - lh.mu.RUnlock() + lh.Mu.RUnlock() default: other++ } @@ -156,18 +156,18 @@ func TestPutMock(t *testing.T) { request = <-srCh So(request.Status, ShouldEqual, RequestStatusUnmodified) - lh.mu.RLock() + lh.Mu.RLock() So(lh.Meta[request.Remote][MetaKeyRequester], ShouldEqual, "John,Sam") So(lh.Meta[request.Remote][MetaKeySets], ShouldEqual, "setA,setB,setC") So(lh.Meta[request.Remote][MetaKeyDate], ShouldEqual, date) - lh.mu.RUnlock() + lh.Mu.RUnlock() }) }) Convey("Finally, Cleanup() defers to the handler", func() { err = p.Cleanup() So(err, ShouldBeNil) - So(lh.cleaned, ShouldBeTrue) + So(lh.Cleaned, ShouldBeTrue) }) }) @@ -340,10 +340,10 @@ func TestPutMock(t *testing.T) { switch r.Remote { case requests[1].Remote: - So(r.Error, ShouldContainSubstring, ErrMockPutFail) + So(r.Error, ShouldContainSubstring, internal.ErrMockPutFail) cases++ case requests[2].Remote: - So(r.Error, ShouldContainSubstring, ErrMockMetaFail) + So(r.Error, ShouldContainSubstring, internal.ErrMockMetaFail) cases++ } case RequestStatusUploaded: @@ -359,7 +359,7 @@ func TestPutMock(t *testing.T) { fails++ if r.Remote == requests[0].Remote { - So(r.Error, ShouldContainSubstring, ErrMockStatFail) + So(r.Error, ShouldContainSubstring, internal.ErrMockStatFail) cases++ } default: @@ -430,7 +430,7 @@ func TestPutMock(t *testing.T) { }) Convey("CreateCollections() also works with 0 or 1 requests", t, func() { - lh := &LocalHandler{} + lh := &internal.LocalHandler{} var requests []*Request @@ -441,7 +441,7 @@ func TestPutMock(t *testing.T) { err = p.CreateCollections() So(err, ShouldBeNil) - So(lh.collections, ShouldBeNil) + So(lh.Collections, ShouldBeNil) ddir := t.TempDir() col := filepath.Join(ddir, "bar") @@ -455,11 +455,11 @@ func TestPutMock(t *testing.T) { err = p.CreateCollections() So(err, ShouldBeNil) - So(lh.collections, ShouldResemble, []string{col}) + So(lh.Collections, ShouldResemble, []string{col}) }) Convey("Relative local paths are made absolute", t, func() { - lh := &LocalHandler{} + lh := &internal.LocalHandler{} p, err := New(lh, []*Request{{Local: "foo", Remote: "/bar"}}) So(err, ShouldBeNil) @@ -490,7 +490,7 @@ func TestPutMock(t *testing.T) { }) Convey("You can't make a Putter with relative remote paths", t, func() { - lh := &LocalHandler{} + lh := &internal.LocalHandler{} _, err := New(lh, []*Request{{Local: "/foo", Remote: "bar"}}) So(err, ShouldNotBeNil) So(err.Error(), ShouldContainSubstring, ErrRemoteNotAbs) @@ -533,7 +533,7 @@ func makeTestRequests(t *testing.T, sourceDir, destDir string) []*Request { for i, path := range sourcePaths { dir := filepath.Dir(path) - err := os.MkdirAll(dir, userPerms) + err := os.MkdirAll(dir, internal.UserPerms) if err != nil { t.Fatal(err) } diff --git a/put/request.go b/put/request.go index 5e23689..4e21e5c 100644 --- a/put/request.go +++ b/put/request.go @@ -33,6 +33,7 @@ import ( "time" "github.com/dgryski/go-farm" + "github.com/wtsi-hgi/ibackup/internal" ) type RequestStatus string @@ -163,13 +164,13 @@ func (r *Request) Prepare() error { func (r *Request) ValidatePaths() error { local, err := filepath.Abs(r.Local) if err != nil { - return Error{ErrLocalNotAbs, r.Local} + return internal.Error{ErrLocalNotAbs, r.Local} } r.Local = local if !filepath.IsAbs(r.Remote) { - return Error{ErrRemoteNotAbs, r.Remote} + return internal.Error{ErrRemoteNotAbs, r.Remote} } return nil @@ -278,14 +279,16 @@ func (r *Request) StatAndAssociateStandardMetadata(lInfo *ObjectInfo, handler Ha func statAndAssociateStandardMetadata(request *Request, diskMeta map[string]string, handler Handler) (*ObjectInfo, error) { - rInfo, err := handler.Stat(request) + exists, meta, err := handler.Stat(request.LocalDataPath(), request.Remote) if err != nil { return nil, err } + rInfo := ObjectInfo{Exists: exists, Meta: meta} + request.Meta.addStandardMeta(diskMeta, rInfo.Meta, request.Requester, request.Set) - return rInfo, nil + return &rInfo, nil } // RemoveAndAddMetadata removes and adds metadata on our Remote based on the @@ -343,16 +346,16 @@ func (r *Request) addMeta(handler Handler, toAdd map[string]string) error { // to Remote, with linking metadata. func (r *Request) Put(handler Handler) error { if r.Hardlink == "" { - return handler.Put(r) + return handler.Put(r.LocalDataPath(), r.Remote, r.Meta.Metadata()) } if !r.onlyUploadEmptyFile { - if err := handler.Put(r.inodeRequest); err != nil { + if err := handler.Put(r.inodeRequest.LocalDataPath(), r.inodeRequest.Remote, r.inodeRequest.Meta.Metadata()); err != nil { return err } } - return handler.Put(r.emptyFileRequest) + return handler.Put(r.emptyFileRequest.LocalDataPath(), r.emptyFileRequest.Remote, r.emptyFileRequest.Meta.Metadata()) } // PathTransformer is a function that given a local path, returns the @@ -404,7 +407,7 @@ func HumgenTransformer(local string) (string, error) { } if !dirIsLustreWithPTUSubDir(parts[1], ptuPart, len(parts)) { - return "", Error{ErrNotHumgenLustre, local} + return "", internal.Error{ErrNotHumgenLustre, local} } return fmt.Sprintf("/humgen/%s/%s/%s/%s", parts[ptuPart], parts[ptuPart+1], parts[2], diff --git a/remove/baton.go b/remove/baton.go deleted file mode 100644 index 3772a88..0000000 --- a/remove/baton.go +++ /dev/null @@ -1,137 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2022, 2023 Genome Research Ltd. - * - * Author: Sendu Bala - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ******************************************************************************/ - -// this file implements a Handler using baton, via extendo. - -package remove - -import ( - "fmt" - "os" - - "github.com/rs/zerolog" - "github.com/wtsi-hgi/ibackup/put" - ex "github.com/wtsi-npg/extendo/v2" - logs "github.com/wtsi-npg/logshim" - "github.com/wtsi-npg/logshim-zerolog/zlog" -) - -// Baton is a Handler that uses Baton (via extendo) to interact with iRODS. -type Baton struct { - put.Baton -} - -const ( - extendoLogLevel = logs.ErrorLevel -) - -// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact -// with iRODS and contains a meta client for interacting with metadata. If you -// don't have baton-do in your PATH, you'll get an error. -func GetBatonHandlerWithMetaClient() (*Baton, error) { - setupExtendoLogger() - - _, err := ex.FindBaton() - if err != nil { - return nil, err - } - - params := ex.DefaultClientPoolParams - params.MaxSize = 1 - pool := ex.NewClientPool(params, "") - - metaClient, err := pool.Get() - if err != nil { - return nil, fmt.Errorf("failed to get metaClient: %w", err) - } - - baton := &Baton{ - Baton: put.Baton{ - PutMetaPool: pool, - MetaClient: metaClient, - }, - } - - return baton, nil -} - -// setupExtendoLogger sets up a STDERR logger that the extendo library will use. -// (We don't actually care about what it might log, but extendo doesn't work -// without this.) -func setupExtendoLogger() { - logs.InstallLogger(zlog.New(zerolog.SyncWriter(os.Stderr), extendoLogLevel)) -} - -func (b *Baton) RemoveFile(path string) error { - it := put.RemotePathToRodsItem(path) - - err := put.TimeoutOp(func() error { - _, errl := b.MetaClient.RemObj(ex.Args{}, *it) - - return errl - }, "remove file error: "+path) - - return err -} - -// RemoveDir removes the given directory from iRODS given it is empty. -func (b *Baton) RemoveDir(path string) error { - it := &ex.RodsItem{ - IPath: path, - } - - err := put.TimeoutOp(func() error { - _, errl := b.MetaClient.RemDir(ex.Args{}, *it) - - return errl - }, "remove meta error: "+path) - - return err -} - -func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { - it := &ex.RodsItem{ - IPath: dirToSearch, - IAVUs: put.MetaToAVUs(meta), - } - - var items []ex.RodsItem - - var err error - - err = put.TimeoutOp(func() error { - items, err = b.MetaClient.MetaQuery(ex.Args{Object: true}, *it) - - return err - }, "query meta error: "+dirToSearch) - - paths := make([]string, len(items)) - - for i, item := range items { - paths[i] = item.IPath - } - - return paths, err -} diff --git a/remove/remove.go b/remove/remove.go index 8589f49..0d0b8c3 100644 --- a/remove/remove.go +++ b/remove/remove.go @@ -35,8 +35,17 @@ import ( "github.com/wtsi-hgi/ibackup/put" ) -type RemoveHandler interface { - put.Handler +type Handler interface { + // RemoveMeta deletes the given metadata from the given object. + RemoveMeta(path string, meta map[string]string) error + + // AddMeta adds the given metadata to the given object. Given metadata keys + // should already have been removed with RemoveMeta() from the remote + // object. + AddMeta(path string, meta map[string]string) error + + // TODO + GetMeta(path string) (map[string]string, error) // RemoveDir deletes a given empty folder RemoveDir(path string) error @@ -51,7 +60,7 @@ type RemoveHandler interface { // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not // associated with any other sets. Otherwise it updates the iRODS metadata for // the path to not include the given set. -func RemovePathFromSetInIRODS(handler RemoveHandler, transformer put.PathTransformer, path string, +func RemovePathFromSetInIRODS(handler Handler, transformer put.PathTransformer, path string, sets, requesters []string, meta map[string]string) error { if len(sets) == 0 { return handleHardlinkAndRemoveFromIRODS(handler, path, transformer, meta) @@ -82,7 +91,7 @@ func RemovePathFromSetInIRODS(handler RemoveHandler, transformer put.PathTransfo // handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the // path is found to be a hardlink, it checks if there are other hardlinks to the // same file, if not, it removes the file. -func handleHardlinkAndRemoveFromIRODS(handler RemoveHandler, path string, transformer put.PathTransformer, +func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer put.PathTransformer, meta map[string]string) error { err := handler.RemoveFile(path) if err != nil { @@ -111,7 +120,7 @@ func handleHardlinkAndRemoveFromIRODS(handler RemoveHandler, path string, transf } // RemoveDirFromIRODS removes the remote path of a given directory from iRODS. -func RemoveDirFromIRODS(handler RemoveHandler, path string, transformer put.PathTransformer) error { +func RemoveDirFromIRODS(handler Handler, path string, transformer put.PathTransformer) error { rpath, err := transformer(path) if err != nil { return err diff --git a/remove/remove_test.go b/remove/remove_test.go index 56a3542..36b13ed 100644 --- a/remove/remove_test.go +++ b/remove/remove_test.go @@ -43,7 +43,7 @@ func TestRemoveMock(t *testing.T) { Convey("Given a mock RemoveHandler", t, func() { requests, expectedCollections := makeMockRequests(t) - lh := GetLocalHandler() + lh := internal.GetLocalHandler() requests[0].Requester = "John" @@ -94,11 +94,11 @@ func TestRemoveMock(t *testing.T) { }) } -func uploadRequest(t *testing.T, lh *LocalHandler, request *put.Request) { +func uploadRequest(t *testing.T, lh *internal.LocalHandler, request *put.Request) { err := os.MkdirAll(filepath.Dir(request.Remote), userPerms) So(err, ShouldBeNil) - err = lh.Put(request) + err = lh.Put(request.LocalDataPath(), request.Remote, request.Meta.Metadata()) So(err, ShouldBeNil) request.Meta.LocalMeta[put.MetaKeyRequester] = request.Requester diff --git a/server/server.go b/server/server.go index c19f5e9..6b9323c 100644 --- a/server/server.go +++ b/server/server.go @@ -92,7 +92,7 @@ type Config struct { SlackMessageDebounce time.Duration // StorageHandler is used to interact with the storage system, e.g. iRODS. - StorageHandler remove.RemoveHandler + StorageHandler remove.Handler } // Server is used to start a web server that provides a REST API to the setdb @@ -117,7 +117,7 @@ type Server struct { stillRunningMsgFreq time.Duration serverAliveCh chan bool uploadTracker *uploadTracker - storageHandler remove.RemoveHandler + storageHandler remove.Handler mapMu sync.RWMutex creatingCollections map[string]bool diff --git a/server/server_test.go b/server/server_test.go index 75c38a2..6c792b9 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -1574,7 +1574,7 @@ func TestServer(t *testing.T) { client := NewClient(addr, certPath, token) - handler := put.GetLocalHandler() + handler := internal.GetLocalHandler() logger := log15.New() @@ -2044,11 +2044,11 @@ func TestServer(t *testing.T) { switch j { case 0: - So(entry.LastError, ShouldEqual, put.ErrMockStatFail) + So(entry.LastError, ShouldEqual, internal.ErrMockStatFail) case 1: - So(entry.LastError, ShouldEqual, put.ErrMockPutFail) + So(entry.LastError, ShouldEqual, internal.ErrMockPutFail) case 3: - So(entry.LastError, ShouldEqual, put.ErrMockMetaFail) + So(entry.LastError, ShouldEqual, internal.ErrMockMetaFail) } } @@ -2233,13 +2233,13 @@ func TestServer(t *testing.T) { }) Convey("The system warns of possibly stuck uploads", func() { - slowDur := 1200 * time.Millisecond - handler.MakePutSlow(discovers[0], slowDur) - requests, errg := client.GetSomeUploadRequests() So(errg, ShouldBeNil) So(len(requests), ShouldEqual, len(discovers)) + slowDur := 1200 * time.Millisecond + handler.MakePutSlow(requests[0].Remote, slowDur) + p, d := makePutter(t, handler, requests, client) defer d() @@ -2293,13 +2293,14 @@ func TestServer(t *testing.T) { Convey("...but if it is beyond the max stuck time, it is killed", func() { testStart := time.Now() - slowDur := 10 * time.Second - handler.MakePutSlow(discovers[0], slowDur) requests, errg := client.GetSomeUploadRequests() So(errg, ShouldBeNil) So(len(requests), ShouldEqual, len(discovers)) + slowDur := 10 * time.Second + handler.MakePutSlow(requests[0].Remote, slowDur) + p, d := makePutter(t, handler, requests, client) defer d() @@ -2670,7 +2671,7 @@ func TestServer(t *testing.T) { remoteBackupPath := filepath.Join(remoteBackupDir, "remoteDB") - handler = put.GetLocalHandler() + handler = internal.GetLocalHandler() s.EnableRemoteDBBackups(remoteBackupPath, handler) diff --git a/set/set_test.go b/set/set_test.go index 70e37b9..ac26407 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -39,6 +39,7 @@ import ( "github.com/shirou/gopsutil/process" . "github.com/smartystreets/goconvey/convey" gas "github.com/wtsi-hgi/go-authserver" + "github.com/wtsi-hgi/ibackup/baton" "github.com/wtsi-hgi/ibackup/internal" "github.com/wtsi-hgi/ibackup/put" "github.com/wtsi-hgi/ibackup/slack" @@ -1774,7 +1775,7 @@ func TestBackup(t *testing.T) { So(err, ShouldBeNil) remotePath := filepath.Join(remoteDir, "db") - handler := put.GetLocalHandler() + handler := internal.GetLocalHandler() db.EnableRemoteBackups(remotePath, handler) @@ -1797,7 +1798,7 @@ func TestBackup(t *testing.T) { remotePath := filepath.Join(remoteDir, "db") - handler, err := put.GetBatonHandler() + handler, err := baton.GetBatonHandler() So(err, ShouldBeNil) db.EnableRemoteBackups(remotePath, handler) From 891de1645c8b4a9adffff4cab712ce49b89791e4 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Wed, 19 Feb 2025 16:50:39 +0000 Subject: [PATCH 32/35] Make remove.handler use baton.GetBatonHandler() --- baton/baton.go | 47 ++++++++++++++++++++++++++++++++++++++--------- remove/remove.go | 5 +++++ server/server.go | 8 ++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/baton/baton.go b/baton/baton.go index 8ccbf61..19b46d0 100644 --- a/baton/baton.go +++ b/baton/baton.go @@ -52,7 +52,7 @@ const ( extendoLogLevel = logs.ErrorLevel workerPoolSizeCollections = 2 numCollClients = workerPoolSizeCollections - numPutMetaClients = 2 + numPutRemoveMetaClients = 3 collClientMaxIndex = numCollClients - 1 putClientIndex = collClientMaxIndex + 1 metaClientIndex = putClientIndex + 1 @@ -72,9 +72,10 @@ type Baton struct { collCh chan string collErrCh chan error collMu sync.Mutex - PutMetaPool *ex.ClientPool - putClient *ex.Client - MetaClient *ex.Client + //PutMetaPool *ex.ClientPool + putClient *ex.Client + MetaClient *ex.Client + removeClient *ex.Client } // GetBatonHandler returns a Handler that uses Baton to interact with iRODS. If @@ -348,20 +349,48 @@ func (b *Baton) CollectionsDone() error { close(b.collErrCh) b.collRunning = false - pool, clientCh, err := b.connect(numPutMetaClients) + return b.createPutRemoveMetaClients() +} + +func (b *Baton) createPutRemoveMetaClients() error { + pool, clientCh, err := b.connect(numPutRemoveMetaClients) if err != nil { b.collMu.Unlock() return err } - b.PutMetaPool = pool b.putClient = <-clientCh b.MetaClient = <-clientCh + b.removeClient = <-clientCh + + pool.Close() return nil } +// TODO +func (b *Baton) InitClients() error { + if b.putClient == nil && b.MetaClient == nil && b.removeClient == nil { + return b.createPutRemoveMetaClients() + } + + if b.putClient != nil && b.MetaClient != nil && b.removeClient != nil { + return nil + } + + return internal.Error{"Clients are not in the same state", ""} +} + +func (b *Baton) CloseClients() { + var openClients []*ex.Client + for _, client := range []*ex.Client{b.removeClient, b.putClient, b.MetaClient} { + openClients = append(openClients, client) + } + + b.closeConnections(openClients) +} + // closeConnections closes the given connections, with a timeout, ignoring // errors. func (b *Baton) closeConnections(clients []*ex.Client) { @@ -523,9 +552,9 @@ func (b *Baton) AddMeta(path string, meta map[string]string) error { // Cleanup stops our clients and closes our client pool. func (b *Baton) Cleanup() error { - b.closeConnections(append(b.collClients, b.putClient, b.MetaClient)) + b.closeConnections(append(b.collClients, b.putClient, b.removeClient, b.MetaClient)) - b.PutMetaPool.Close() + // b.PutMetaPool.Close() b.collMu.Lock() defer b.collMu.Unlock() @@ -623,5 +652,5 @@ func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, } func (b *Baton) AllClientsStopped() bool { - return !b.PutMetaPool.IsOpen() && !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && b.collClients == nil + return !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && b.collClients == nil } diff --git a/remove/remove.go b/remove/remove.go index 0d0b8c3..be2ab9d 100644 --- a/remove/remove.go +++ b/remove/remove.go @@ -55,6 +55,11 @@ type Handler interface { // QueryMeta return paths to all objects with given metadata QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) + + // TODO + InitClients() error + + CloseClients() } // RemovePathFromSetInIRODS removes the given path from iRODS if the path is not diff --git a/server/server.go b/server/server.go index 6b9323c..381f167 100644 --- a/server/server.go +++ b/server/server.go @@ -219,9 +219,17 @@ func (s *Server) EnableJobSubmission(putCmd, deployment, cwd, queue string, numC // inside removeQueue from iRODS and data base. This function should be called // inside a go routine, so the user API request is not locked. func (s *Server) handleRemoveRequests(reserveGroup string) { + err := s.storageHandler.InitClients() + if err != nil { + s.Logger.Printf("Failed to init lients: %s", err.Error()) + } + for { item, removeReq, err := s.reserveRemoveRequest(reserveGroup) if err != nil { + if s.removeQueue.Stats().Items == 0 { + s.storageHandler.CloseClients() + } break } From 12d625dddc48b4a36f4a923b641d28f2e5c5ffd9 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Thu, 20 Feb 2025 16:18:35 +0000 Subject: [PATCH 33/35] Refactor code to move baton tests into baton pkg and add new tests --- baton/baton.go | 56 +++++----- baton/baton_test.go | 243 ++++++++++++++++++++++++++++++++++++++++++ internal/mock.go | 20 +++- put/put.go | 27 +++-- put/request.go | 8 +- remove/remove.go | 12 ++- remove/remove_test.go | 2 +- 7 files changed, 320 insertions(+), 48 deletions(-) create mode 100644 baton/baton_test.go diff --git a/baton/baton.go b/baton/baton.go index 19b46d0..ba39c4f 100644 --- a/baton/baton.go +++ b/baton/baton.go @@ -335,6 +335,8 @@ func (b *Baton) setClientByIndex(clientIndex int, client *ex.Client) { } } +//TODO + // CollectionsDone closes the connections used for connection creation, and // creates new ones for doing puts and metadata operations. func (b *Baton) CollectionsDone() error { @@ -349,7 +351,9 @@ func (b *Baton) CollectionsDone() error { close(b.collErrCh) b.collRunning = false - return b.createPutRemoveMetaClients() + return nil + + // return b.createPutRemoveMetaClients() } func (b *Baton) createPutRemoveMetaClients() error { @@ -369,23 +373,32 @@ func (b *Baton) createPutRemoveMetaClients() error { return nil } -// TODO +// InitClients sets three clients if they do not currently exist. Returns error +// if some exist and some do not. func (b *Baton) InitClients() error { if b.putClient == nil && b.MetaClient == nil && b.removeClient == nil { return b.createPutRemoveMetaClients() } if b.putClient != nil && b.MetaClient != nil && b.removeClient != nil { + if !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && !b.removeClient.IsRunning() { + return b.createPutRemoveMetaClients() + } + return nil } return internal.Error{"Clients are not in the same state", ""} } +// CloseClients closes remove, put and meta clients. func (b *Baton) CloseClients() { var openClients []*ex.Client + for _, client := range []*ex.Client{b.removeClient, b.putClient, b.MetaClient} { - openClients = append(openClients, client) + if client != nil { + openClients = append(openClients, client) + } } b.closeConnections(openClients) @@ -403,15 +416,13 @@ func (b *Baton) closeConnections(clients []*ex.Client) { } } -// TODO only take remote - // Stat gets mtime and metadata info for the request Remote object. -func (b *Baton) Stat(local, remote string) (bool, map[string]string, error) { +func (b *Baton) Stat(remote string) (bool, map[string]string, error) { var it ex.RodsItem err := TimeoutOp(func() error { var errl error - it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem(local, remote)) + it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem("", remote)) return errl }, "stat failed: "+remote) @@ -428,13 +439,19 @@ func (b *Baton) Stat(local, remote string) (bool, map[string]string, error) { } // requestToRodsItem converts a Request in to an extendo RodsItem without AVUs. +// If you provide an empty local path, it will not be set. func requestToRodsItem(local, remote string) *ex.RodsItem { - return &ex.RodsItem{ - IDirectory: filepath.Dir(local), - IFile: filepath.Base(local), - IPath: filepath.Dir(remote), - IName: filepath.Base(remote), + item := &ex.RodsItem{ + IPath: filepath.Dir(remote), + IName: filepath.Base(remote), } + + if local != "" { + item.IDirectory = filepath.Dir(local) + item.IFile = filepath.Base(local) + } + + return item } // RodsItemToMeta pulls out the AVUs from a RodsItem and returns them as a map. @@ -451,8 +468,8 @@ func RodsItemToMeta(it ex.RodsItem) map[string]string { // Put uploads request Local to the Remote object, overwriting it if it already // exists. It calculates and stores the md5 checksum remotely, comparing to the // local checksum. -func (b *Baton) Put(local, remote string, meta map[string]string) error { - item := requestToRodsItemWithAVUs(local, remote, meta) +func (b *Baton) Put(local, remote string) error { + item := requestToRodsItem(local, remote) // iRODS treats /dev/null specially, so unless that changes we have to check // for it and create a temporary empty file in its place. @@ -483,15 +500,6 @@ func (b *Baton) Put(local, remote string, meta map[string]string) error { return err } -// requestToRodsItemWithAVUs converts a Request in to an extendo RodsItem with -// AVUs. -func requestToRodsItemWithAVUs(local, remote string, meta map[string]string) *ex.RodsItem { - item := requestToRodsItem(local, remote) - item.IAVUs = MetaToAVUs(meta) - - return item -} - func MetaToAVUs(meta map[string]string) []ex.AVU { avus := make([]ex.AVU, len(meta)) i := 0 @@ -645,7 +653,7 @@ func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, paths := make([]string, len(items)) for i, item := range items { - paths[i] = item.IPath + paths[i] = filepath.Join(item.IPath, item.IName) } return paths, err diff --git a/baton/baton_test.go b/baton/baton_test.go new file mode 100644 index 0000000..d562fbd --- /dev/null +++ b/baton/baton_test.go @@ -0,0 +1,243 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Author: Rosie Kern + * Author: Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package baton + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "github.com/wtsi-hgi/ibackup/internal" +) + +func TestBaton(t *testing.T) { + h, errgbh := GetBatonHandler() + if errgbh != nil { + t.Logf("GetBatonHandler error: %s", errgbh) + SkipConvey("Skipping baton tests since couldn't find baton", t, func() {}) + + return + } + + remotePath := os.Getenv("IBACKUP_TEST_COLLECTION") + if remotePath == "" { + SkipConvey("Skipping baton tests since IBACKUP_TEST_COLLECTION is not defined", t, func() {}) + + return + } + + resetIRODS(remotePath) + + localPath := t.TempDir() + + Convey("Given clients on a baton handler", t, func() { + err := h.InitClients() + So(err, ShouldBeNil) + + So(h.removeClient.IsRunning(), ShouldBeTrue) + So(h.MetaClient.IsRunning(), ShouldBeTrue) + So(h.putClient.IsRunning(), ShouldBeTrue) + + meta := map[string]string{ + "ibackup:test:a": "1", + "ibackup:test:b": "2", + } + + Convey("And a local file", func() { + file1local := filepath.Join(localPath, "file1") + file1remote := filepath.Join(remotePath, "file1") + + internal.CreateTestFileOfLength(t, file1local, 1) + + Convey("You can stat a file not in iRODS", func() { + exists, _, errs := h.Stat(file1remote) + So(errs, ShouldBeNil) + So(exists, ShouldBeFalse) + }) + + Convey("You can put the file and its metadata in iRODS", func() { + err = h.Put(file1local, file1remote) + So(err, ShouldBeNil) + + So(isObjectInIRODS(remotePath, "file1"), ShouldBeTrue) + + Convey("And putting a new file in the same location will overwrite existing object", func() { + file2local := filepath.Join(localPath, "file2") + sizeOfFile2 := 10 + + internal.CreateTestFileOfLength(t, file2local, sizeOfFile2) + + err = h.Put(file2local, file1remote) + So(err, ShouldBeNil) + + So(getSizeOfObject(file1remote), ShouldEqual, sizeOfFile2) + }) + + Convey("And then you can add metadata to it", func() { + errm := h.AddMeta(file1remote, meta) + So(errm, ShouldBeNil) + + So(getRemoteMeta(file1remote), ShouldContainSubstring, "ibackup:test:a") + So(getRemoteMeta(file1remote), ShouldContainSubstring, "ibackup:test:b") + }) + + Convey("And given metadata on the file in iRODS", func() { + for k, v := range meta { + addRemoteMeta(file1remote, k, v) + } + + Convey("You can get the metadata from a file in iRODS", func() { + fileMeta, errm := h.GetMeta(file1remote) + So(errm, ShouldBeNil) + + So(fileMeta, ShouldResemble, meta) + }) + + Convey("You can stat a file and get its metadata", func() { + exists, fileMeta, errm := h.Stat(file1remote) + So(errm, ShouldBeNil) + So(exists, ShouldBeTrue) + So(fileMeta, ShouldResemble, meta) + }) + + Convey("You can query if files contain specific metadata", func() { + files, errq := h.QueryMeta(remotePath, map[string]string{"ibackup:test:a": "1"}) + So(errq, ShouldBeNil) + + So(len(files), ShouldEqual, 1) + So(files, ShouldContain, file1remote) + + files, errq = h.QueryMeta(remotePath, map[string]string{"ibackup:test:a": "2"}) + So(errq, ShouldBeNil) + + So(len(files), ShouldEqual, 0) + }) + + Convey("You can remove specific metadata from a file in iRODS", func() { + errm := h.RemoveMeta(file1remote, map[string]string{"ibackup:test:a": "1"}) + So(errm, ShouldBeNil) + + fileMeta, errm := h.GetMeta(file1remote) + So(errm, ShouldBeNil) + + So(fileMeta, ShouldResemble, map[string]string{"ibackup:test:b": "2"}) + }) + }) + + Convey("And then you can remove it from iRODS", func() { + err = h.RemoveFile(file1remote) + So(err, ShouldBeNil) + + So(isObjectInIRODS(remotePath, "file1"), ShouldBeFalse) + }) + }) + }) + + Convey("You can open collection clients and put an empty dir in iRODS", func() { + dir1remote := filepath.Join(remotePath, "dir1") + err = h.EnsureCollection(dir1remote) + So(err, ShouldBeNil) + + So(h.collPool.IsOpen(), ShouldBeTrue) + + for _, client := range h.collClients { + So(client.IsRunning(), ShouldBeTrue) + } + + So(isObjectInIRODS(remotePath, "dir1"), ShouldBeTrue) + + Convey("Then you can close the collection clients", func() { + err = h.CollectionsDone() + So(err, ShouldBeNil) + + So(h.collPool.IsOpen(), ShouldBeFalse) + So(len(h.collClients), ShouldEqual, 0) + + Convey("And then you can remove the dir from iRODS", func() { + err = h.RemoveDir(dir1remote) + So(err, ShouldBeNil) + + So(isObjectInIRODS(remotePath, "dir1"), ShouldBeFalse) + }) + }) + }) + + Convey("You can close those clients", func() { + h.CloseClients() + + So(h.AllClientsStopped(), ShouldBeTrue) + }) + }) +} + +func resetIRODS(remotePath string) { + if remotePath == "" { + return + } + + exec.Command("irm", "-r", remotePath).Run() //nolint:errcheck + + exec.Command("imkdir", remotePath).Run() //nolint:errcheck +} + +func isObjectInIRODS(remotePath, name string) bool { + output, err := exec.Command("ils", remotePath).CombinedOutput() + So(err, ShouldBeNil) + + return strings.Contains(string(output), name) +} + +func getRemoteMeta(path string) string { + output, err := exec.Command("imeta", "ls", "-d", path).CombinedOutput() + So(err, ShouldBeNil) + + return string(output) +} + +func addRemoteMeta(path, key, val string) { + output, err := exec.Command("imeta", "add", "-d", path, key, val).CombinedOutput() + if strings.Contains(string(output), "CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME") { + return + } + + So(err, ShouldBeNil) +} + +func getSizeOfObject(path string) int { + output, err := exec.Command("ils", "-l", path).CombinedOutput() + So(err, ShouldBeNil) + + cols := strings.Fields(string(output)) + size, err := strconv.Atoi(cols[3]) + So(err, ShouldBeNil) + + return size +} diff --git a/internal/mock.go b/internal/mock.go index 5d10563..4074391 100644 --- a/internal/mock.go +++ b/internal/mock.go @@ -80,10 +80,8 @@ func (l *LocalHandler) EnsureCollection(dir string) error { return os.MkdirAll(dir, UserPerms) } -// CollectionsDone says we connected and prepares us for metadata handling. +// CollectionsDone TODO func (l *LocalHandler) CollectionsDone() error { - l.Connected = true - return nil } @@ -95,7 +93,7 @@ func (l *LocalHandler) MakeStatFail(remote string) { // Stat returns info about the Remote file, which is a local file on disk. // Returns an error if statFail == Remote. -func (l *LocalHandler) Stat(_, remote string) (bool, map[string]string, error) { +func (l *LocalHandler) Stat(remote string) (bool, map[string]string, error) { if l.statFail == remote { return false, nil, Error{ErrMockStatFail, ""} } @@ -136,7 +134,7 @@ func (l *LocalHandler) MakePutSlow(remote string, dur time.Duration) { } // Put just copies from Local to Remote. Returns an error if putFail == Remote. -func (l *LocalHandler) Put(local, remote string, meta map[string]string) error { +func (l *LocalHandler) Put(local, remote string) error { if l.putFail == remote { return Error{ErrMockPutFail, ""} } @@ -293,3 +291,15 @@ func doesMetaContainMeta(sourceMeta, targetMeta map[string]string) bool { return valid } + +// InitClients says we connected. +func (l *LocalHandler) InitClients() error { + l.Connected = true + + return nil +} + +// CloseClients says we closed connections. +func (l *LocalHandler) CloseClients() { + l.Connected = false +} diff --git a/put/put.go b/put/put.go index bb36229..69c6249 100644 --- a/put/put.go +++ b/put/put.go @@ -70,19 +70,22 @@ type Handler interface { // CollectionsDone is called after all collections have been created. This // method can do things like cleaning up connections created for collection - // creation. It can also create new connections for subsequent Put() and - // *Meta calls that are likely to occur. + // creation. CollectionsDone() error - // Stat checks if the Request's Remote object exists. If it does, records - // its metadata in the returned ObjectInfo. Returns an error if there was a - // problem finding out information (but not if the object does not exist). - Stat(local, remote string) (bool, map[string]string, error) + // InitClients creates new connections for subsequent remove and meta + // commands. + InitClients() error - // Put uploads the Request's Local file to the Remote location, overwriting - // any existing object, and ensuring that a locally calculated and remotely + // Stat checks if the provided Remote object exists. If it does, records its + // metadata and returns it. Returns an error if there was a problem finding + // out information (but not if the object does not exist). + Stat(remote string) (bool, map[string]string, error) + + // Put uploads the Local file to the Remote location, overwriting any + // existing object, and ensuring that a locally calculated and remotely // calculated md5 checksum match. - Put(local, remote string, meta map[string]string) error + Put(local, remote string) error // RemoveMeta deletes the given metadata from the given object. RemoveMeta(path string, meta map[string]string) error @@ -96,6 +99,7 @@ type Handler interface { // needed. Cleanup() error + // GetMeta returns the meta for a given path in iRODS. GetMeta(path string) (map[string]string, error) } @@ -240,6 +244,11 @@ func (p *Putter) CreateCollections() error { merr = multierror.Append(merr, cdErr) } + err := p.handler.InitClients() + if err != nil { + merr = multierror.Append(merr, err) + } + return merr.ErrorOrNil() } diff --git a/put/request.go b/put/request.go index 4e21e5c..895cdec 100644 --- a/put/request.go +++ b/put/request.go @@ -279,7 +279,7 @@ func (r *Request) StatAndAssociateStandardMetadata(lInfo *ObjectInfo, handler Ha func statAndAssociateStandardMetadata(request *Request, diskMeta map[string]string, handler Handler) (*ObjectInfo, error) { - exists, meta, err := handler.Stat(request.LocalDataPath(), request.Remote) + exists, meta, err := handler.Stat(request.Remote) if err != nil { return nil, err } @@ -346,16 +346,16 @@ func (r *Request) addMeta(handler Handler, toAdd map[string]string) error { // to Remote, with linking metadata. func (r *Request) Put(handler Handler) error { if r.Hardlink == "" { - return handler.Put(r.LocalDataPath(), r.Remote, r.Meta.Metadata()) + return handler.Put(r.LocalDataPath(), r.Remote) } if !r.onlyUploadEmptyFile { - if err := handler.Put(r.inodeRequest.LocalDataPath(), r.inodeRequest.Remote, r.inodeRequest.Meta.Metadata()); err != nil { + if err := handler.Put(r.inodeRequest.LocalDataPath(), r.inodeRequest.Remote); err != nil { return err } } - return handler.Put(r.emptyFileRequest.LocalDataPath(), r.emptyFileRequest.Remote, r.emptyFileRequest.Meta.Metadata()) + return handler.Put(r.emptyFileRequest.LocalDataPath(), r.emptyFileRequest.Remote) } // PathTransformer is a function that given a local path, returns the diff --git a/remove/remove.go b/remove/remove.go index be2ab9d..cefe27a 100644 --- a/remove/remove.go +++ b/remove/remove.go @@ -44,21 +44,23 @@ type Handler interface { // object. AddMeta(path string, meta map[string]string) error - // TODO + // GetMeta returns the meta for a given path in iRODS. GetMeta(path string) (map[string]string, error) - // RemoveDir deletes a given empty folder + // RemoveDir deletes a given empty folder. RemoveDir(path string) error - // RemoveFile deletes a given file + // RemoveFile deletes a given file. RemoveFile(path string) error - // QueryMeta return paths to all objects with given metadata + // QueryMeta return paths to all objects with given metadata. QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) - // TODO + // InitClients creates new connections for subsequent remove and meta + // commands. InitClients() error + // CloseClients closes all open connections. CloseClients() } diff --git a/remove/remove_test.go b/remove/remove_test.go index 36b13ed..3dc2686 100644 --- a/remove/remove_test.go +++ b/remove/remove_test.go @@ -98,7 +98,7 @@ func uploadRequest(t *testing.T, lh *internal.LocalHandler, request *put.Request err := os.MkdirAll(filepath.Dir(request.Remote), userPerms) So(err, ShouldBeNil) - err = lh.Put(request.LocalDataPath(), request.Remote, request.Meta.Metadata()) + err = lh.Put(request.LocalDataPath(), request.Remote) So(err, ShouldBeNil) request.Meta.LocalMeta[put.MetaKeyRequester] = request.Requester From 22ac24f8bbeaeecee071e4525cc64595ab4be3c2 Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Thu, 20 Feb 2025 16:51:19 +0000 Subject: [PATCH 34/35] Add tests for mock (localHandler) --- baton/baton_test.go | 2 +- internal/mock.go | 2 + internal/mock_test.go | 177 ++++++++++++++++++++++++++++++++++++++++++ remove/remove_test.go | 53 +------------ 4 files changed, 181 insertions(+), 53 deletions(-) create mode 100644 internal/mock_test.go diff --git a/baton/baton_test.go b/baton/baton_test.go index d562fbd..34b76dd 100644 --- a/baton/baton_test.go +++ b/baton/baton_test.go @@ -83,7 +83,7 @@ func TestBaton(t *testing.T) { So(exists, ShouldBeFalse) }) - Convey("You can put the file and its metadata in iRODS", func() { + Convey("You can put the file in iRODS", func() { err = h.Put(file1local, file1remote) So(err, ShouldBeNil) diff --git a/internal/mock.go b/internal/mock.go index 4074391..7936d4d 100644 --- a/internal/mock.go +++ b/internal/mock.go @@ -82,6 +82,8 @@ func (l *LocalHandler) EnsureCollection(dir string) error { // CollectionsDone TODO func (l *LocalHandler) CollectionsDone() error { + l.Collections = nil + return nil } diff --git a/internal/mock_test.go b/internal/mock_test.go new file mode 100644 index 0000000..f4605b6 --- /dev/null +++ b/internal/mock_test.go @@ -0,0 +1,177 @@ +/******************************************************************************* + * Copyright (c) 2025 Genome Research Ltd. + * + * Author: Rosie Kern + * Author: Iaroslav Popov + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + ******************************************************************************/ + +package internal + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestRemoveMock(t *testing.T) { + Convey("Given a mock RemoveHandler", t, func() { + lh := GetLocalHandler() + + err := lh.InitClients() + So(err, ShouldBeNil) + + So(lh.Connected, ShouldBeTrue) + + sourceDir := t.TempDir() + destDir := t.TempDir() + + meta := map[string]string{ + "ibackup:test:a": "1", + "ibackup:test:b": "2", + } + + Convey("And a local file", func() { + file1local := filepath.Join(sourceDir, "file1") + file1remote := filepath.Join(destDir, "file1") + + CreateTestFileOfLength(t, file1local, 1) + + Convey("You can stat a file that's not been uploaded", func() { + exists, _, errs := lh.Stat(file1remote) + So(errs, ShouldBeNil) + So(exists, ShouldBeFalse) + }) + + Convey("You can upload the file", func() { + err = lh.Put(file1local, file1remote) + So(err, ShouldBeNil) + + _, err = os.Stat(file1remote) + So(err, ShouldBeNil) + + Convey("And putting a new file in the same location will overwrite existing object", func() { + file2local := filepath.Join(sourceDir, "file2") + sizeOfFile2 := 10 + + CreateTestFileOfLength(t, file2local, sizeOfFile2) + + err = lh.Put(file2local, file1remote) + So(err, ShouldBeNil) + + fileInfo, errs := os.Stat(file1remote) + So(errs, ShouldBeNil) + So(fileInfo.Size(), ShouldEqual, sizeOfFile2) + }) + + Convey("And then you can add metadata to it", func() { + errm := lh.AddMeta(file1remote, meta) + So(errm, ShouldBeNil) + + So(lh.Meta[file1remote], ShouldResemble, meta) + }) + + Convey("And given metadata on the uploaded file", func() { + lh.Meta[file1remote] = meta + + Convey("You can get the metadata from the uploaded file", func() { + fileMeta, errm := lh.GetMeta(file1remote) + So(errm, ShouldBeNil) + + So(fileMeta, ShouldResemble, meta) + }) + + Convey("You can stat a file and get its metadata", func() { + exists, fileMeta, errm := lh.Stat(file1remote) + So(errm, ShouldBeNil) + So(exists, ShouldBeTrue) + So(fileMeta, ShouldResemble, meta) + }) + + Convey("You can query if files contain specific metadata", func() { + files, errq := lh.QueryMeta(destDir, map[string]string{"ibackup:test:a": "1"}) + So(errq, ShouldBeNil) + + So(len(files), ShouldEqual, 1) + So(files, ShouldContain, file1remote) + + files, errq = lh.QueryMeta(destDir, map[string]string{"ibackup:test:a": "2"}) + So(errq, ShouldBeNil) + + So(len(files), ShouldEqual, 0) + }) + + Convey("You can remove specific metadata from an uploaded file", func() { + errm := lh.RemoveMeta(file1remote, map[string]string{"ibackup:test:a": "1"}) + So(errm, ShouldBeNil) + + fileMeta, errm := lh.GetMeta(file1remote) + So(errm, ShouldBeNil) + + So(fileMeta, ShouldResemble, map[string]string{"ibackup:test:b": "2"}) + }) + }) + + Convey("And then you can remove the uploaded file", func() { + err = lh.RemoveFile(file1remote) + So(err, ShouldBeNil) + + _, err = os.Stat(file1remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") + }) + }) + }) + + Convey("You can open collection clients and upload an empty dir", func() { + dir1remote := filepath.Join(destDir, "dir1") + err = lh.EnsureCollection(dir1remote) + So(err, ShouldBeNil) + + So(len(lh.Collections), ShouldEqual, 1) + + _, err = os.Stat(dir1remote) + So(err, ShouldBeNil) + + Convey("Then you can close the collection clients", func() { + err = lh.CollectionsDone() + So(err, ShouldBeNil) + + So(len(lh.Collections), ShouldEqual, 0) + + Convey("And then you can remove the uploaded dir", func() { + err = lh.RemoveDir(dir1remote) + So(err, ShouldBeNil) + + _, err = os.Stat(dir1remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") + }) + }) + }) + + Convey("You can close those clients", func() { + lh.CloseClients() + + So(lh.Connected, ShouldBeFalse) + }) + }) +} diff --git a/remove/remove_test.go b/remove/remove_test.go index 3dc2686..9d4c092 100644 --- a/remove/remove_test.go +++ b/remove/remove_test.go @@ -40,58 +40,7 @@ import ( const userPerms = 0700 func TestRemoveMock(t *testing.T) { - Convey("Given a mock RemoveHandler", t, func() { - requests, expectedCollections := makeMockRequests(t) - - lh := internal.GetLocalHandler() - - requests[0].Requester = "John" - - for _, request := range requests { - uploadRequest(t, lh, request) - } - - Convey("RemoveFile removes a file", func() { - filePath := requests[0].Remote - - err := lh.RemoveFile(filePath) - So(err, ShouldBeNil) - - _, err = os.Stat(filePath) - So(err, ShouldNotBeNil) - - So(lh.Meta[filePath], ShouldBeNil) - - Convey("RemoveDir removes an empty directory", func() { - dirPath := filepath.Dir(filePath) - - err = lh.RemoveDir(dirPath) - So(err, ShouldBeNil) - - _, err = os.Stat(dirPath) - So(err, ShouldNotBeNil) - }) - }) - - Convey("queryMeta returns all paths with matching metadata", func() { - paths, errq := lh.QueryMeta("", map[string]string{"a": "1"}) - So(errq, ShouldBeNil) - - So(len(paths), ShouldEqual, 5) - - paths, err := lh.QueryMeta("", map[string]string{put.MetaKeyRequester: requests[0].Requester}) - So(err, ShouldBeNil) - - So(len(paths), ShouldEqual, 1) - - Convey("queryMeta only returns paths in the provided scope", func() { - paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"}) - So(errq, ShouldBeNil) - - So(len(paths), ShouldEqual, 2) - }) - }) - }) + //TODO } func uploadRequest(t *testing.T, lh *internal.LocalHandler, request *put.Request) { From 78ed27eea247b5ca8c8d8e0a711348133ffbf0ba Mon Sep 17 00:00:00 2001 From: Rosie Kern Date: Fri, 21 Feb 2025 15:18:46 +0000 Subject: [PATCH 35/35] Add tests for remove pkg and refactor remove functions --- baton/baton.go | 66 ++++++-------------- baton/baton_test.go | 2 +- internal/mock.go | 4 ++ main_test.go | 22 +++++++ remove/remove.go | 41 ++++++------ remove/remove_test.go | 142 +++++++++++++++++++++++++++--------------- server/server.go | 2 +- server/setdb.go | 19 ++++-- set/db.go | 3 +- 9 files changed, 172 insertions(+), 129 deletions(-) diff --git a/baton/baton.go b/baton/baton.go index ba39c4f..6bd8a9a 100644 --- a/baton/baton.go +++ b/baton/baton.go @@ -74,7 +74,7 @@ type Baton struct { collMu sync.Mutex //PutMetaPool *ex.ClientPool putClient *ex.Client - MetaClient *ex.Client + metaClient *ex.Client removeClient *ex.Client } @@ -318,7 +318,7 @@ func (b *Baton) getClientByIndex(clientIndex int) *ex.Client { case putClientIndex: return b.putClient case metaClientIndex: - return b.MetaClient + return b.metaClient default: return b.collClients[clientIndex] } @@ -329,7 +329,7 @@ func (b *Baton) setClientByIndex(clientIndex int, client *ex.Client) { case putClientIndex: b.putClient = client case metaClientIndex: - b.MetaClient = client + b.metaClient = client default: b.collClients[clientIndex] = client } @@ -365,7 +365,7 @@ func (b *Baton) createPutRemoveMetaClients() error { } b.putClient = <-clientCh - b.MetaClient = <-clientCh + b.metaClient = <-clientCh b.removeClient = <-clientCh pool.Close() @@ -376,12 +376,12 @@ func (b *Baton) createPutRemoveMetaClients() error { // InitClients sets three clients if they do not currently exist. Returns error // if some exist and some do not. func (b *Baton) InitClients() error { - if b.putClient == nil && b.MetaClient == nil && b.removeClient == nil { + if b.putClient == nil && b.metaClient == nil && b.removeClient == nil { return b.createPutRemoveMetaClients() } - if b.putClient != nil && b.MetaClient != nil && b.removeClient != nil { - if !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && !b.removeClient.IsRunning() { + if b.putClient != nil && b.metaClient != nil && b.removeClient != nil { + if !b.putClient.IsRunning() && !b.metaClient.IsRunning() && !b.removeClient.IsRunning() { return b.createPutRemoveMetaClients() } @@ -395,7 +395,7 @@ func (b *Baton) InitClients() error { func (b *Baton) CloseClients() { var openClients []*ex.Client - for _, client := range []*ex.Client{b.removeClient, b.putClient, b.MetaClient} { + for _, client := range []*ex.Client{b.removeClient, b.putClient, b.metaClient} { if client != nil { openClients = append(openClients, client) } @@ -422,7 +422,7 @@ func (b *Baton) Stat(remote string) (bool, map[string]string, error) { err := TimeoutOp(func() error { var errl error - it, errl = b.MetaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem("", remote)) + it, errl = b.metaClient.ListItem(ex.Args{Timestamp: true, AVU: true}, *requestToRodsItem("", remote)) return errl }, "stat failed: "+remote) @@ -518,7 +518,7 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { it.IAVUs = MetaToAVUs(meta) err := TimeoutOp(func() error { - _, errl := b.MetaClient.MetaRem(ex.Args{}, *it) + _, errl := b.metaClient.MetaRem(ex.Args{}, *it) return errl }, "remove meta error: "+path) @@ -528,7 +528,7 @@ func (b *Baton) RemoveMeta(path string, meta map[string]string) error { // GetMeta gets all the metadata for the given object in iRODS. func (b *Baton) GetMeta(path string) (map[string]string, error) { - it, err := b.MetaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ + it, err := b.metaClient.ListItem(ex.Args{AVU: true, Timestamp: true, Size: true}, ex.RodsItem{ IPath: filepath.Dir(path), IName: filepath.Base(path), }) @@ -550,7 +550,7 @@ func (b *Baton) AddMeta(path string, meta map[string]string) error { it.IAVUs = MetaToAVUs(meta) err := TimeoutOp(func() error { - _, errl := b.MetaClient.MetaAdd(ex.Args{}, *it) + _, errl := b.metaClient.MetaAdd(ex.Args{}, *it) return errl }, "add meta error: "+path) @@ -560,7 +560,7 @@ func (b *Baton) AddMeta(path string, meta map[string]string) error { // Cleanup stops our clients and closes our client pool. func (b *Baton) Cleanup() error { - b.closeConnections(append(b.collClients, b.putClient, b.removeClient, b.MetaClient)) + b.closeConnections(append(b.collClients, b.putClient, b.removeClient, b.metaClient)) // b.PutMetaPool.Close() @@ -577,41 +577,11 @@ func (b *Baton) Cleanup() error { return nil } -// GetBatonHandlerWithMetaClient returns a Handler that uses Baton to interact -// with iRODS and contains a meta client for interacting with metadata. If you -// don't have baton-do in your PATH, you'll get an error. -// func GetBatonHandlerWithMetaClient() (*Baton, error) { -// setupExtendoLogger() - -// _, err := ex.FindBaton() -// if err != nil { -// return nil, err -// } - -// params := ex.DefaultClientPoolParams -// params.MaxSize = 1 -// pool := ex.NewClientPool(params, "") - -// metaClient, err := pool.Get() -// if err != nil { -// return nil, fmt.Errorf("failed to get metaClient: %w", err) -// } - -// baton := &Baton{ -// Baton: put.Baton{ -// PutMetaPool: pool, -// MetaClient: metaClient, -// }, -// } - -// return baton, nil -// } - func (b *Baton) RemoveFile(path string) error { it := RemotePathToRodsItem(path) err := TimeoutOp(func() error { - _, errl := b.MetaClient.RemObj(ex.Args{}, *it) + _, errl := b.removeClient.RemObj(ex.Args{}, *it) return errl }, "remove file error: "+path) @@ -626,7 +596,7 @@ func (b *Baton) RemoveDir(path string) error { } err := TimeoutOp(func() error { - _, errl := b.MetaClient.RemDir(ex.Args{}, *it) + _, errl := b.removeClient.RemDir(ex.Args{}, *it) return errl }, "remove meta error: "+path) @@ -634,6 +604,8 @@ func (b *Baton) RemoveDir(path string) error { return err } +// QueryMeta return paths to all objects with given metadata inside the provided +// scope. func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { it := &ex.RodsItem{ IPath: dirToSearch, @@ -645,7 +617,7 @@ func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, var err error err = TimeoutOp(func() error { - items, err = b.MetaClient.MetaQuery(ex.Args{Object: true}, *it) + items, err = b.metaClient.MetaQuery(ex.Args{Object: true}, *it) return err }, "query meta error: "+dirToSearch) @@ -660,5 +632,5 @@ func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, } func (b *Baton) AllClientsStopped() bool { - return !b.putClient.IsRunning() && !b.MetaClient.IsRunning() && b.collClients == nil + return !b.putClient.IsRunning() && !b.metaClient.IsRunning() && b.collClients == nil } diff --git a/baton/baton_test.go b/baton/baton_test.go index 34b76dd..661b9d5 100644 --- a/baton/baton_test.go +++ b/baton/baton_test.go @@ -63,7 +63,7 @@ func TestBaton(t *testing.T) { So(err, ShouldBeNil) So(h.removeClient.IsRunning(), ShouldBeTrue) - So(h.MetaClient.IsRunning(), ShouldBeTrue) + So(h.metaClient.IsRunning(), ShouldBeTrue) So(h.putClient.IsRunning(), ShouldBeTrue) meta := map[string]string{ diff --git a/internal/mock.go b/internal/mock.go index 7936d4d..8c681a8 100644 --- a/internal/mock.go +++ b/internal/mock.go @@ -254,16 +254,20 @@ func (l *LocalHandler) GetMeta(path string) (map[string]string, error) { return meta, nil } +// RemoveDir removes the empty dir. func (l *LocalHandler) RemoveDir(path string) error { return os.Remove(path) } +// RemoveFile removes the file and its metadata. func (l *LocalHandler) RemoveFile(path string) error { delete(l.Meta, path) return os.Remove(path) } +// QueryMeta return paths to all objects with given metadata inside the provided +// scope. func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) { var objects []string diff --git a/main_test.go b/main_test.go index 3700bec..d839db6 100644 --- a/main_test.go +++ b/main_test.go @@ -2002,6 +2002,8 @@ func TestRemove(t *testing.T) { dir1 := filepath.Join(testDir, "dir") dir2 := filepath.Join(path, "path/to/other/dir/") + //dir3 := filepath.Join(dir1, "dir") + tempTestFileOfPaths, err := os.CreateTemp(dir, "testFileSet") So(err, ShouldBeNil) @@ -2011,15 +2013,21 @@ func TestRemove(t *testing.T) { err = os.MkdirAll(dir2, 0755) So(err, ShouldBeNil) + // err = os.MkdirAll(dir3, 0755) + // So(err, ShouldBeNil) + file1 := filepath.Join(path, "file1") file2 := filepath.Join(path, "file2") file3 := filepath.Join(dir1, "file3") file4 := filepath.Join(testDir, "dir_not_removed") + // file5 := filepath.Join(dir3, "file5") + internal.CreateTestFile(t, file1, "some data1") internal.CreateTestFile(t, file2, "some data2") internal.CreateTestFile(t, file3, "some data3") internal.CreateTestFile(t, file4, "some data3") + // internal.CreateTestFile(t, file5, "some data3") err = os.Link(file1, linkPath) So(err, ShouldBeNil) @@ -2117,6 +2125,20 @@ func TestRemove(t *testing.T) { 0, dir2+" => ") }) + // TODO test fails + // SkipConvey("Remove removes the dir even if it is not specified in the set but is part of it", func() { + // s.removePath(t, setName, dir3, 2) + + // s.confirmOutputContains(t, []string{"status", "--name", setName, "-d"}, + // 0, dir2) + + // s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + // 0, dir3+"/") + + // s.confirmOutputDoesNotContain(t, []string{"status", "--name", setName, "-d"}, + // 0, dir3+" => ") + // }) + Convey("Remove takes a flag --items and removes all provided files and dirs from the set", func() { tempTestFileOfPathsToRemove, errt := os.CreateTemp(dir, "testFileSet") So(errt, ShouldBeNil) diff --git a/remove/remove.go b/remove/remove.go index cefe27a..041aef2 100644 --- a/remove/remove.go +++ b/remove/remove.go @@ -24,7 +24,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -// package remove is used to interact with iRODS. +// package remove is used to remove from remote storage. package remove @@ -44,7 +44,7 @@ type Handler interface { // object. AddMeta(path string, meta map[string]string) error - // GetMeta returns the meta for a given path in iRODS. + // GetMeta returns the meta for a given remote path. GetMeta(path string) (map[string]string, error) // RemoveDir deletes a given empty folder. @@ -53,7 +53,8 @@ type Handler interface { // RemoveFile deletes a given file. RemoveFile(path string) error - // QueryMeta return paths to all objects with given metadata. + // QueryMeta return paths to all objects with given metadata inside the + // provided scope. QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) // InitClients creates new connections for subsequent remove and meta @@ -64,15 +65,10 @@ type Handler interface { CloseClients() } -// RemovePathFromSetInIRODS removes the given path from iRODS if the path is not -// associated with any other sets. Otherwise it updates the iRODS metadata for -// the path to not include the given set. -func RemovePathFromSetInIRODS(handler Handler, transformer put.PathTransformer, path string, +// UpdateSetsAndRequestersOnRemoteFile updates the given file's metadata in +// remote storage for the path with the given sets and requesters. +func UpdateSetsAndRequestersOnRemoteFile(handler Handler, path string, sets, requesters []string, meta map[string]string) error { - if len(sets) == 0 { - return handleHardlinkAndRemoveFromIRODS(handler, path, transformer, meta) - } - metaToRemove := map[string]string{ put.MetaKeySets: meta[put.MetaKeySets], put.MetaKeyRequester: meta[put.MetaKeyRequester], @@ -95,10 +91,11 @@ func RemovePathFromSetInIRODS(handler Handler, transformer put.PathTransformer, return handler.AddMeta(path, newMeta) } -// handleHardLinkAndRemoveFromIRODS removes the given path from iRODS. If the -// path is found to be a hardlink, it checks if there are other hardlinks to the -// same file, if not, it removes the file. -func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer put.PathTransformer, +// RemoveRemoteFileAndHandleHardlink removes the given path from remote storage. +// If the path is found to be a hardlink, it checks if there are other hardlinks +// (inside the provided dir to search) to the same file, if not, it removes the +// file. +func RemoveRemoteFileAndHandleHardlink(handler Handler, path string, dirToSearch string, //nolint:revive meta map[string]string) error { err := handler.RemoveFile(path) if err != nil { @@ -109,12 +106,9 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return nil } - dirToSearch, err := transformer("/") - if err != nil { - return err - } - - items, err := handler.QueryMeta(dirToSearch, map[string]string{put.MetaKeyRemoteHardlink: meta[put.MetaKeyRemoteHardlink]}) + items, err := handler.QueryMeta(dirToSearch, map[string]string{ + put.MetaKeyRemoteHardlink: meta[put.MetaKeyRemoteHardlink], + }) if err != nil { return err } @@ -126,8 +120,9 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer return handler.RemoveFile(meta[put.MetaKeyRemoteHardlink]) } -// RemoveDirFromIRODS removes the remote path of a given directory from iRODS. -func RemoveDirFromIRODS(handler Handler, path string, transformer put.PathTransformer) error { +// RemoveRemoteDir removes the remote path of a given directory from the remote +// storage. +func RemoveRemoteDir(handler Handler, path string, transformer put.PathTransformer) error { //nolint:revive rpath, err := transformer(path) if err != nil { return err diff --git a/remove/remove_test.go b/remove/remove_test.go index 9d4c092..8f2127b 100644 --- a/remove/remove_test.go +++ b/remove/remove_test.go @@ -29,7 +29,6 @@ package remove import ( "os" "path/filepath" - "strings" "testing" . "github.com/smartystreets/goconvey/convey" @@ -40,69 +39,110 @@ import ( const userPerms = 0700 func TestRemoveMock(t *testing.T) { - //TODO -} + Convey("Given a connected local handler", t, func() { + lh := internal.GetLocalHandler() -func uploadRequest(t *testing.T, lh *internal.LocalHandler, request *put.Request) { - err := os.MkdirAll(filepath.Dir(request.Remote), userPerms) - So(err, ShouldBeNil) + sourceDir := t.TempDir() + destDir := t.TempDir() - err = lh.Put(request.LocalDataPath(), request.Remote) - So(err, ShouldBeNil) + transformer := put.PrefixTransformer(sourceDir, destDir) - request.Meta.LocalMeta[put.MetaKeyRequester] = request.Requester - err = lh.AddMeta(request.Remote, request.Meta.LocalMeta) - So(err, ShouldBeNil) -} + dirToSearch, err := transformer("/") + So(err, ShouldBeNil) -// makeMockRequests creates some local directories and files, and returns -// requests that all share the same metadata, with remotes pointing to another -// local temp directory (but the remote sub-directories are not created). -// Also returns the execpted remote directories that would have to be created. -func makeMockRequests(t *testing.T) ([]*put.Request, []string) { - t.Helper() + Convey("Given an uploaded empty dir", func() { + dir1local := filepath.Join(sourceDir, "dir") + dir1remote := filepath.Join(destDir, "dir") - sourceDir := t.TempDir() - destDir := t.TempDir() + err = os.MkdirAll(dir1remote, userPerms) + So(err, ShouldBeNil) - requests := makeTestRequests(t, sourceDir, destDir) + Convey("You can remove a remote folder using the local path", func() { + err = RemoveRemoteDir(lh, dir1local, transformer) + So(err, ShouldBeNil) - return requests, []string{ - filepath.Join(destDir, "a", "b", "c"), - filepath.Join(destDir, "a", "b", "d", "e"), - } -} + _, err = os.Stat(dir1remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") + }) + }) + + Convey("Given a file in two sets", func() { + file1remote := filepath.Join(destDir, "file1") + + internal.CreateTestFileOfLength(t, file1remote, 1) + + meta := map[string]string{ + put.MetaKeyRequester: "testUser1,testUser2", + put.MetaKeySets: "set1,set2", + } + + err = lh.AddMeta(file1remote, meta) + So(err, ShouldBeNil) + + Convey("You can update the metadata for sets and requesters on the remote file", func() { + err = UpdateSetsAndRequestersOnRemoteFile(lh, file1remote, []string{"set2"}, []string{"testUser2"}, meta) + So(err, ShouldBeNil) + + fileMeta, errg := lh.GetMeta(file1remote) + So(errg, ShouldBeNil) + + So(fileMeta, ShouldResemble, + map[string]string{ + put.MetaKeyRequester: "testUser2", + put.MetaKeySets: "set2", + }) + }) + + Convey("You can remove the file from remote", func() { + err = RemoveRemoteFileAndHandleHardlink(lh, file1remote, dirToSearch, meta) + So(err, ShouldBeNil) + + _, err = os.Stat(file1remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") + }) + + Convey("And given two hardlinks to the same file", func() { + link1remote := filepath.Join(destDir, "link1") + link2remote := filepath.Join(destDir, "link2") + inodeRemote := filepath.Join(destDir, "inode") + + internal.CreateTestFileOfLength(t, link1remote, 1) + internal.CreateTestFileOfLength(t, link2remote, 1) + internal.CreateTestFileOfLength(t, inodeRemote, 1) + + hardlinkMeta := map[string]string{ + put.MetaKeyHardlink: "hardlink", + put.MetaKeyRemoteHardlink: inodeRemote, + } -func makeTestRequests(t *testing.T, sourceDir, destDir string) []*put.Request { - t.Helper() + err = lh.AddMeta(link1remote, hardlinkMeta) + So(err, ShouldBeNil) - sourcePaths := []string{ - filepath.Join(sourceDir, "a", "b", "c", "file.1"), - filepath.Join(sourceDir, "a", "b", "file.2"), - filepath.Join(sourceDir, "a", "b", "d", "file.3"), - filepath.Join(sourceDir, "a", "b", "d", "e", "file.4"), - filepath.Join(sourceDir, "a", "b", "d", "e", "file.5"), - } + err = lh.AddMeta(link2remote, hardlinkMeta) + So(err, ShouldBeNil) - requests := make([]*put.Request, len(sourcePaths)) - localMeta := map[string]string{"a": "1", "b": "2"} + Convey("You can remove the first hardlink and the inode file will stay", func() { + err = RemoveRemoteFileAndHandleHardlink(lh, link1remote, dirToSearch, hardlinkMeta) + So(err, ShouldBeNil) - for i, path := range sourcePaths { - dir := filepath.Dir(path) + _, err = os.Stat(link1remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") - err := os.MkdirAll(dir, userPerms) - if err != nil { - t.Fatal(err) - } + _, err = os.Stat(inodeRemote) + So(err, ShouldBeNil) - internal.CreateTestFile(t, path, "1\n") + Convey("Then you can remove the second hardlink and the inode file will also get removed", func() { + err = RemoveRemoteFileAndHandleHardlink(lh, link2remote, dirToSearch, hardlinkMeta) + So(err, ShouldBeNil) - requests[i] = &put.Request{ - Local: path, - Remote: strings.Replace(path, sourceDir, destDir, 1), - Meta: &put.Meta{LocalMeta: localMeta}, - } - } + _, err = os.Stat(link2remote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") - return requests + _, err = os.Stat(inodeRemote) + So(err.Error(), ShouldContainSubstring, "no such file or directory") + }) + }) + }) + }) + }) } diff --git a/server/server.go b/server/server.go index 381f167..8efa8d3 100644 --- a/server/server.go +++ b/server/server.go @@ -404,7 +404,7 @@ func (s *Server) clientTTRC(data interface{}) queue.SubQueue { s.Logger.Printf("item data not a hostPID") } - s.slacker.SendMessage(slack.Warn, fmt.Sprintf("client host pid %s assumed killed", hostPID)) + s.sendSlackMessage(slack.Warn, fmt.Sprintf("client host pid %s assumed killed", hostPID)) s.iRODSTracker.deleteIRODSConnections(hostPID) diff --git a/server/setdb.go b/server/setdb.go index 9305641..5b7bfb4 100644 --- a/server/setdb.go +++ b/server/setdb.go @@ -561,7 +561,7 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *RemoveReq) error { } if !removeReq.IsRemovedFromIRODS { - err = s.removeFileFromIRODS(removeReq.Set, removeReq.Path, transformer) + err = s.updateOrRemoveRemoteFile(removeReq.Set, removeReq.Path, transformer) if err != nil { s.setErrorOnEntry(entry, removeReq.Set.ID(), removeReq.Path, err) @@ -579,7 +579,7 @@ func (s *Server) removeFileFromIRODSandDB(removeReq *RemoveReq) error { return s.db.UpdateBasedOnRemovedEntry(removeReq.Set.ID(), entry) } -func (s *Server) removeFileFromIRODS(set *set.Set, path string, transformer put.PathTransformer) error { +func (s *Server) updateOrRemoveRemoteFile(set *set.Set, path string, transformer put.PathTransformer) error { rpath, err := transformer(path) if err != nil { return err @@ -595,7 +595,16 @@ func (s *Server) removeFileFromIRODS(set *set.Set, path string, transformer put. return err } - return remove.RemovePathFromSetInIRODS(s.storageHandler, transformer, rpath, sets, requesters, remoteMeta) + if len(sets) == 0 { + dirToSearch, err := transformer("/") + if err != nil { + return err + } + + return remove.RemoveRemoteFileAndHandleHardlink(s.storageHandler, rpath, dirToSearch, remoteMeta) + } + + return remove.UpdateSetsAndRequestersOnRemoteFile(s.storageHandler, rpath, sets, requesters, remoteMeta) } func (s *Server) handleSetsAndRequesters(set *set.Set, meta map[string]string) ([]string, []string, error) { @@ -673,7 +682,7 @@ func removeElementFromSlice(slice []string, element string) ([]string, error) { func (s *Server) setErrorOnEntry(entry *set.Entry, sid, path string, err error) { entry.LastError = err.Error() - erru := s.db.UploadEntry(sid, path, entry) + erru := s.db.UpdateEntry(sid, path, entry) if erru != nil { s.Logger.Printf("%s", erru.Error()) } @@ -686,7 +695,7 @@ func (s *Server) removeDirFromIRODSandDB(removeReq *RemoveReq) error { } if !removeReq.IsDirEmpty && !removeReq.IsRemovedFromIRODS { - err = remove.RemoveDirFromIRODS(s.storageHandler, removeReq.Path, transformer) + err = remove.RemoveRemoteDir(s.storageHandler, removeReq.Path, transformer) if err != nil { return err } diff --git a/set/db.go b/set/db.go index 9e7a083..d1b537f 100644 --- a/set/db.go +++ b/set/db.go @@ -248,7 +248,8 @@ func updateDatabaseSetWithUserSetDetails(dbSet, userSet *Set) error { return nil } -func (d *DB) UploadEntry(sid, key string, entry *Entry) error { +// UpdateEntry puts the updated entry into the database for the given set. +func (d *DB) UpdateEntry(sid, key string, entry *Entry) error { return d.db.Update(func(tx *bolt.Tx) error { _, b, err := d.getEntry(tx, sid, key) if err != nil {