g.transfer/g.remove.path/t.rast.copytree: new helper modules for filesystem operations

src/general/g.remove.path/Makefile

@@ -0,0 +1,7 @@
+MODULE_TOPDIR = ../../
+
+PGM = g.remove.path
+
+include $(MODULE_TOPDIR)/include/Make/Script.make
+
+default: script $(TEST_DST)

src/general/g.remove.path/g.remove.path.html

@@ -0,0 +1,20 @@
+<h2>DESCRIPTION</h2>
+
+<em>g.remove.path</em> is a simple helper module to be used in actinia to
+clean-up temporary files and directories on a worker node.
+In line with <em>g.remove</em> no files or directories are removed without
+the <b>f</b>-flag. For directories also the <b>r</b>-flag is needed.
+The <b>path</b> option supports the use of wildcards (*), handle with care.
+
+
+<h2>EXAMPLES</h2>
+
+<div class="code"><pre>
+temp_file=$(g.tempfile pid=12345)
+g.remove.path -f path="$temp_file"
+</pre></div>
+
+
+<h2>AUTHOR</h2>
+
+Stefan Blumentrath

src/general/g.remove.path/g.remove.path.py

@@ -0,0 +1,88 @@
+#! /usr/bin/python3
+"""
+MODULE:    g.remove.path
+AUTHOR(S): Stefan Blumentrath
+PURPOSE:   Remove temporary files or directories
+COPYRIGHT: (C) 2024 by Stefan Blumentrath and the GRASS Development Team
+
+This program is free software under the GNU General
+Public License (>=v2). Read the file COPYING that
+comes with GRASS for details.
+"""
+
+# %module
+# % description: Remove temporary files or directories
+# % keyword: general
+# % keyword: remove
+# % keyword: file
+# % keyword: directory
+# % keyword: cleanup
+# %end
+
+# %option
+# % key: path
+# % description: Path to the file or directory to remove
+# % required: yes
+# %end
+
+# %flag
+# % key: f
+# % label: Force removal
+# % description: Force removal of files or directories
+# %end
+
+# %flag
+# % key: r
+# % label: Remove directories recursively
+# % description: Remove directories recursively
+# %end
+
+# ruff: noqa: PTH207
+
+import shutil
+import sys
+from glob import glob
+from pathlib import Path
+
+import grass.script as gs
+
+
+def main():
+    """Do the main work"""
+    options, flags = gs.parser()
+    paths_to_remove = glob(options["path"])
+    if not paths_to_remove:
+        gs.warning(_("Nothing found to remove with <{}>.").format(options["path"]))
+
+    if flags["f"]:
+        gs.info(_("Removing the following files and directories:"))
+        for user_path in paths_to_remove:
+            user_path = Path(user_path)
+            if user_path.is_symlink() or user_path.is_file():
+                try:
+                    user_path.unlink()
+                except Exception:
+                    gs.warning(
+                        _("Could not remove file or symlink <{}>").format(user_path)
+                    )
+            elif user_path.is_dir():
+                if flags["r"]:
+                    try:
+                        shutil.rmtree(str(user_path))
+                    except Exception:
+                        gs.warning(
+                            _("Could not remove directory <{}>").format(user_path)
+                        )
+                gs.warning(
+                    _(
+                        "Cannot remove <{}>. It is a directory. Use the r-flag to remove it."
+                    ).format(user_path)
+                )
+    else:
+        gs.info(_("Set to remove the following files and directories:"))
+        gs.info(_("Use the f-flag to actually remove them."))
+        gs.info(_("\n".join(paths_to_remove)))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/general/g.remove.path/testsuite/test_g_remove_path.py

@@ -0,0 +1,117 @@
+"""Test g.remove.path
+
+(C) 2024 by NVE, Stefan Blumentrath and the GRASS GIS Development Team
+This program is free software under the GNU General Public
+License (>=v2). Read the file COPYING that comes with GRASS
+for details.
+
+:authors: Stefan Blumentrath
+"""
+
+from pathlib import Path
+
+import grass.script as gs
+from grass.gunittest.case import TestCase
+from grass.gunittest.gmodules import SimpleModule
+
+
+class TestGRemovePath(TestCase):
+    """Test case for removing files and directories"""
+
+    @classmethod
+    def setUpClass(cls):
+        """Initiate the working environment"""
+        cls.tempdir = Path(gs.tempdir())
+        (cls.tempdir / "dir_1").mkdir(parents=True, exist_ok=True)
+        (cls.tempdir / "dir_2").mkdir(parents=True, exist_ok=True)
+        (cls.tempdir / "dir_3").mkdir(parents=True, exist_ok=True)
+        (cls.tempdir / "dir_4").mkdir(parents=True, exist_ok=True)
+        (cls.tempdir / "dir_4" / "dir_1").mkdir(parents=True, exist_ok=True)
+        (cls.tempdir / "file_1").write_text("", encoding="UTF8")
+        (cls.tempdir / "file_2").write_text("", encoding="UTF8")
+        (cls.tempdir / "file_3").write_text("", encoding="UTF8")
+        (cls.tempdir / "dir_3" / "file_1").write_text("", encoding="UTF8")
+        (cls.tempdir / "dir_4" / "file_1").write_text("", encoding="UTF8")
+
+    @classmethod
+    def tearDownClass(cls):
+        """Remove the temporary data"""
+        gs.utils.try_rmdir(cls.tempdir)
+
+    def test_g_remove_path_wildcard_no_removal(self):
+        """Test file removal dry-run with wildcard"""
+        tmp_files_before = list(self.tempdir.glob("*"))
+        tmp_files_before.sort()
+        # Check that g.remove.path runs successfully
+        g_remove_list = SimpleModule(
+            "g.remove.path",
+            path=f"{self.tempdir}/*",
+        ).run()
+
+        tmp_files_after = list(self.tempdir.glob("*"))
+        tmp_files_after.sort()
+        # Check that no files are removed
+        self.assertTrue(
+            [str(tmp_file) for tmp_file in tmp_files_before]
+            == [str(tmp_file) for tmp_file in tmp_files_after]
+        )
+        self.assertTrue(
+            all(
+                str(tmp_file) in g_remove_list.outputs.stderr
+                for tmp_file in tmp_files_before
+            )
+        )
+
+    def test_g_remove_path_wildcard_no_dir_removal(self):
+        """Test file removal dry-run with wildcard"""
+        tmp_files_before = list(self.tempdir.glob("dir*"))
+        tmp_files_before.sort()
+        # Check that g.remove.path runs successfully
+        g_remove_list = SimpleModule(
+            "g.remove.path", path=f"{self.tempdir}/dir*", flags="f"
+        ).run()
+
+        tmp_files_after = list(self.tempdir.glob("dir*"))
+        tmp_files_after.sort()
+        # Check that no files are removed
+        self.assertTrue(
+            [str(tmp_file) for tmp_file in tmp_files_before]
+            == [str(tmp_file) for tmp_file in tmp_files_after]
+        )
+        self.assertTrue(
+            all(
+                str(tmp_file) in g_remove_list.outputs.stderr
+                for tmp_file in tmp_files_before
+            )
+        )
+
+    def test_g_remove_path_wildcard_with_removal(self):
+        """Test file removal with wildcard"""
+        # Check that g.remove.path runs successfully
+        SimpleModule(
+            "g.remove.path",
+            path=f"{self.tempdir}/dir_3/*",
+            flags="rf",
+        ).run()
+
+        # Check that no files are removed
+        self.assertTrue(len(list((self.tempdir / "dir_3").glob("*"))) == 0)
+
+    def test_g_remove_path_no_dir_removal(self):
+        """Test file removal with wildcard"""
+        # Check that g.remove.path runs successfully
+        g_remove_list = SimpleModule(
+            "g.remove.path",
+            path=f"{self.tempdir}/dir_4/*",
+            flags="f",
+        ).run()
+
+        self.assertTrue("WARNING" in g_remove_list.outputs.stderr)
+        # Check that no dirs are removed
+        self.assertTrue(len(list((self.tempdir / "dir_4").glob("*"))) == 1)
+
+
+if __name__ == "__main__":
+    from grass.gunittest.main import test
+
+    test()

src/general/g.transfer/Makefile

@@ -0,0 +1,7 @@
+MODULE_TOPDIR = ../../
+
+PGM = g.transfer
+
+include $(MODULE_TOPDIR)/include/Make/Script.make
+
+default: script $(TEST_DST)

src/general/g.transfer/g.transfer.html

@@ -0,0 +1,24 @@
+<h2>DESCRIPTION</h2>
+
+<em>g.transfer</em> is a simple helper module to be used in actinia to
+transfer files or directories from local storage on a worker node
+to persistent network storage.
+<p>
+If the <b>target</b> directory does not exist it is created. The
+<b>source</b> option supports the use of wildcards (*), so multiple
+files and/or directories can be transfered. The <b>nprocs</b> option
+allows to transfer multiple files or directories in parallel. With
+the <b>m</b>-flag the source is moved and not copied (the default).
+
+
+<h2>EXAMPLES</h2>
+
+<div class="code"><pre>
+temp_file=$(g.tempfile pid=12345)
+g.transfer -f source="$temp_file" target=/tmp
+</pre></div>
+
+
+<h2>AUTHOR</h2>
+
+Stefan Blumentrath

src/general/g.transfer/g.transfer.py

@@ -0,0 +1,109 @@
+#! /usr/bin/python3
+"""
+MODULE:    g.transfer
+AUTHOR(S): Stefan Blumentrath
+PURPOSE:   Move or copy files or directories from source to target
+COPYRIGHT: (C) 2024 by Stefan Blumentrath and the GRASS Development Team
+
+This program is free software under the GNU General
+Public License (>=v2). Read the file COPYING that
+comes with GRASS for details.
+"""
+
+# %module
+# % description: Move or copy files or directories from source to target
+# % keyword: general
+# % keyword: file
+# % keyword: directory
+# % keyword: move
+# % keyword: copy
+# %end
+
+# %option
+# % key: source
+# % description: Path to source of files or directories to transfer (supports wildcards (*))
+# % required: yes
+# %end
+
+# %option G_OPT_M_DIR
+# % key: target
+# % description: Path to target to transfer files or directories to
+# % required: yes
+# %end
+
+# %option G_OPT_M_NPROCS
+# %end
+
+# %flag
+# % key: m
+# % label: Move files or directories (default is copy)
+# % description: Move files or directories (default is copy)
+# %end
+
+# ruff: noqa: PTH207
+
+import shutil
+import sys
+from functools import partial
+from glob import glob
+from multiprocessing import Pool
+from pathlib import Path
+
+import grass.script as gs
+
+
+def transfer(source, target=None, move=False):
+    """Function to transfer files from source to target"""
+    source = Path(source)
+    if move:
+        gs.verbose(
+            _("Moving <{source}> to <{target}>").format(source=source, target=target)
+        )
+        shutil.move(source, target)
+        return
+    if source.is_dir():
+        gs.verbose(
+            _("Copying directory tree <{source}> to <{target}>").format(
+                source=source, target=target
+            )
+        )
+        target_dir = target / source.name
+        shutil.copytree(source, target_dir, dirs_exist_ok=True)
+        return
+    gs.verbose(
+        _("Copying file <{source}> to <{target}>").format(source=source, target=target)
+    )
+    shutil.copy2(source, target)
+
+
+def main():
+    """Do the main work"""
+    options, flags = gs.parser()
+    paths_to_transfer = glob(options["source"])
+    if not paths_to_transfer:
+        gs.warning(
+            _("Nothing found to transfer with source <{}>.").format(options["source"])
+        )
+        sys.exit(0)
+
+    target_directory = Path(options["target"]) if options["target"] else Path.cwd()
+
+    if target_directory.exists() and target_directory.is_file():
+        gs.fatal(
+            _("Target <{}> exists and is not a directory.").format(options["target"])
+        )
+    target_directory.mkdir(exist_ok=True, parents=True)
+
+    transfer_function = partial(transfer, target=target_directory, move=flags["m"])
+    nprocs = int(options["nprocs"])
+
+    if nprocs > 1:
+        with Pool(nprocs) as pool:
+            pool.map(transfer_function, paths_to_transfer)
+    else:
+        for transfer_path in paths_to_transfer:
+            transfer_function(transfer_path)
+
+
+if __name__ == "__main__":
+    sys.exit(main())