From 46dcfae19b14475601312903df7637fa24c41121 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Fri, 27 Sep 2024 15:01:37 +0200 Subject: [PATCH 01/13] new module to cleanup tempfiles --- src/general/g.remove.path/Makefile | 7 ++ src/general/g.remove.path/g.remove.path.html | 17 +++ src/general/g.remove.path/g.remove.path.py | 88 +++++++++++++ .../testsuite/test_g_remove_path.py | 117 ++++++++++++++++++ 4 files changed, 229 insertions(+) create mode 100644 src/general/g.remove.path/Makefile create mode 100644 src/general/g.remove.path/g.remove.path.html create mode 100644 src/general/g.remove.path/g.remove.path.py create mode 100644 src/general/g.remove.path/testsuite/test_g_remove_path.py diff --git a/src/general/g.remove.path/Makefile b/src/general/g.remove.path/Makefile new file mode 100644 index 00000000..8e16fb4e --- /dev/null +++ b/src/general/g.remove.path/Makefile @@ -0,0 +1,7 @@ +MODULE_TOPDIR = ../../ + +PGM = g.remove.path + +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script $(TEST_DST) diff --git a/src/general/g.remove.path/g.remove.path.html b/src/general/g.remove.path/g.remove.path.html new file mode 100644 index 00000000..08974dbf --- /dev/null +++ b/src/general/g.remove.path/g.remove.path.html @@ -0,0 +1,17 @@ +

DESCRIPTION

+ +g.remove.path is a simple helper module to unzip all zip-files in a directory +in parallel. The user can select an output directory, the number of parallel +processes (nprocs) and to remove the extracted zip-file (r-flag). + + +

EXAMPLES

+ +
+g.remove.path -f path="$(g.tempfile pid=12345)"
+
+ + +

AUTHOR

+ +Stefan Blumentrath diff --git a/src/general/g.remove.path/g.remove.path.py b/src/general/g.remove.path/g.remove.path.py new file mode 100644 index 00000000..4050612b --- /dev/null +++ b/src/general/g.remove.path/g.remove.path.py @@ -0,0 +1,88 @@ +#! /usr/bin/python3 +""" +MODULE: g.remove.path +AUTHOR(S): Stefan Blumentrath +PURPOSE: Remove temporary files or directories +COPYRIGHT: (C) 2024 by Stefan Blumentrath and the GRASS Development Team + +This program is free software under the GNU General +Public License (>=v2). Read the file COPYING that +comes with GRASS for details. +""" + +# %module +# % description: Remove temporary files or directories +# % keyword: general +# % keyword: remove +# % keyword: file +# % keyword: directory +# % keyword: cleanup +# %end + +# %option +# % key: paths +# % description: Path to the file or directory to remove +# % required: yes +# %end + +# %flag +# % key: f +# % label: Force removal +# % description: Force removal of files or directories +# %end + +# %flag +# % key: r +# % label: Remove directories recursively +# % description: Remove directories recursively +# %end + +# ruff: noqa: PTH207 + +import shutil +import sys +from glob import glob +from pathlib import Path + +import grass.script as gs + + +def main(): + """Do the main work""" + options, flags = gs.parser() + paths_to_remove = glob(options["paths"]) + if not paths_to_remove: + gs.warning(_("Nothing found to remove with <{}>.").format(options["paths"])) + + if flags["f"]: + gs.info(_("Removing the following files and directories:")) + for user_path in paths_to_remove: + user_path = Path(user_path) + if user_path.is_symlink() or user_path.is_file(): + try: + user_path.unlink() + except Exception: + gs.warning( + _("Could not remove file or symlink <{}>").format(user_path) + ) + elif user_path.is_dir(): + if flags["r"]: + try: + shutil.rmtree(str(user_path)) + except Exception: + gs.warning( + _("Could not remove directory <{}>").format(user_path) + ) + gs.warning( + _( + "Cannot remove <{}>. It is a directory. Use the r-flag to remove it." + ).format(user_path) + ) + else: + gs.info(_("Set to remove the following files and directories:")) + gs.info(_("Use the f-flag to actually remove them.")) + gs.info(_("\n".join(paths_to_remove))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/general/g.remove.path/testsuite/test_g_remove_path.py b/src/general/g.remove.path/testsuite/test_g_remove_path.py new file mode 100644 index 00000000..cea2425a --- /dev/null +++ b/src/general/g.remove.path/testsuite/test_g_remove_path.py @@ -0,0 +1,117 @@ +"""Test i.satskred + +(C) 2023 by NVE, Stefan Blumentrath and the GRASS GIS Development Team +This program is free software under the GNU General Public +License (>=v2). Read the file COPYING that comes with GRASS +for details. + +:authors: Stefan Blumentrath +""" + +from pathlib import Path + +import grass.script as gs +from grass.gunittest.case import TestCase +from grass.gunittest.gmodules import SimpleModule + + +class TestGUnzipParallel(TestCase): + """Test case for removing files and directories""" + + @classmethod + def setUpClass(cls): + """Initiate the working environment""" + cls.tempdir = Path(gs.tempdir()) + (cls.tempdir / "dir_1").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_2").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_3").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_4").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_4" / "dir_1").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "file_1").write_text("", encoding="UTF8") + (cls.tempdir / "file_2").write_text("", encoding="UTF8") + (cls.tempdir / "file_3").write_text("", encoding="UTF8") + (cls.tempdir / "dir_3" / "file_1").write_text("", encoding="UTF8") + (cls.tempdir / "dir_4" / "file_1").write_text("", encoding="UTF8") + + @classmethod + def tearDownClass(cls): + """Remove the temporary data""" + gs.utils.try_rmdir(cls.tempdir) + + def test_g_remove_path_wildcard_no_removal(self): + """Test file removal dry-run with wildcard""" + tmp_files_before = list(self.tempdir.glob("*")) + tmp_files_before.sort() + # Check that g.remove.path runs successfully + g_remove_list = SimpleModule( + "g.remove.path", + paths=f"{self.tempdir}/*", + ).run() + + tmp_files_after = list(self.tempdir.glob("*")) + tmp_files_after.sort() + # Check that no files are removed + self.assertTrue( + [str(tmp_file) for tmp_file in tmp_files_before] + == [str(tmp_file) for tmp_file in tmp_files_after] + ) + self.assertTrue( + all( + str(tmp_file) in g_remove_list.outputs.stderr + for tmp_file in tmp_files_before + ) + ) + + def test_g_remove_path_wildcard_no_dir_removal(self): + """Test file removal dry-run with wildcard""" + tmp_files_before = list(self.tempdir.glob("dir*")) + tmp_files_before.sort() + # Check that g.remove.path runs successfully + g_remove_list = SimpleModule( + "g.remove.path", paths=f"{self.tempdir}/dir*", flags="f" + ).run() + + tmp_files_after = list(self.tempdir.glob("dir*")) + tmp_files_after.sort() + # Check that no files are removed + self.assertTrue( + [str(tmp_file) for tmp_file in tmp_files_before] + == [str(tmp_file) for tmp_file in tmp_files_after] + ) + self.assertTrue( + all( + str(tmp_file) in g_remove_list.outputs.stderr + for tmp_file in tmp_files_before + ) + ) + + def test_g_remove_path_wildcard_with_removal(self): + """Test file removal with wildcard""" + # Check that g.remove.path runs successfully + g_remove_list = SimpleModule( + "g.remove.path", + paths=f"{self.tempdir}/dir_3/*", + flags="rf", + ).run() + + # Check that no files are removed + self.assertTrue(len(list((self.tempdir / "dir_3").glob("*"))) == 0) + + def test_g_remove_path_no_dir_removal(self): + """Test file removal with wildcard""" + # Check that g.remove.path runs successfully + g_remove_list = SimpleModule( + "g.remove.path", + paths=f"{self.tempdir}/dir_4/*", + flags="f", + ).run() + + self.assertTrue("WARNING" in g_remove_list.outputs.stderr) + # Check that no dirs are removed + self.assertTrue(len(list((self.tempdir / "dir_4").glob("*"))) == 1) + + +if __name__ == "__main__": + from grass.gunittest.main import test + + test() From 5288169d5d89012f5a6df03f96b9e883f61c21e1 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:45:08 +0200 Subject: [PATCH 02/13] new helper module --- src/general/g.transfer/g.transfer.py | 109 +++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/general/g.transfer/g.transfer.py diff --git a/src/general/g.transfer/g.transfer.py b/src/general/g.transfer/g.transfer.py new file mode 100644 index 00000000..fe8b4ac9 --- /dev/null +++ b/src/general/g.transfer/g.transfer.py @@ -0,0 +1,109 @@ +#! /usr/bin/python3 +""" +MODULE: g.transfer +AUTHOR(S): Stefan Blumentrath +PURPOSE: Move or copy files or directories from source to target +COPYRIGHT: (C) 2024 by Stefan Blumentrath and the GRASS Development Team + +This program is free software under the GNU General +Public License (>=v2). Read the file COPYING that +comes with GRASS for details. +""" + +# %module +# % description: Move or copy files or directories from source to target +# % keyword: general +# % keyword: file +# % keyword: directory +# % keyword: move +# % keyword: copy +# %end + +# %option +# % key: source +# % description: Path to source of files or directories to transfer (supports wildcards (*)) +# % required: yes +# %end + +# %option G_OPT_M_DIR +# % key: target +# % description: Path to target to transfer files or directories to +# % required: yes +# %end + +# %option G_OPT_M_NPROCS +# %end + +# %flag +# % key: m +# % label: Move files or directories (default is copy) +# % description: Move files or directories (default is copy) +# %end + +# ruff: noqa: PTH207 + +import shutil +import sys +from functools import partial +from glob import glob +from multiprocessing import Pool +from pathlib import Path + +import grass.script as gs + + +def transfer(source, target=None, move=False): + """Function to transfer files from source to target""" + source = Path(source) + if move: + gs.verbose( + _("Moving <{source}> to <{target}>").format(source=source, target=target) + ) + shutil.move(source, target) + return + if source.is_dir(): + gs.verbose( + _("Copying directory tree <{source}> to <{target}>").format( + source=source, target=target + ) + ) + target_dir = target / source.name + shutil.copytree(source, target_dir, dirs_exist_ok=True) + return + gs.verbose( + _("Copying file <{source}> to <{target}>").format(source=source, target=target) + ) + shutil.copy2(source, target) + + +def main(): + """Do the main work""" + options, flags = gs.parser() + paths_to_transfer = glob(options["source"]) + if not paths_to_transfer: + gs.warning( + _("Nothing found to transfer with source <{}>.").format(options["source"]) + ) + sys.exit(0) + + target_directory = Path(options["target"]) if options["target"] else Path.cwd() + + if target_directory.exists() and target_directory.is_file(): + gs.fatal( + _("Target <{}> exists and is not a directory.").format(options["target"]) + ) + target_directory.mkdir(exist_ok=True, parents=True) + + transfer_function = partial(transfer, target=target_directory, move=flags["m"]) + nprocs = int(options["nprocs"]) + + if nprocs > 1: + with Pool(nprocs) as pool: + pool.map(transfer_function, paths_to_transfer) + else: + for transfer_path in paths_to_transfer: + transfer_function(transfer_path) + + +if __name__ == "__main__": + sys.exit(main()) From 6ec66538efbf67bebf918320f06a22737360085d Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:45:24 +0200 Subject: [PATCH 03/13] new helper module --- .../g.transfer/testsuite/test_g_transfer.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 src/general/g.transfer/testsuite/test_g_transfer.py diff --git a/src/general/g.transfer/testsuite/test_g_transfer.py b/src/general/g.transfer/testsuite/test_g_transfer.py new file mode 100644 index 00000000..8b0692d5 --- /dev/null +++ b/src/general/g.transfer/testsuite/test_g_transfer.py @@ -0,0 +1,103 @@ +"""Test g.transfer + +(C) 2024 by NVE, Stefan Blumentrath and the GRASS GIS Development Team +This program is free software under the GNU General Public +License (>=v2). Read the file COPYING that comes with GRASS +for details. + +:authors: Stefan Blumentrath +""" + +from pathlib import Path + +import grass.script as gs +from grass.gunittest.case import TestCase + + +class TestGTransfer(TestCase): + """Test case for removing files and directories""" + + @classmethod + def setUpClass(cls): + """Initiate the working environment""" + cls.tempdir = Path(gs.tempdir()) + cls.tempdir_target = Path(gs.tempdir()) + (cls.tempdir / "dir_1").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_2").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_3").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_4").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "dir_4" / "dir_1").mkdir(parents=True, exist_ok=True) + (cls.tempdir / "file_1").write_text("Test", encoding="UTF8") + (cls.tempdir / "file_2").write_text("Test", encoding="UTF8") + (cls.tempdir / "file_3").write_text("Test", encoding="UTF8") + (cls.tempdir / "dir_3" / "file_1").write_text("Test", encoding="UTF8") + (cls.tempdir / "dir_4" / "file_1").write_text("Test", encoding="UTF8") + + @classmethod + def tearDownClass(cls): + """Remove the temporary data""" + gs.utils.try_rmdir(cls.tempdir) + gs.utils.try_rmdir(cls.tempdir_target) + + def test_g_transfer_no_removal(self): + """Test copying files and directories""" + + # Check that g.transfer runs successfully + self.assertModule( + "g.transfer", + source=f"{self.tempdir}/dir_4", + target=f"{self.tempdir_target}", + ) + + # Check that no files are removed + self.assertTrue((self.tempdir / "dir_4").is_dir()) + self.assertTrue((self.tempdir / "dir_4" / "dir_1").is_dir()) + self.assertFileExists(str(self.tempdir / "dir_4" / "file_1")) + # Check that files are copied + self.assertTrue((self.tempdir_target / "dir_4").is_dir()) + self.assertTrue((self.tempdir_target / "dir_4" / "dir_1").is_dir()) + self.assertFileExists(str(self.tempdir_target / "dir_4" / "file_1")) + + def test_g_transfer_move(self): + """Test moving files and directories""" + + # Check that g.transfer runs successfully + self.assertModule( + "g.transfer", + flags="m", + source=f"{self.tempdir}/dir_3", + target=f"{self.tempdir_target}", + ) + + # Check that no files are removed + self.assertFalse((self.tempdir / "dir_3").is_dir()) + self.assertFalse((self.tempdir / "dir_3" / "file_1").is_file()) + # Check that files are copied + self.assertTrue((self.tempdir_target / "dir_3").is_dir()) + self.assertFileExists(str(self.tempdir_target / "dir_3" / "file_1")) + + def test_g_transfer_with_wildcard(self): + """Test copying files with wildcard""" + + # Check that g.transfer runs successfully + self.assertModule( + "g.transfer", + source=f"{self.tempdir}/file_*", + target=f"{self.tempdir_target}", + nprocs="2", + ) + + # Check that no files are removed + self.assertTrue((self.tempdir / "file_1").is_file()) + self.assertTrue((self.tempdir / "file_2").is_file()) + self.assertTrue((self.tempdir / "file_3").is_file()) + # Check that files are copied + self.assertTrue((self.tempdir / "file_1").is_file()) + self.assertTrue((self.tempdir / "file_2").is_file()) + self.assertTrue((self.tempdir / "file_3").is_file()) + + +if __name__ == "__main__": + from grass.gunittest.main import test + + test() From b64c699992ffb1702f5a4fab5f689f9eac9dd28e Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:45:35 +0200 Subject: [PATCH 04/13] new helper module --- src/general/g.transfer/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/general/g.transfer/Makefile diff --git a/src/general/g.transfer/Makefile b/src/general/g.transfer/Makefile new file mode 100644 index 00000000..78850007 --- /dev/null +++ b/src/general/g.transfer/Makefile @@ -0,0 +1,7 @@ +MODULE_TOPDIR = ../../ + +PGM = g.transfer + +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script $(TEST_DST) From f4f1ed0995a67b93de0e53663d921fda1d8a62ba Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:45:43 +0200 Subject: [PATCH 05/13] new helper module --- src/general/g.transfer/g.transfer.html | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/general/g.transfer/g.transfer.html diff --git a/src/general/g.transfer/g.transfer.html b/src/general/g.transfer/g.transfer.html new file mode 100644 index 00000000..cf1527d2 --- /dev/null +++ b/src/general/g.transfer/g.transfer.html @@ -0,0 +1,24 @@ +

DESCRIPTION

+ +g.transfer is a simple helper module to be used in actinia to +transfer files or directories from local storage on a worker node +to persistent network storage. +

+If the target directory does not exist it is created. The +source option supports the use of wildcards (*), so multiple +files and/or directories can be transfered. The nprocs option +allows to transfer multiple files or directories in parallel. With +the m-flag the source is moved and not copied (the default). + + +

EXAMPLES

+ +
+temp_file=$(g.tempfile pid=12345)
+g.transfer -f source="$temp_file" target=/tmp
+
+ + +

AUTHOR

+ +Stefan Blumentrath From a41c32bc1c8277aee87abda5b27679d5ce3df3b8 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:46:52 +0200 Subject: [PATCH 06/13] rename option --- src/general/g.remove.path/g.remove.path.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/general/g.remove.path/g.remove.path.py b/src/general/g.remove.path/g.remove.path.py index 4050612b..6e293611 100644 --- a/src/general/g.remove.path/g.remove.path.py +++ b/src/general/g.remove.path/g.remove.path.py @@ -20,7 +20,7 @@ # %end # %option -# % key: paths +# % key: path # % description: Path to the file or directory to remove # % required: yes # %end @@ -50,9 +50,9 @@ def main(): """Do the main work""" options, flags = gs.parser() - paths_to_remove = glob(options["paths"]) + paths_to_remove = glob(options["path"]) if not paths_to_remove: - gs.warning(_("Nothing found to remove with <{}>.").format(options["paths"])) + gs.warning(_("Nothing found to remove with <{}>.").format(options["path"])) if flags["f"]: gs.info(_("Removing the following files and directories:")) From 1fd7deb391631d46162b05df684c3d722578e012 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:47:03 +0200 Subject: [PATCH 07/13] rename option --- .../g.remove.path/testsuite/test_g_remove_path.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/general/g.remove.path/testsuite/test_g_remove_path.py b/src/general/g.remove.path/testsuite/test_g_remove_path.py index cea2425a..9c0ce37e 100644 --- a/src/general/g.remove.path/testsuite/test_g_remove_path.py +++ b/src/general/g.remove.path/testsuite/test_g_remove_path.py @@ -45,7 +45,7 @@ def test_g_remove_path_wildcard_no_removal(self): # Check that g.remove.path runs successfully g_remove_list = SimpleModule( "g.remove.path", - paths=f"{self.tempdir}/*", + path=f"{self.tempdir}/*", ).run() tmp_files_after = list(self.tempdir.glob("*")) @@ -68,7 +68,7 @@ def test_g_remove_path_wildcard_no_dir_removal(self): tmp_files_before.sort() # Check that g.remove.path runs successfully g_remove_list = SimpleModule( - "g.remove.path", paths=f"{self.tempdir}/dir*", flags="f" + "g.remove.path", path=f"{self.tempdir}/dir*", flags="f" ).run() tmp_files_after = list(self.tempdir.glob("dir*")) @@ -88,9 +88,9 @@ def test_g_remove_path_wildcard_no_dir_removal(self): def test_g_remove_path_wildcard_with_removal(self): """Test file removal with wildcard""" # Check that g.remove.path runs successfully - g_remove_list = SimpleModule( + SimpleModule( "g.remove.path", - paths=f"{self.tempdir}/dir_3/*", + path=f"{self.tempdir}/dir_3/*", flags="rf", ).run() @@ -102,7 +102,7 @@ def test_g_remove_path_no_dir_removal(self): # Check that g.remove.path runs successfully g_remove_list = SimpleModule( "g.remove.path", - paths=f"{self.tempdir}/dir_4/*", + path=f"{self.tempdir}/dir_4/*", flags="f", ).run() From c713d7911df6ee654f1eb191a02e68b65405ced2 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:47:24 +0200 Subject: [PATCH 08/13] add actual manual --- src/general/g.remove.path/g.remove.path.html | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/general/g.remove.path/g.remove.path.html b/src/general/g.remove.path/g.remove.path.html index 08974dbf..365f806d 100644 --- a/src/general/g.remove.path/g.remove.path.html +++ b/src/general/g.remove.path/g.remove.path.html @@ -1,14 +1,17 @@

DESCRIPTION

-g.remove.path is a simple helper module to unzip all zip-files in a directory -in parallel. The user can select an output directory, the number of parallel -processes (nprocs) and to remove the extracted zip-file (r-flag). +g.remove.path is a simple helper module to be used in actinia to +clean-up temporary files and directories on a worker node. +In line with g.remove no files or directories are removed without +the f-flag. For directories also the r-flag is needed. +The path option supports the use of wildcards (*), handle with care.

EXAMPLES

-g.remove.path -f path="$(g.tempfile pid=12345)"
+temp_file=$(g.tempfile pid=12345)
+g.remove.path -f path="$temp_file"
From 28d0145be3fcd4000f357bceea92ccdcf8140b62 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 16:48:24 +0200 Subject: [PATCH 09/13] rename option --- src/general/g.remove.path/testsuite/test_g_remove_path.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/general/g.remove.path/testsuite/test_g_remove_path.py b/src/general/g.remove.path/testsuite/test_g_remove_path.py index 9c0ce37e..6755dc29 100644 --- a/src/general/g.remove.path/testsuite/test_g_remove_path.py +++ b/src/general/g.remove.path/testsuite/test_g_remove_path.py @@ -1,6 +1,6 @@ -"""Test i.satskred +"""Test g.remove.path -(C) 2023 by NVE, Stefan Blumentrath and the GRASS GIS Development Team +(C) 2024 by NVE, Stefan Blumentrath and the GRASS GIS Development Team This program is free software under the GNU General Public License (>=v2). Read the file COPYING that comes with GRASS for details. @@ -15,7 +15,7 @@ from grass.gunittest.gmodules import SimpleModule -class TestGUnzipParallel(TestCase): +class TestGRemovePath(TestCase): """Test case for removing files and directories""" @classmethod From dddeab7926b07cd90fd7def714da7c35729230fb Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 17:01:14 +0200 Subject: [PATCH 10/13] add new module --- .../t.rast.copytree/t.rast.copytree.py | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 src/temporal/t.rast.copytree/t.rast.copytree.py diff --git a/src/temporal/t.rast.copytree/t.rast.copytree.py b/src/temporal/t.rast.copytree/t.rast.copytree.py new file mode 100644 index 00000000..d81c8814 --- /dev/null +++ b/src/temporal/t.rast.copytree/t.rast.copytree.py @@ -0,0 +1,194 @@ +#! /usr/bin/python3 +""" +MODULE: t.rast.copytree +AUTHOR(S): Stefan Blumentrath +PURPOSE: Transfer raster map files from STRDS in external GDAL format to target directory +COPYRIGHT: (C) 2024 by Stefan Blumentrath and the GRASS Development Team + +This program is free software under the GNU General +Public License (>=v2). Read the file COPYING that +comes with GRASS for details. +""" + +# %module +# % description: Transfer raster map files from STRDS in external GDAL format to target directory +# % keyword: temporal +# % keyword: move +# % keyword: copy +# % keyword: GDAL +# % keyword: directory +# %end + +# %option G_OPT_STRDS_INPUT +# %end + +# %option G_OPT_T_WHERE +# %end + +# %option G_OPT_M_DIR +# % key: output_directory +# % description: Path to the output / destination directory +# % required: yes +# %end + +# %option G_OPT_M_DIR +# % key: source_directory +# % description: Path to the source directory +# % required: no +# %end + +# %option +# % key: suffix +# % type: string +# % description: Suffix of files to transfer +# % required: no +# %end + +# %option +# % key: temporal_tree +# % type: string +# % description: Strftime format to create temporal directory name or tree (e.g. "%Y/%m/%d") +# % required: no +# %end + +# %option G_OPT_M_NPROCS +# %end + +# %flag +# % key: m +# % label: Move files into destination (default is copy) +# % description: Move files into destination (default is copy) +# %end + +# %flag +# % key: s +# % label: Use semantic label in directory structure +# % description: Use semantic label in directory structure +# %end + +# %rules +# % collective: source_directory, suffix +# %end + +import shutil +import sys +from itertools import starmap +from multiprocessing import Pool +from pathlib import Path + +import grass.script as gs +import grass.temporal as tgis + + +def transfer_results( + raster_map_rows, + source_directory, + output_directory=None, + temporal_tree="%Y/%m/%d", + sep="|", + suffix="tif", + use_semantic_label=False, + move=False, + nprocs=1, +): + """Transfer resulting maps to Network storage""" + # Set relevant time frame + target_directories = set() + transfer_tuples = [] + return_list = [] + for map_row in raster_map_rows: + start_day = map_row["start_time"] + if use_semantic_label: + semantic_label = map_row["semantic_label"] + target_directory = ( + output_directory / semantic_label / start_day.strftime(temporal_tree) + ) + else: + target_directory = output_directory / start_day.strftime(temporal_tree) + target_directories.add(target_directory) + transfer_tuples.append( + ( + f"{source_directory / map_row['name']!s}{suffix}", + str(target_directory), + ) + ) + return_list.append( + sep.join( + [ + map_row["name"], + map_row["start_time"].strftime("%Y-%m-%d %H:%M:%S"), + map_row["end_time"].strftime("%Y-%m-%d %H:%M:%S"), + map_row["semantic_label"] or "", + f"{target_directory / map_row['name']!s}{suffix}", + ] + ) + ) + + # Create target directory structure + for target_directory in target_directories: + target_directory.mkdir(exist_ok=True, parents=True) + + # Transfer files in parallel + # Support several functions (move, copy, ...) + transfer_function = shutil.move if move else shutil.copy + + if nprocs > 1: + with Pool(nprocs) as pool: + pool.starmap(transfer_function, transfer_tuples) + else: + list(starmap(transfer_function, transfer_tuples)) + return return_list + + +def main(): + """Do the main work""" + options, flags = gs.parser() + + # Check if maps are exported to GDAL formats + if options["source_directory"]: + source_directory = Path(options["source_directory"]) + suffix = ( + f".{options['suffix']}" + if not options["suffix"].startswith(".") + else options["suffix"] + ) + else: + external = { + line.split(": ")[0]: line.split(": ")[1] + for line in gs.read_command("r.external.out", flags="p").split("\n") + if ": " in line + } + if not external: + gs.warning( + _( + "Neither source directory given nor external linking (r.external.out) defined." + ) + ) + sys.exit(0) + source_directory = Path(external["directory"]) + suffix = f"{external['extension']}" if external["extension"] != "" else "" + + tgis.init() + input_strds = tgis.open_old_stds(options["input"], "strds") + input_strds_maps = input_strds.get_registered_maps( + columns="name,start_time,end_time,semantic_label", where=options["where"] + ) + + register_strings = transfer_results( + input_strds_maps, + source_directory, + output_directory=Path(options["output_directory"]), + temporal_tree=options["temporal_tree"] or "%Y/%m/%d", + sep="|", + suffix=suffix, + use_semantic_label=flags["s"], + move=flags["m"], + nprocs=1, + ) + + # Print register information + print("\n".join(register_strings)) + + +if __name__ == "__main__": + sys.exit(main()) From 399b265decac61c07f4f49082353c5c44722925b Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 17:01:18 +0200 Subject: [PATCH 11/13] add new module --- .../t.rast.copytree/t.rast.copytree.html | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/temporal/t.rast.copytree/t.rast.copytree.html diff --git a/src/temporal/t.rast.copytree/t.rast.copytree.html b/src/temporal/t.rast.copytree/t.rast.copytree.html new file mode 100644 index 00000000..f31f05da --- /dev/null +++ b/src/temporal/t.rast.copytree/t.rast.copytree.html @@ -0,0 +1,48 @@ +

DESCRIPTION

+ +t.rast.copytree is a simple helper module to copy extrenal registered raster +maps from a STRDS into a temporal directory tree. The tree structure can be defined +with the temporal_tree option and the s-flag (to include the +semantic_label in the directory structure). The output directory needs to +exist, but the directory structure below will be created if necessary. Parallel +transfer of files is supported with the nprocs option. Using the m-flag +files can be moved instead of copied. + +

EXAMPLES

+ +
+temp_dir=$(g.tempfile -d pid=1)
+mkdir $temp_dir
+target_dir=$(g.tempfile -d pid=1)
+mkdir $target_dir
+
+g.region -ag s=0 n=80 w=0 e=120 res=1
+r.external.out format="GTiff" directory=$temp_dir extension="tif" options="COMPRESS=LZW"
+for rmap_idx in 1 2 3
+do
+  for prefix in a b
+  do
+    r.mapcalc expression="${prefix}_${rmap_idx} = ${rmap_idx}00 --overwrite
+    r.support map="${prefix}_${rmap_idx} semantic_label=$prefix
+  done
+done
+t.create type="strds" temporaltype="absolute" output="A" \
+    title="A test" description="A test" --overwrite
+t.register -i type="raster" input="A" maps="a_1,a_2,a_3" \
+    start="2001-01-01" increment="3 months" --overwrite
+t.create type="strds" temporaltype="absolute" output="B" \
+    title="B test" description="B test" --overwrite
+t.register -i type="raster" input="B" maps="b_1,b_2,b_3" \
+    start="2001-01-01" increment="1 day" --overwrite
+
+t.rast.copytree -m input="A" temporal_tree="%Y/%m" nprocs=2 \
+    output_directory=$target_dir
+
+t.rast.copytree -s input="B" temporal_tree="%Y/%m/%d" nprocs=2 \
+output_directory=$target_dir
+
+ + +

AUTHOR

+ +Stefan Blumentrath From fe15c655d5b9c4fa285a3edbb00cc3a8bc0299e5 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 17:01:24 +0200 Subject: [PATCH 12/13] add new module --- src/temporal/t.rast.copytree/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/temporal/t.rast.copytree/Makefile diff --git a/src/temporal/t.rast.copytree/Makefile b/src/temporal/t.rast.copytree/Makefile new file mode 100644 index 00000000..c47bf57f --- /dev/null +++ b/src/temporal/t.rast.copytree/Makefile @@ -0,0 +1,7 @@ +MODULE_TOPDIR = ../../ + +PGM = t.rast.copytree + +include $(MODULE_TOPDIR)/include/Make/Script.make + +default: script $(TEST_DST) From 370afc778ddcc7a111604de648a7651f6cca5c87 Mon Sep 17 00:00:00 2001 From: ninsbl Date: Mon, 30 Sep 2024 17:03:01 +0200 Subject: [PATCH 13/13] add new module --- .../testsuite/test_t_rast_copytree.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/temporal/t.rast.copytree/testsuite/test_t_rast_copytree.py diff --git a/src/temporal/t.rast.copytree/testsuite/test_t_rast_copytree.py b/src/temporal/t.rast.copytree/testsuite/test_t_rast_copytree.py new file mode 100644 index 00000000..6e1c6a6b --- /dev/null +++ b/src/temporal/t.rast.copytree/testsuite/test_t_rast_copytree.py @@ -0,0 +1,137 @@ +"""Test t.rast.copytree + +(C) 2024 by NVE, Stefan Blumentrath and the GRASS GIS Development Team +This program is free software under the GNU General Public +License (>=v2). Read the file COPYING that comes with GRASS +for details. + +:authors: Stefan Blumentrath +""" + +# ruff: noqa: RUF012 + +from pathlib import Path + +import grass.script as gs +from grass.gunittest.case import TestCase + + +class TestTRastCopytree(TestCase): + """Test case for moving files from STRDS to directory trees""" + + default_region = {"s": 0, "n": 80, "w": 0, "e": 120, "b": 0, "t": 50} + + @classmethod + def setUpClass(cls): + """Initiate the temporal GIS and set the region""" + cls.use_temp_region() + cls.tempdir = Path(gs.tempdir()) + cls.tempdir_target = Path(gs.tempdir()) + cls.runModule("g.region", **cls.default_region, res=1, res3=1) + + cls.runModule( + "r.external.out", + format="GTiff", + directory=str(cls.tempdir), + extension="tif", + ) + for rmap_idx in range(1, 4): + for prefix in ("a", "b"): + cls.runModule( + "r.mapcalc", + expression=f"{prefix}_{rmap_idx} = {rmap_idx}00", + overwrite=True, + ) + cls.runModule( + "r.support", map=f"{prefix}_{rmap_idx}", semantic_label=prefix + ) + + cls.runModule( + "t.create", + type="strds", + temporaltype="absolute", + output="A", + title="A test", + description="A test", + overwrite=True, + ) + cls.runModule( + "t.register", + flags="i", + type="raster", + input="A", + maps="a_1,a_2,a_3", + start="2001-01-01", + increment="3 months", + overwrite=True, + ) + + cls.runModule( + "t.create", + type="strds", + temporaltype="absolute", + output="B", + title="B test", + description="B test", + overwrite=True, + ) + cls.runModule( + "t.register", + flags="i", + type="raster", + input="B", + maps="b_1,b_2,b_3", + start="2001-01-01", + increment="1 day", + overwrite=True, + ) + + @classmethod + def tearDownClass(cls): + """Remove the temporary region""" + cls.del_temp_region() + cls.runModule("t.remove", flags="df", type="strds", inputs="A") + cls.runModule("r.external.out", flags="r") + gs.utils.try_rmdir(str(cls.tempdir_target)) + gs.utils.try_rmdir(str(cls.tempdir)) + + def test_t_rast_copytree_move(self): + """Test moving files into directory tree""" + # Check that t.rast.copytree runs successfully + self.assertModule( + "t.rast.copytree", + flags="m", + input="A", + temporal_tree="%Y/%m", + output_directory=str(self.tempdir_target), + nprocs="1", + ) + + self.assertFileExists(str(self.tempdir_target / "2001/01/a_1.tif")) + self.assertFileExists(str(self.tempdir_target / "2001/04/a_2.tif")) + self.assertFileExists(str(self.tempdir_target / "2001/07/a_3.tif")) + + def test_t_rast_copytree_copy_semantic_label(self): + """Test moving files into directory tree""" + # Check that t.rast.copytree runs successfully + self.assertModule( + "t.rast.copytree", + flags="s", + input="B", + temporal_tree="%Y/%m/%d", + output_directory=str(self.tempdir_target), + nprocs="1", + ) + + self.assertFileExists(str(self.tempdir / "b_1.tif")) + self.assertFileExists(str(self.tempdir / "b_2.tif")) + self.assertFileExists(str(self.tempdir / "b_3.tif")) + self.assertFileExists(str(self.tempdir_target / "b/2001/01/01/b_1.tif")) + self.assertFileExists(str(self.tempdir_target / "b/2001/01/02/b_2.tif")) + self.assertFileExists(str(self.tempdir_target / "b/2001/01/03/b_3.tif")) + + +if __name__ == "__main__": + from grass.gunittest.main import test + + test()