diff --git a/README.md b/README.md index 94d85c4..c17c5b6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec Basic Usage: git fleximod [options] Available Commands: - checkout: Checkout submodules according to git submodule hash configuration. status: Display the status of submodules. update: Update submodules to the tag indicated in .gitmodules variable fxtag. test: Make sure that fxtags and submodule hashes are consistant, @@ -55,9 +54,9 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec Here are some common usage examples: -Checkout submodules, including optional ones: +Update all submodules, including optional ones: ```bash - git fleximod checkout --optional + git fleximod update --optional ``` Updating a specific submodule to the fxtag indicated in .gitmodules: diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py index 7a4470e..1f60b18 100755 --- a/git_fleximod/git_fleximod.py +++ b/git_fleximod/git_fleximod.py @@ -132,8 +132,9 @@ def submodule_sparse_checkout( shutil.copy(sparsefile, gitsparse) # Finally checkout the repo - sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags") + sprepo_git.git_operation("fetch", "origin", "--tags") sprepo_git.git_operation("checkout", tag) + print(f"Successfully checked out {name:>20} at {tag}") rgit.config_set_value(f'submodule "{name}"',"active","true") rgit.config_set_value(f'submodule "{name}"',"url",url) @@ -242,7 +243,7 @@ def submodules_status(gitmodules, root_dir): ahash = git.git_operation("status").partition("\n")[0].split()[-1] if tag and atag == tag: print(f" {name:>20} at tag {tag}") - elif tag and ahash == tag: + elif tag and ahash[:len(tag)] == tag: print(f" {name:>20} at hash {ahash}") elif tag: print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}") diff --git a/git_fleximod/gitinterface.py b/git_fleximod/gitinterface.py index 4d4c4f4..203c500 100644 --- a/git_fleximod/gitinterface.py +++ b/git_fleximod/gitinterface.py @@ -1,25 +1,31 @@ import os import sys from . import utils +from pathlib import Path class GitInterface: def __init__(self, repo_path, logger): logger.debug("Initialize GitInterface for {}".format(repo_path)) - self.repo_path = repo_path + if isinstance(repo_path, str): + self.repo_path = Path(repo_path).resolve() + elif isinstance(repo_path, Path): + self.repo_path = repo_path.resolve() + else: + raise TypeError("repo_path must be a str or Path object") self.logger = logger try: import git self._use_module = True try: - self.repo = git.Repo(repo_path) # Initialize GitPython repo + self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo except git.exc.InvalidGitRepositoryError: self.git = git self._init_git_repo() msg = "Using GitPython interface to git" except ImportError: self._use_module = False - if not os.path.exists(os.path.join(repo_path, ".git")): + if not (repo_path / ".git").exists(): self._init_git_repo() msg = "Using shell interface to git" self.logger.info(msg) @@ -32,13 +38,13 @@ def _git_command(self, operation, *args): except Exception as e: sys.exit(e) else: - return ["git", "-C", self.repo_path, operation] + list(args) + return ["git", "-C", str(self.repo_path), operation] + list(args) def _init_git_repo(self): if self._use_module: - self.repo = self.git.Repo.init(self.repo_path) + self.repo = self.git.Repo.init(str(self.repo_path)) else: - command = ("git", "-C", self.repo_path, "init") + command = ("git", "-C", str(self.repo_path), "init") utils.execute_subprocess(command) # pylint: disable=unused-argument @@ -58,7 +64,7 @@ def config_get_value(self, section, name): config = self.repo.config_reader() return config.get_value(section, name) else: - cmd = ("git", "-C", self.repo_path, "config", "--get", f"{section}.{name}") + cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") output = utils.execute_subprocess(cmd, output_to_caller=True) return output.strip() @@ -68,6 +74,6 @@ def config_set_value(self, section, name, value): writer.set_value(section, name, value) writer.release() # Ensure changes are saved else: - cmd = ("git", "-C", self.repo_path, "config", f"{section}.{name}", value) + cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) self.logger.info(cmd) utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/git_fleximod/gitmodules.py b/git_fleximod/gitmodules.py index ae0ebe1..68c82d0 100644 --- a/git_fleximod/gitmodules.py +++ b/git_fleximod/gitmodules.py @@ -1,14 +1,14 @@ -import os import shutil -from configparser import ConfigParser +from pathlib import Path +from configparser import RawConfigParser, ConfigParser from .lstripreader import LstripReader -class GitModules(ConfigParser): +class GitModules(RawConfigParser): def __init__( self, logger, - confpath=os.getcwd(), + confpath=Path.cwd(), conffile=".gitmodules", includelist=None, excludelist=None, @@ -25,25 +25,32 @@ def __init__( confpath, conffile, includelist, excludelist ) ) - ConfigParser.__init__(self) - self.conf_file = os.path.join(confpath, conffile) - # first create a backup of this file to be restored on deletion of the object - shutil.copy(self.conf_file, self.conf_file + ".save") - self.read_file(LstripReader(self.conf_file), source=conffile) + super().__init__() + self.conf_file = (Path(confpath) / Path(conffile)) + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=conffile) self.includelist = includelist self.excludelist = excludelist + self.isdirty = False + + def reload(self): + self.clear() + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file) + def set(self, name, option, value): """ Sets a configuration value for a specific submodule: Ensures the appropriate section exists for the submodule. Calls the parent class's set method to store the value. """ + self.isdirty = True self.logger.debug("set called {} {} {}".format(name, option, value)) section = f'submodule "{name}"' if not self.has_section(section): self.add_section(section) - ConfigParser.set(self, section, option, str(value)) + super().set(section, option, str(value)) # pylint: disable=redefined-builtin, arguments-differ def get(self, name, option, raw=False, vars=None, fallback=None): @@ -62,12 +69,14 @@ def get(self, name, option, raw=False, vars=None, fallback=None): return None def save(self): - print("Called gitmodules save, not expected") - # self.write(open(self.conf_file, "w")) - + if self.isdirty: + self.logger.info("Writing {}".format(self.conf_file)) + with open(self.conf_file, "w") as fd: + self.write(fd) + self.isdirty = False + def __del__(self): - self.logger.debug("Destroying GitModules object") - shutil.move(self.conf_file + ".save", self.conf_file) + self.save() def sections(self): """Strip the submodule part out of section and just use the name""" diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py new file mode 100755 index 0000000..b607ad9 --- /dev/null +++ b/git_fleximod/metoflexi.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +from configparser import ConfigParser +import sys +import shutil +from pathlib import Path +import argparse +import logging +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules +from git_fleximod import utils + +logger = None + +def find_root_dir(filename=".git"): + d = Path.cwd() + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.is_dir(): + return attempt + d = d.parent + return None + + +def get_parser(): + description = """ + %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument('-e', '--externals', nargs='?', + default='Externals.cfg', + help='The externals description filename. ' + 'Default: %(default)s.') + + parser.add_argument( + "-C", + "--path", + default=find_root_dir(), + help="Toplevel repository directory. Defaults to top git directory relative to current.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + return parser + +def commandline_arguments(args=None): + parser = get_parser() + + options = parser.parse_args(args) + handlers = [logging.StreamHandler()] + + if options.debug: + try: + open("fleximod.log", "w") + except PermissionError: + sys.exit("ABORT: Could not write file fleximod.log") + level = logging.DEBUG + handlers.append(logging.FileHandler("fleximod.log")) + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers + ) + + return( + options.path, + options.gitmodules, + options.externals + ) + +class ExternalRepoTranslator: + """ + Translates external repositories configured in an INI-style externals file. + """ + + def __init__(self, rootpath, gitmodules, externals): + self.rootpath = rootpath + if gitmodules: + self.gitmodules = GitModules(logger, confpath=rootpath) + self.externals = (rootpath / Path(externals)).resolve() + print(f"Translating {self.externals}") + self.git = GitInterface(rootpath, logger) + +# def __del__(self): +# if (self.rootpath / "save.gitignore"): + + + def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, protocol): + """ + Translates a single repository based on configuration details. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + tag (str): The tag to use for the external repository. + url (str): The URL of the external repository. + path (str): The relative path within the main repository for the external repository. + efile (str): The external file or file containing submodules. + hash_ (str): The commit hash to checkout (if applicable). + sparse (str): Boolean indicating whether to use sparse checkout (if applicable). + protocol (str): The protocol to use (e.g., 'git', 'http'). + """ + assert protocol != "svn", "SVN protocol is not currently supported" + print(f"Translating repository {section}") + if efile: + file_path = Path(path) / Path(efile) + newroot = (self.rootpath / file_path).parent.resolve() + if not newroot.exists(): + newroot.mkdir(parents=True) + logger.info("Newroot is {}".format(newroot)) + newt = ExternalRepoTranslator(newroot, ".gitmodules", efile) + newt.translate_repo() + if protocol == "externals_only": + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + + return + + newpath = (self.rootpath / Path(path)) + if newpath.exists(): + shutil.rmtree(newpath) + logger.info("Creating directory {}".format(newpath)) + newpath.mkdir(parents=True) + if tag: + logger.info("cloning {}".format(section)) + try: + self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path) + except: + self.git.git_operation("clone", url, path) + with utils.pushd(newpath): + ngit = GitInterface(newpath, logger) + ngit.git_operation("checkout", tag) + +# if (newpath / ".gitignore").exists(): +# logger.info("Moving .gitignore file in {}".format(newpath)) +# (newpath / ".gitignore").rename((newpath / "save.gitignore")) + + if hash_: + self.git.git_operation("clone", url, path) + git = GitInterface(newpath, logger) + git.git_operation("fetch", "origin") + git.git_operation("checkout", hash_) + if sparse: + print("setting as sparse submodule {}".format(section)) + sparsefile = (newpath / Path(sparse)) + newfile = (newpath / ".git" / "info" / "sparse-checkout") + print(f"sparsefile {sparsefile} newfile {newfile}") + shutil.copy(sparsefile, newfile) + logger.info("adding submodule {}".format(section)) + self.gitmodules.save() + self.git.git_operation("submodule", "add", "-f", "--name", section, url, path) + self.git.git_operation("submodule","absorbgitdirs") + self.gitmodules.reload() + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + + + def translate_repo(self): + """ + Translates external repositories defined within an external file. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + external_file (str): The path to the external file containing repository definitions. + """ + econfig = ConfigParser() + econfig.read((self.rootpath / Path(self.externals))) + + for section in econfig.sections(): + if section == "externals_description": + logger.info("skipping section {}".format(section)) + return + logger.info("Translating section {}".format(section)) + tag = econfig.get(section, "tag", raw=False, fallback=None) + url = econfig.get(section, "repo_url", raw=False, fallback=None) + path = econfig.get(section, "local_path", raw=False, fallback=None) + efile = econfig.get(section, "externals", raw=False, fallback=None) + hash_ = econfig.get(section, "hash", raw=False, fallback=None) + sparse = econfig.get(section, "sparse", raw=False, fallback=None) + protocol = econfig.get(section, "protocol", raw=False, fallback=None) + + self.translate_single_repo(section, tag, url, path, efile, hash_, sparse, protocol) + + + +def _main(): + rootpath, gitmodules, externals = commandline_arguments() + global logger + logger = logging.getLogger(__name__) + with utils.pushd(rootpath): + t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals) + logger.info("Translating {}".format(rootpath)) + t.translate_repo() + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/pyproject.toml b/pyproject.toml index e8c7bd5..212b014 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ packages = [ [tool.poetry.scripts] git-fleximod = "git_fleximod.git_fleximod:main" +me2flexi = "git_fleximod.metoflexi:_main" fsspec = "fsspec.fuse:main" [tool.poetry.dependencies]