From dd2878532da77b3e8be22fc53d8f4edf51b2e48e Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:53:26 -0600 Subject: [PATCH 01/16] Create update_pyproject_version.py --- .../utils/update_pyproject_version.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py diff --git a/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py b/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py new file mode 100644 index 000000000..a518e8fd6 --- /dev/null +++ b/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py @@ -0,0 +1,135 @@ +import requests +from tomlkit import parse, dumps, inline_table +import os +from urllib.parse import urljoin + +def fetch_remote_pyproject_version(git_url, branch="main", subdirectory=""): + """ + Fetch the version from a remote pyproject.toml file in a GitHub repository. + + Args: + git_url (str): The Git repository URL. + branch (str): The branch to fetch from. + subdirectory (str): The subdirectory where the pyproject.toml resides. + + Returns: + str: The version string if found, otherwise None. + """ + try: + if "github.com" not in git_url: + raise ValueError("Only GitHub repositories are supported in this script.") + + # Remove trailing .git if present. + repo_path = git_url.split("github.com/")[1] + if repo_path.endswith(".git"): + repo_path = repo_path[:-4] + + # Build the raw URL. + base_url = f"https://raw.githubusercontent.com/{repo_path}/{branch}/" + if subdirectory and not subdirectory.endswith('/'): + subdirectory += '/' + pyproject_url = urljoin(base_url, f"{subdirectory}pyproject.toml") + + response = requests.get(pyproject_url) + response.raise_for_status() + doc = parse(response.text) + version = doc.get("tool", {}).get("poetry", {}).get("version") + return version + + except Exception as e: + print(f"Error fetching pyproject.toml from {git_url}: {e}") + return None + +def update_pyproject_with_versions(file_path): + """ + Read the local pyproject.toml, update versions for Git dependencies (including extras), + and ensure dependencies referenced in extras are marked as optional. + + Args: + file_path (str): Path to the local pyproject.toml file. + + Returns: + A tomlkit Document with updated dependency information, or None on error. + """ + try: + with open(file_path, "r") as f: + content = f.read() + doc = parse(content) + deps = doc["tool"]["poetry"]["dependencies"] + extras = doc["tool"]["poetry"].get("extras", {}) + + for dep_name, details in deps.items(): + # Check if the dependency is a table (dict-like) with a 'git' key. + if isinstance(details, dict) and "git" in details: + git_url = details["git"] + branch = details.get("branch", "main") + subdirectory = details.get("subdirectory", "") + + print(f"Updating dependency: {dep_name}") + print(f" Repository: {git_url}") + print(f" Branch: {branch}") + print(f" Subdirectory: {subdirectory}") + + version = fetch_remote_pyproject_version(git_url, branch=branch, subdirectory=subdirectory) + if version: + print(f" Found version: {version}") + # Create an inline table for the dependency using inline_table() + inline_dep = inline_table() + inline_dep["version"] = f"^{version}" + inline_dep["optional"] = True + deps[dep_name] = inline_dep + else: + print(f" Failed to fetch version for {dep_name}") + details["optional"] = True + else: + # For non-Git dependencies referenced in extras, + # if it's a simple string, convert it to an inline table. + for extra_name, extra_deps in extras.items(): + if dep_name in extra_deps: + if isinstance(details, str): + inline_dep = inline_table() + inline_dep["version"] = details + inline_dep["optional"] = True + deps[dep_name] = inline_dep + elif isinstance(details, dict): + details["optional"] = True + + # Clean extras: ensure each extra's list references valid dependencies. + for extra_name, extra_deps in extras.items(): + extras[extra_name] = [dep for dep in extra_deps if dep in deps] + + return doc + except Exception as e: + print(f"Error reading or updating pyproject.toml: {e}") + return None + +def update_and_write_pyproject(input_file_path, output_file_path=None): + """ + Updates the specified pyproject.toml file with resolved versions for Git dependencies + and writes the updated document to a file. + + Args: + input_file_path (str): Path to the original pyproject.toml file. + output_file_path (str, optional): Path to write the updated file. If not specified, + the input file will be overwritten. + + Returns: + True if the update and write succeed, False otherwise. + """ + updated_doc = update_pyproject_with_versions(input_file_path) + if updated_doc is None: + print("Failed to update the pyproject.toml document.") + return False + + # If no output path is provided, overwrite the input file. + if output_file_path is None: + output_file_path = input_file_path + + try: + with open(output_file_path, "w") as f: + f.write(dumps(updated_doc)) + print(f"Updated pyproject.toml written to {output_file_path}") + return True + except Exception as e: + print(f"Error writing updated pyproject.toml: {e}") + return False From 6a3b66a7ebd7404394069ce6317aac6c5e727df4 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:58:41 -0600 Subject: [PATCH 02/16] Delete pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py --- .../utils/update_pyproject_version.py | 135 ------------------ 1 file changed, 135 deletions(-) delete mode 100644 pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py diff --git a/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py b/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py deleted file mode 100644 index a518e8fd6..000000000 --- a/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/update_pyproject_version.py +++ /dev/null @@ -1,135 +0,0 @@ -import requests -from tomlkit import parse, dumps, inline_table -import os -from urllib.parse import urljoin - -def fetch_remote_pyproject_version(git_url, branch="main", subdirectory=""): - """ - Fetch the version from a remote pyproject.toml file in a GitHub repository. - - Args: - git_url (str): The Git repository URL. - branch (str): The branch to fetch from. - subdirectory (str): The subdirectory where the pyproject.toml resides. - - Returns: - str: The version string if found, otherwise None. - """ - try: - if "github.com" not in git_url: - raise ValueError("Only GitHub repositories are supported in this script.") - - # Remove trailing .git if present. - repo_path = git_url.split("github.com/")[1] - if repo_path.endswith(".git"): - repo_path = repo_path[:-4] - - # Build the raw URL. - base_url = f"https://raw.githubusercontent.com/{repo_path}/{branch}/" - if subdirectory and not subdirectory.endswith('/'): - subdirectory += '/' - pyproject_url = urljoin(base_url, f"{subdirectory}pyproject.toml") - - response = requests.get(pyproject_url) - response.raise_for_status() - doc = parse(response.text) - version = doc.get("tool", {}).get("poetry", {}).get("version") - return version - - except Exception as e: - print(f"Error fetching pyproject.toml from {git_url}: {e}") - return None - -def update_pyproject_with_versions(file_path): - """ - Read the local pyproject.toml, update versions for Git dependencies (including extras), - and ensure dependencies referenced in extras are marked as optional. - - Args: - file_path (str): Path to the local pyproject.toml file. - - Returns: - A tomlkit Document with updated dependency information, or None on error. - """ - try: - with open(file_path, "r") as f: - content = f.read() - doc = parse(content) - deps = doc["tool"]["poetry"]["dependencies"] - extras = doc["tool"]["poetry"].get("extras", {}) - - for dep_name, details in deps.items(): - # Check if the dependency is a table (dict-like) with a 'git' key. - if isinstance(details, dict) and "git" in details: - git_url = details["git"] - branch = details.get("branch", "main") - subdirectory = details.get("subdirectory", "") - - print(f"Updating dependency: {dep_name}") - print(f" Repository: {git_url}") - print(f" Branch: {branch}") - print(f" Subdirectory: {subdirectory}") - - version = fetch_remote_pyproject_version(git_url, branch=branch, subdirectory=subdirectory) - if version: - print(f" Found version: {version}") - # Create an inline table for the dependency using inline_table() - inline_dep = inline_table() - inline_dep["version"] = f"^{version}" - inline_dep["optional"] = True - deps[dep_name] = inline_dep - else: - print(f" Failed to fetch version for {dep_name}") - details["optional"] = True - else: - # For non-Git dependencies referenced in extras, - # if it's a simple string, convert it to an inline table. - for extra_name, extra_deps in extras.items(): - if dep_name in extra_deps: - if isinstance(details, str): - inline_dep = inline_table() - inline_dep["version"] = details - inline_dep["optional"] = True - deps[dep_name] = inline_dep - elif isinstance(details, dict): - details["optional"] = True - - # Clean extras: ensure each extra's list references valid dependencies. - for extra_name, extra_deps in extras.items(): - extras[extra_name] = [dep for dep in extra_deps if dep in deps] - - return doc - except Exception as e: - print(f"Error reading or updating pyproject.toml: {e}") - return None - -def update_and_write_pyproject(input_file_path, output_file_path=None): - """ - Updates the specified pyproject.toml file with resolved versions for Git dependencies - and writes the updated document to a file. - - Args: - input_file_path (str): Path to the original pyproject.toml file. - output_file_path (str, optional): Path to write the updated file. If not specified, - the input file will be overwritten. - - Returns: - True if the update and write succeed, False otherwise. - """ - updated_doc = update_pyproject_with_versions(input_file_path) - if updated_doc is None: - print("Failed to update the pyproject.toml document.") - return False - - # If no output path is provided, overwrite the input file. - if output_file_path is None: - output_file_path = input_file_path - - try: - with open(output_file_path, "w") as f: - f.write(dumps(updated_doc)) - print(f"Updated pyproject.toml written to {output_file_path}") - return True - except Exception as e: - print(f"Error writing updated pyproject.toml: {e}") - return False From 99d4d7d33f7dd8433b4269730b04eeba0085f565 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:59:05 -0600 Subject: [PATCH 03/16] Create update_pyproject_version.py --- scripts/update_pyproject_version.py | 135 ++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 scripts/update_pyproject_version.py diff --git a/scripts/update_pyproject_version.py b/scripts/update_pyproject_version.py new file mode 100644 index 000000000..a518e8fd6 --- /dev/null +++ b/scripts/update_pyproject_version.py @@ -0,0 +1,135 @@ +import requests +from tomlkit import parse, dumps, inline_table +import os +from urllib.parse import urljoin + +def fetch_remote_pyproject_version(git_url, branch="main", subdirectory=""): + """ + Fetch the version from a remote pyproject.toml file in a GitHub repository. + + Args: + git_url (str): The Git repository URL. + branch (str): The branch to fetch from. + subdirectory (str): The subdirectory where the pyproject.toml resides. + + Returns: + str: The version string if found, otherwise None. + """ + try: + if "github.com" not in git_url: + raise ValueError("Only GitHub repositories are supported in this script.") + + # Remove trailing .git if present. + repo_path = git_url.split("github.com/")[1] + if repo_path.endswith(".git"): + repo_path = repo_path[:-4] + + # Build the raw URL. + base_url = f"https://raw.githubusercontent.com/{repo_path}/{branch}/" + if subdirectory and not subdirectory.endswith('/'): + subdirectory += '/' + pyproject_url = urljoin(base_url, f"{subdirectory}pyproject.toml") + + response = requests.get(pyproject_url) + response.raise_for_status() + doc = parse(response.text) + version = doc.get("tool", {}).get("poetry", {}).get("version") + return version + + except Exception as e: + print(f"Error fetching pyproject.toml from {git_url}: {e}") + return None + +def update_pyproject_with_versions(file_path): + """ + Read the local pyproject.toml, update versions for Git dependencies (including extras), + and ensure dependencies referenced in extras are marked as optional. + + Args: + file_path (str): Path to the local pyproject.toml file. + + Returns: + A tomlkit Document with updated dependency information, or None on error. + """ + try: + with open(file_path, "r") as f: + content = f.read() + doc = parse(content) + deps = doc["tool"]["poetry"]["dependencies"] + extras = doc["tool"]["poetry"].get("extras", {}) + + for dep_name, details in deps.items(): + # Check if the dependency is a table (dict-like) with a 'git' key. + if isinstance(details, dict) and "git" in details: + git_url = details["git"] + branch = details.get("branch", "main") + subdirectory = details.get("subdirectory", "") + + print(f"Updating dependency: {dep_name}") + print(f" Repository: {git_url}") + print(f" Branch: {branch}") + print(f" Subdirectory: {subdirectory}") + + version = fetch_remote_pyproject_version(git_url, branch=branch, subdirectory=subdirectory) + if version: + print(f" Found version: {version}") + # Create an inline table for the dependency using inline_table() + inline_dep = inline_table() + inline_dep["version"] = f"^{version}" + inline_dep["optional"] = True + deps[dep_name] = inline_dep + else: + print(f" Failed to fetch version for {dep_name}") + details["optional"] = True + else: + # For non-Git dependencies referenced in extras, + # if it's a simple string, convert it to an inline table. + for extra_name, extra_deps in extras.items(): + if dep_name in extra_deps: + if isinstance(details, str): + inline_dep = inline_table() + inline_dep["version"] = details + inline_dep["optional"] = True + deps[dep_name] = inline_dep + elif isinstance(details, dict): + details["optional"] = True + + # Clean extras: ensure each extra's list references valid dependencies. + for extra_name, extra_deps in extras.items(): + extras[extra_name] = [dep for dep in extra_deps if dep in deps] + + return doc + except Exception as e: + print(f"Error reading or updating pyproject.toml: {e}") + return None + +def update_and_write_pyproject(input_file_path, output_file_path=None): + """ + Updates the specified pyproject.toml file with resolved versions for Git dependencies + and writes the updated document to a file. + + Args: + input_file_path (str): Path to the original pyproject.toml file. + output_file_path (str, optional): Path to write the updated file. If not specified, + the input file will be overwritten. + + Returns: + True if the update and write succeed, False otherwise. + """ + updated_doc = update_pyproject_with_versions(input_file_path) + if updated_doc is None: + print("Failed to update the pyproject.toml document.") + return False + + # If no output path is provided, overwrite the input file. + if output_file_path is None: + output_file_path = input_file_path + + try: + with open(output_file_path, "w") as f: + f.write(dumps(updated_doc)) + print(f"Updated pyproject.toml written to {output_file_path}") + return True + except Exception as e: + print(f"Error writing updated pyproject.toml: {e}") + return False From 215ebd6188bbc85285daea8d808ecd5bd456ee85 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:14:34 -0600 Subject: [PATCH 04/16] Create bump_pyproject_version.py --- scripts/bump_pyproject_version.py | 144 ++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/bump_pyproject_version.py diff --git a/scripts/bump_pyproject_version.py b/scripts/bump_pyproject_version.py new file mode 100644 index 000000000..e6a64b219 --- /dev/null +++ b/scripts/bump_pyproject_version.py @@ -0,0 +1,144 @@ +import sys +import argparse +from packaging.version import Version, InvalidVersion +from tomlkit import parse, dumps + +def read_pyproject_version(file_path): + """ + Reads the current version from the provided pyproject.toml file. + + Args: + file_path (str): Path to the pyproject.toml file. + + Returns: + (str, Document): A tuple containing the current version string and the + tomlkit Document. + """ + with open(file_path, "r") as f: + content = f.read() + doc = parse(content) + try: + version = doc["tool"]["poetry"]["version"] + except KeyError: + raise KeyError("No version found under [tool.poetry] in the given pyproject.toml") + return version, doc + +def bump_version(current_version, bump_type): + """ + Bumps the current version up using semantic versioning. + This function also supports bumping versions that have a dev segment. + + Args: + current_version (str): The current version (e.g. "0.1.0" or "0.6.0.dev1"). + bump_type (str): Type of bump: "major", "minor", or "patch". + + Returns: + str: The new version string. + """ + try: + ver = Version(current_version) + except InvalidVersion as e: + raise ValueError(f"Invalid current version '{current_version}': {e}") + + # Determine if this is a dev release. + is_dev = ver.dev is not None + major, minor, patch = ver.release + + if bump_type == "major": + major += 1 + minor = 0 + patch = 0 + new_version = f"{major}.{minor}.{patch}" + if is_dev: + new_version += ".dev1" + elif bump_type == "minor": + minor += 1 + patch = 0 + new_version = f"{major}.{minor}.{patch}" + if is_dev: + new_version += ".dev1" + elif bump_type == "patch": + if is_dev: + # Bump the dev counter (e.g. 0.6.0.dev1 -> 0.6.0.dev2). + new_dev = ver.dev + 1 + new_version = f"{major}.{minor}.{patch}.dev{new_dev}" + else: + patch += 1 + new_version = f"{major}.{minor}.{patch}" + else: + raise ValueError("bump_type must be one of: 'major', 'minor', 'patch'") + + return new_version + +def set_version(current_version, new_version): + """ + Validates that the new version is not lower than the current version. + + Args: + current_version (str): The current version. + new_version (str): The version to be set. + + Returns: + str: The new version if it's valid. + + Raises: + ValueError: If new_version is lower than current_version. + """ + try: + current_ver = Version(current_version) + new_ver = Version(new_version) + except InvalidVersion as e: + raise ValueError(f"Invalid version provided: {e}") + + if new_ver < current_ver: + raise ValueError("You cannot bump the version downwards. The target version must be higher than the current version.") + + return new_version + +def update_pyproject_version(file_path, new_version): + """ + Updates the pyproject.toml file with the specified new version. + + Args: + file_path (str): The path to the pyproject.toml file. + new_version (str): The new version string to set. + + Returns: + None + """ + current_version, doc = read_pyproject_version(file_path) + doc["tool"]["poetry"]["version"] = new_version + with open(file_path, "w") as f: + f.write(dumps(doc)) + print(f"Bumped version from {current_version} to {new_version} in {file_path}.") + +def main(): + parser = argparse.ArgumentParser( + description="Bump or set version in pyproject.toml using semantic versioning. Supports .dev versions." + ) + parser.add_argument("file", help="Path to the pyproject.toml file") + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--bump", choices=["major", "minor", "patch"], help="Type of version bump") + group.add_argument("--set", dest="set_version", help="Set the version explicitly (e.g. 0.2.0 or 0.2.0.dev1)") + + args = parser.parse_args() + + current_version, _ = read_pyproject_version(args.file) + + try: + if args.bump: + new_version = bump_version(current_version, args.bump) + elif args.set_version: + new_version = set_version(current_version, args.set_version) + else: + print("No operation specified.") + sys.exit(1) + + update_pyproject_version(args.file, new_version) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() From cd4b39fa88b0c3674c32b94c140e83d3ab6810f1 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:28:57 -0600 Subject: [PATCH 05/16] Update bump_pyproject_version.py --- scripts/bump_pyproject_version.py | 37 +++++++++++++++++++------------ 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/scripts/bump_pyproject_version.py b/scripts/bump_pyproject_version.py index e6a64b219..10502e0e3 100644 --- a/scripts/bump_pyproject_version.py +++ b/scripts/bump_pyproject_version.py @@ -26,11 +26,14 @@ def read_pyproject_version(file_path): def bump_version(current_version, bump_type): """ Bumps the current version up using semantic versioning. - This function also supports bumping versions that have a dev segment. + This function supports: + - Bumping a stable version to start a dev cycle. + - Bumping within a dev cycle. + - Finalizing a dev version (i.e. removing the .dev suffix). Args: - current_version (str): The current version (e.g. "0.1.0" or "0.6.0.dev1"). - bump_type (str): Type of bump: "major", "minor", or "patch". + current_version (str): The current version (e.g. "1.0.0" or "1.0.1.dev2"). + bump_type (str): Type of bump: "major", "minor", "patch", or "finalize". Returns: str: The new version string. @@ -40,33 +43,39 @@ def bump_version(current_version, bump_type): except InvalidVersion as e: raise ValueError(f"Invalid current version '{current_version}': {e}") - # Determine if this is a dev release. + # Is this a dev version? is_dev = ver.dev is not None major, minor, patch = ver.release - if bump_type == "major": + if bump_type == "finalize": + if is_dev: + # Remove the dev segment to finalize this dev release. + new_version = f"{major}.{minor}.{patch}" + else: + raise ValueError("Current version is stable; nothing to finalize.") + elif bump_type == "major": major += 1 minor = 0 patch = 0 new_version = f"{major}.{minor}.{patch}" - if is_dev: - new_version += ".dev1" + # Always start a dev cycle for a bump if desired. + new_version += ".dev1" elif bump_type == "minor": minor += 1 patch = 0 new_version = f"{major}.{minor}.{patch}" - if is_dev: - new_version += ".dev1" + new_version += ".dev1" elif bump_type == "patch": if is_dev: - # Bump the dev counter (e.g. 0.6.0.dev1 -> 0.6.0.dev2). + # If already in a dev cycle, bump the dev counter. new_dev = ver.dev + 1 new_version = f"{major}.{minor}.{patch}.dev{new_dev}" else: + # For a stable version, bump the patch and start a dev cycle. patch += 1 - new_version = f"{major}.{minor}.{patch}" + new_version = f"{major}.{minor}.{patch}.dev1" else: - raise ValueError("bump_type must be one of: 'major', 'minor', 'patch'") + raise ValueError("bump_type must be one of: 'major', 'minor', 'patch', or 'finalize'") return new_version @@ -114,12 +123,12 @@ def update_pyproject_version(file_path, new_version): def main(): parser = argparse.ArgumentParser( - description="Bump or set version in pyproject.toml using semantic versioning. Supports .dev versions." + description="Bump or set version in pyproject.toml using semantic versioning. Supports .dev versions and finalizing dev releases." ) parser.add_argument("file", help="Path to the pyproject.toml file") group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("--bump", choices=["major", "minor", "patch"], help="Type of version bump") + group.add_argument("--bump", choices=["major", "minor", "patch", "finalize"], help="Type of version bump (use 'finalize' to remove the .dev segment from a dev release)") group.add_argument("--set", dest="set_version", help="Set the version explicitly (e.g. 0.2.0 or 0.2.0.dev1)") args = parser.parse_args() From 62461f04ee992e990a5705fdabd2bd4a835be67c Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:05:47 -0600 Subject: [PATCH 06/16] Create add_comment_to_file.py --- .../utils/add_comment_to_file.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/add_comment_to_file.py diff --git a/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/add_comment_to_file.py b/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/add_comment_to_file.py new file mode 100644 index 000000000..51ca12bfc --- /dev/null +++ b/pkgs/experimental/swarmauri_experimental/swarmauri_experimental/utils/add_comment_to_file.py @@ -0,0 +1,50 @@ +import os + +# Define a map for comment styles based on file extensions +COMMENT_STYLES = { + '.py': '#', # Python + '.js': '//', # JavaScript + '.java': '//', # Java + '.html': '', # HTML + '.css': '/* */', # CSS +} + +def get_comment_style(file_path): + """ + Determine the comment style based on the file extension. + """ + _, ext = os.path.splitext(file_path) + return COMMENT_STYLES.get(ext, None) + +def add_comment(file_path, comment_text): + """ + Add a comment to the top of a file with the appropriate comment style. + """ + comment_style = get_comment_style(file_path) + if not comment_style: + print(f"Unsupported file type for: {file_path}") + return + + # Read the existing content + with open(file_path, 'r') as file: + content = file.readlines() + + # Prepare the comment based on the style + if comment_style in ['#', '//']: + comment = f"{comment_style} {comment_text}\n" + elif comment_style == '': + comment = f"\n" + elif comment_style == '/* */': + comment = f"/* {comment_text} */\n" + else: + print(f"Unsupported comment style: {comment_style}") + return + + # Write the comment at the top and add the existing content + with open(file_path, 'w') as file: + file.write(comment) + file.writelines(content) + +# Example usage +add_comment('example.py', 'This is a Python file') +add_comment('example.js', 'This is a JavaScript file') From 94b39e6a538e73ffccafafbfcf2ca353012d7953 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:06:22 -0600 Subject: [PATCH 07/16] tooling - add monorepo_manager --- pkgs/tooling/monorepo_manager/LICENSE | 0 pkgs/tooling/monorepo_manager/README.md | 171 ++++++++++ .../monorepo_manager/__init__.py | 14 + .../monorepo_manager/monorepo_manager/cli.py | 163 +++++++++ .../monorepo_manager/poetry_ops.py | 308 ++++++++++++++++++ .../monorepo_manager/pyproject_ops.py | 188 +++++++++++ .../monorepo_manager/remote_ops.py | 187 +++++++++++ .../monorepo_manager/test_ops.py | 208 ++++++++++++ .../monorepo_manager/version_ops.py | 208 ++++++++++++ pkgs/tooling/monorepo_manager/pyproject.toml | 21 ++ 10 files changed, 1468 insertions(+) create mode 100644 pkgs/tooling/monorepo_manager/LICENSE create mode 100644 pkgs/tooling/monorepo_manager/README.md create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/__init__.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/cli.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/pyproject_ops.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/remote_ops.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/test_ops.py create mode 100644 pkgs/tooling/monorepo_manager/monorepo_manager/version_ops.py create mode 100644 pkgs/tooling/monorepo_manager/pyproject.toml diff --git a/pkgs/tooling/monorepo_manager/LICENSE b/pkgs/tooling/monorepo_manager/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md new file mode 100644 index 000000000..632624a98 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/README.md @@ -0,0 +1,171 @@ +Below is an example of a comprehensive `README.md` for your unified monorepo management tool: + +--- + +```markdown +# Monorepo Manager + +**Monorepo Manager** is a unified command-line tool for managing a Python monorepo that contains multiple standalone packages, each with its own `pyproject.toml`. It consolidates several common tasks—including dependency management, version bumping, remote dependency resolution, test analysis, and project configuration updates—into one robust CLI. + +## Features + +- **Dependency Management** + - **Lock:** Generate a `poetry.lock` file. + - **Install:** Install dependencies with options for extras and development dependencies. + - **Show Freeze:** Display installed packages using `pip freeze`. + +- **Version Management** + - **Version:** Bump (major, minor, patch, finalize) or explicitly set package versions in `pyproject.toml`. + +- **Remote Operations** + - **Remote Fetch:** Fetch the version from a remote GitHub repository’s `pyproject.toml`. + - **Remote Update:** Update a local `pyproject.toml` file with version information from remote Git dependencies. + +- **Test Analysis** + - **Test:** Parse a JSON file with test results, display summary statistics, and evaluate threshold conditions. + +- **Pyproject Operations** + - **Pyproject:** Extract local (path) and Git-based dependencies from a `pyproject.toml` file and optionally update dependency versions. + +## Installation + + ```bash + pip install monorepo-manager + ``` + +_This installs the `monorepo-manager` CLI (provided via the entry point `monorepo-manager`) into your system PATH._ + +## Usage + +Once installed, you can invoke the CLI using: + +```bash +monorepo-manager --help +``` + +This displays the list of available commands. + +### Command Examples + +#### 1. Lock Dependencies + +Generate a `poetry.lock` file for a package by specifying the directory or file path: + +```bash +# Lock using a directory containing pyproject.toml: +monorepo-manager lock --directory ./packages/package1 + +# Lock using an explicit pyproject.toml file: +monorepo-manager lock --file ./packages/package1/pyproject.toml +``` + +#### 2. Install Dependencies + +Install dependencies with various options: + +```bash +# Basic installation: +monorepo-manager install --directory ./packages/package1 + +# Install using an explicit pyproject.toml file: +monorepo-manager install --file ./packages/package1/pyproject.toml + +# Install including development dependencies: +monorepo-manager install --directory ./packages/package1 --dev + +# Install including extras (e.g., extras named "full"): +monorepo-manager install --directory ./packages/package2 --extras full + +# Install including all extras: +monorepo-manager install --directory ./packages/package2 --all-extras +``` + +#### 3. Version Management + +Bump the version or set it explicitly for a given package: + +```bash +# Bump the patch version (e.g. from 1.2.3.dev1 to 1.2.3.dev2): +monorepo-manager version ./packages/package1/pyproject.toml --bump patch + +# Finalize a development version (remove the ".dev" suffix): +monorepo-manager version ./packages/package1/pyproject.toml --bump finalize + +# Set an explicit version: +monorepo-manager version ./packages/package1/pyproject.toml --set 2.0.0.dev1 +``` + +#### 4. Remote Operations + +Fetch version information from a remote GitHub repository’s `pyproject.toml` and update your local configuration accordingly. + +```bash +# Fetch the remote version: +monorepo-manager remote fetch --git-url https://github.com/YourOrg/YourRepo.git --branch main --subdir "src/" + +# Update a local pyproject.toml with versions resolved from remote dependencies. +# (If --output is omitted, the input file is overwritten.) +monorepo-manager remote update --input ./packages/package1/pyproject.toml --output ./packages/package1/pyproject.updated.toml +``` + +#### 5. Test Analysis + +Analyze test results provided in a JSON file, and enforce percentage thresholds for passed and skipped tests: + +```bash +# Analyze test results without thresholds: +monorepo-manager test test-results.json + +# Analyze test results with thresholds: require that passed tests are greater than 75% and skipped tests are less than 20%: +monorepo-manager test test-results.json --required-passed gt:75 --required-skipped lt:20 +``` + +#### 6. Pyproject Operations + +Operate on the `pyproject.toml` file to extract dependency information and optionally update dependency versions: + +```bash +# Extract local (path) and Git-based dependencies: +monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml + +# Update local dependency versions to 2.0.0 (updates parent file and, if possible, each dependency's own pyproject.toml): +monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml --update-version 2.0.0 +``` + +## Development + +### Project Structure + +``` +monorepo_manager/ +├── __init__.py +├── cli.py # Main CLI entry point +├── poetry_ops.py # Poetry operations (lock, install, build, publish, etc.) +├── version_ops.py # Version bumping and setting operations +├── remote_ops.py # Remote Git dependency version fetching/updating +├── test_ops.py # Test result analysis operations +└── pyproject_ops.py # pyproject.toml dependency extraction and updates +pyproject.toml # Package configuration file containing metadata +README.md # This file +``` + +### Running Tests + +For development purposes, you can use your favorite test runner (such as `pytest`) to run tests for the CLI and modules. + +```bash +pytest +``` + +## Contributing + +Contributions are welcome! Feel free to open issues or submit pull requests for improvements or bug fixes. + +## License + +This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. +``` + +--- + +This `README.md` provides a detailed overview, installation instructions, and comprehensive usage examples for each command offered by your CLI tool. Feel free to adjust sections or add any additional details specific to your project needs. \ No newline at end of file diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/__init__.py b/pkgs/tooling/monorepo_manager/monorepo_manager/__init__.py new file mode 100644 index 000000000..429280c47 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/__init__.py @@ -0,0 +1,14 @@ +# monorepo_manager/__init__.py + +try: + # For Python 3.8 and newer + from importlib.metadata import version, PackageNotFoundError +except ImportError: + # For older Python versions, use the backport + from importlib_metadata import version, PackageNotFoundError + +try: + __version__ = version("monorepo-manager") +except PackageNotFoundError: + # If the package is not installed (for example, during development) + __version__ = "0.0.0" diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py new file mode 100644 index 000000000..c3c4d3196 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +cli.py + +This is the main entry point for the monorepo management CLI. +It provides commands to: + - Manage Poetry-based operations (lock, install, show pip-freeze, recursive build, publish) + - Manage version operations (bump or set versions in pyproject.toml) + - Manage remote operations (fetch/update Git dependency versions) + - Analyze test results from a JSON file + - Operate on pyproject.toml files (extract and update dependency versions) + +The commands are intentionally named with simple terms (e.g. "lock" instead of "poetry lock", +"install" instead of "poetry install", and "test" instead of "test-analyze"). +""" + +import argparse +import sys + +# Import operations from the local modules. +from monorepo_manager import poetry_ops +from monorepo_manager import version_ops +from monorepo_manager import remote_ops +from monorepo_manager import test_ops +from monorepo_manager import pyproject_ops + + +def main(): + parser = argparse.ArgumentParser( + description="A CLI for managing a Python monorepo with multiple standalone packages." + ) + subparsers = parser.add_subparsers(dest="command", required=True, help="Available commands") + + # ------------------------------------------------ + # Command: lock + # ------------------------------------------------ + lock_parser = subparsers.add_parser("lock", help="Generate a poetry.lock file") + lock_parser.add_argument("--directory", type=str, help="Directory containing a pyproject.toml") + lock_parser.add_argument("--file", type=str, help="Explicit path to a pyproject.toml file") + + # ------------------------------------------------ + # Command: install + # ------------------------------------------------ + install_parser = subparsers.add_parser("install", help="Install dependencies") + install_parser.add_argument("--directory", type=str, help="Directory containing a pyproject.toml") + install_parser.add_argument("--file", type=str, help="Explicit path to a pyproject.toml file") + install_parser.add_argument("--extras", type=str, help="Extras to include (e.g. 'full')") + install_parser.add_argument("--dev", action="store_true", help="Include dev dependencies") + install_parser.add_argument("--all-extras", action="store_true", help="Include all extras") + + # ------------------------------------------------ + # Command: version + # ------------------------------------------------ + version_parser = subparsers.add_parser("version", help="Bump or set package version") + version_parser.add_argument("pyproject_file", type=str, help="Path to the pyproject.toml file") + vgroup = version_parser.add_mutually_exclusive_group(required=True) + vgroup.add_argument("--bump", choices=["major", "minor", "patch", "finalize"], + help="Bump the version (e.g. patch, major, minor, finalize)") + vgroup.add_argument("--set", dest="set_ver", help="Explicit version to set (e.g. 2.0.0.dev1)") + + # ------------------------------------------------ + # Command: remote + # ------------------------------------------------ + remote_parser = subparsers.add_parser("remote", help="Remote operations for Git dependencies") + remote_subparsers = remote_parser.add_subparsers(dest="remote_cmd", required=True) + + # remote fetch: fetch version from remote GitHub pyproject.toml + fetch_parser = remote_subparsers.add_parser("fetch", help="Fetch version from remote GitHub pyproject.toml") + fetch_parser.add_argument("--git-url", type=str, required=True, help="GitHub repository URL") + fetch_parser.add_argument("--branch", type=str, default="main", help="Branch name (default: main)") + fetch_parser.add_argument("--subdir", type=str, default="", help="Subdirectory where pyproject.toml is located") + + # remote update: update a local pyproject.toml with remote resolved versions. + update_parser = remote_subparsers.add_parser("update", help="Update local pyproject.toml with remote versions") + update_parser.add_argument("--input", required=True, help="Path to the local pyproject.toml") + update_parser.add_argument("--output", help="Optional output file path (defaults to overwriting the input)") + + # ------------------------------------------------ + # Command: test + # ------------------------------------------------ + test_parser = subparsers.add_parser("test", help="Analyze test results from a JSON file") + test_parser.add_argument("file", help="Path to the JSON file with test results") + test_parser.add_argument("--required-passed", type=str, help="Threshold for passed tests (e.g. 'gt:75')") + test_parser.add_argument("--required-skipped", type=str, help="Threshold for skipped tests (e.g. 'lt:20')") + + # ------------------------------------------------ + # Command: pyproject + # ------------------------------------------------ + pyproject_parser = subparsers.add_parser("pyproject", help="Operate on pyproject.toml dependencies") + pyproject_parser.add_argument("--pyproject", required=True, help="Path to the pyproject.toml file") + pyproject_parser.add_argument("--update-version", type=str, help="Update local dependency versions to this version") + + # ------------------------------------------------ + # Dispatch Commands + # ------------------------------------------------ + args = parser.parse_args() + + if args.command == "lock": + poetry_ops.poetry_lock(directory=args.directory, file=args.file) + + elif args.command == "install": + poetry_ops.poetry_install( + directory=args.directory, + file=args.file, + extras=args.extras, + with_dev=args.dev, + all_extras=args.all_extras + ) + + elif args.command == "version": + version_ops.bump_or_set_version(args.pyproject_file, bump=args.bump, set_ver=args.set_ver) + + elif args.command == "remote": + if args.remote_cmd == "fetch": + ver = remote_ops.fetch_remote_pyproject_version( + git_url=args.git_url, + branch=args.branch, + subdirectory=args.subdir + ) + if ver: + print(f"Fetched remote version: {ver}") + else: + print("Failed to fetch remote version.") + elif args.remote_cmd == "update": + success = remote_ops.update_and_write_pyproject(args.input, args.output) + if not success: + sys.exit(1) + + elif args.command == "test": + test_ops.analyze_test_file( + file_path=args.file, + required_passed=args.required_passed, + required_skipped=args.required_skipped + ) + + elif args.command == "pyproject": + print("Extracting dependencies from pyproject.toml ...") + paths = pyproject_ops.extract_path_dependencies(args.pyproject) + if paths: + print("Local (path) dependencies:") + print(", ".join(paths)) + else: + print("No local path dependencies found.") + + git_deps = pyproject_ops.extract_git_dependencies(args.pyproject) + if git_deps: + print("\nGit dependencies:") + for name, details in git_deps.items(): + print(f"{name}: {details}") + else: + print("No Git dependencies found.") + + if args.update_version: + print(f"\nUpdating local dependency versions to {args.update_version} ...") + pyproject_ops.update_dependency_versions(args.pyproject, args.update_version) + + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py new file mode 100644 index 000000000..71bbb70fd --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +""" +poetry_ops.py + +Provides functions to: + - Install Poetry + - Run 'poetry lock' and 'poetry install' + - Extract path dependencies from pyproject.toml + - Recursively build packages (based on the dependencies) + - Show the installed packages (via pip freeze) + - Set versions and dependency versions in pyproject.toml files + - Publish built packages to PyPI + - Publish packages based on path dependencies + +Intended for use in a unified monorepo management CLI. +""" + +import argparse +import os +import subprocess +import sys +import toml + + +def run_command(command, cwd=None): + """Run a shell command and handle errors.""" + try: + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + shell=True, + check=True, + ) + if result.stdout: + print(result.stdout) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Error running command: {e.stderr}", file=sys.stderr) + sys.exit(e.returncode) + + +def install_poetry(): + """Install Poetry.""" + print("Installing Poetry...") + run_command("curl -sSL https://install.python-poetry.org | python3") + # Update PATH so that ~/.local/bin is included for subsequent commands. + os.environ["PATH"] = f"{os.path.expanduser('~')}/.local/bin:{os.environ.get('PATH', '')}" + + +def poetry_lock(directory=None, file=None): + """ + Run 'poetry lock' in the specified directory or on the specified file's directory. + + :param directory: Directory containing pyproject.toml. + :param file: Path to a specific pyproject.toml file. + """ + location = directory if directory else os.path.dirname(file) + print(f"Generating poetry.lock in {location}...") + run_command("poetry lock", cwd=location) + + +def poetry_install(directory=None, file=None, extras=None, with_dev=False, all_extras=False): + """ + Run 'poetry install' in the specified directory or file. + + :param directory: Directory containing pyproject.toml. + :param file: Path to a specific pyproject.toml file. + :param extras: Extras to include (e.g., "full"). + :param with_dev: Boolean flag to include dev dependencies. + :param all_extras: Boolean flag to include all extras. + """ + location = directory if directory else os.path.dirname(file) + print(f"Installing dependencies in {location}...") + command = ["poetry", "install", "--no-cache", "-vv"] + if all_extras: + command.append("--all-extras") + elif extras: + command.append(f"--extras {extras}") + if with_dev: + command.append("--with dev") + run_command(" ".join(command), cwd=location) + + +def extract_path_dependencies(pyproject_path): + """ + Extract path dependencies from a pyproject.toml file. + + Looks for dependency entries that are defined as tables with a "path" key. + + :param pyproject_path: Path to the pyproject.toml file. + :return: List of dependency paths found. + """ + print(f"Extracting path dependencies from {pyproject_path}...") + try: + with open(pyproject_path, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_path}: {e}", file=sys.stderr) + sys.exit(1) + + dependencies = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) + path_deps = [v["path"] for v in dependencies.values() if isinstance(v, dict) and "path" in v] + return path_deps + + +def recursive_build(directory=None, file=None): + """ + Recursively build packages based on path dependencies extracted from a pyproject.toml. + + :param directory: Directory containing pyproject.toml. + :param file: Specific pyproject.toml file to use. + """ + pyproject_path = file if file else os.path.join(directory, "pyproject.toml") + base_dir = os.path.dirname(pyproject_path) + dependencies = extract_path_dependencies(pyproject_path) + + print("Building specified packages...") + for package_path in dependencies: + full_path = os.path.join(base_dir, package_path) + pyproject_file = os.path.join(full_path, "pyproject.toml") + if os.path.isdir(full_path) and os.path.isfile(pyproject_file): + print(f"Building package: {full_path}") + run_command("poetry build", cwd=full_path) + else: + print(f"Skipping {full_path}: not a valid package directory") + + +def show_pip_freeze(): + """ + Show the installed packages using pip freeze. + """ + print("Installed packages (pip freeze):") + run_command("pip freeze") + + +def set_version_in_pyproject(version, directory=None, file=None): + """ + Set the version for the package in the pyproject.toml file(s). + + :param version: The new version string to set. + :param directory: If provided, recursively update all pyproject.toml files found in the directory. + :param file: If provided, update the specific pyproject.toml file. + """ + def update_file(pyproject_file, version): + print(f"Setting version to {version} in {pyproject_file}...") + try: + with open(pyproject_file, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_file}: {e}", file=sys.stderr) + return + + if "tool" in data and "poetry" in data["tool"]: + data["tool"]["poetry"]["version"] = version + else: + print(f"Invalid pyproject.toml structure in {pyproject_file}.", file=sys.stderr) + return + + try: + with open(pyproject_file, "w") as f: + toml.dump(data, f) + print(f"Version set to {version} in {pyproject_file}.") + except Exception as e: + print(f"Error writing {pyproject_file}: {e}", file=sys.stderr) + + if directory: + for root, _, files in os.walk(directory): + if "pyproject.toml" in files: + pyproject_file = os.path.join(root, "pyproject.toml") + update_file(pyproject_file, version) + elif file: + update_file(file, version) + else: + print("Error: A directory or a file must be specified.", file=sys.stderr) + sys.exit(1) + + +def set_dependency_versions(version, directory=None, file=None): + """ + Update versions for path dependencies found in pyproject.toml files. + + For each dependency that is defined with a "path", update its version to '^'. + Also attempts to update the dependency's own pyproject.toml. + + :param version: The new version string (without the caret). + :param directory: A directory to search for pyproject.toml files. + :param file: A specific pyproject.toml file to update. + """ + def update_file(pyproject_file, version): + print(f"Setting dependency versions to {version} in {pyproject_file}...") + try: + with open(pyproject_file, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_file}: {e}", file=sys.stderr) + return + + dependencies = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) + updated_dependencies = {} + + for dep_name, dep_value in dependencies.items(): + if isinstance(dep_value, dict) and "path" in dep_value: + # Update the version entry while preserving other keys except 'path' + updated_dep = {"version": f"^{version}"} + for key, val in dep_value.items(): + if key != "path": + updated_dep[key] = val + updated_dependencies[dep_name] = updated_dep + + # Update the dependency's own pyproject.toml file if present. + dependency_path = os.path.join(os.path.dirname(pyproject_file), dep_value["path"]) + dependency_pyproject = os.path.join(dependency_path, "pyproject.toml") + if os.path.isfile(dependency_pyproject): + print(f"Updating version in dependency: {dependency_pyproject} to {version}") + set_version_in_pyproject(version, file=dependency_pyproject) + else: + print(f"Warning: pyproject.toml not found at {dependency_pyproject}") + else: + updated_dependencies[dep_name] = dep_value + + # Write back the updated dependencies. + if "tool" in data and "poetry" in data["tool"]: + data["tool"]["poetry"]["dependencies"] = updated_dependencies + else: + print(f"Error: Invalid pyproject.toml structure in {pyproject_file}.", file=sys.stderr) + return + + try: + with open(pyproject_file, "w") as f: + toml.dump(data, f) + print(f"Dependency versions set to {version} in {pyproject_file}.") + except Exception as e: + print(f"Error writing {pyproject_file}: {e}", file=sys.stderr) + + if directory: + for root, _, files in os.walk(directory): + if "pyproject.toml" in files: + update_file(os.path.join(root, "pyproject.toml"), version) + elif file: + update_file(file, version) + else: + print("Error: A directory or a file must be specified.", file=sys.stderr) + sys.exit(1) + + +def publish_package(directory=None, file=None, username=None, password=None): + """ + Build and publish packages to PyPI. + + :param directory: Directory containing one or more packages. + :param file: Specific pyproject.toml file to use. + :param username: PyPI username. + :param password: PyPI password. + """ + if directory: + print(f"Publishing all packages in {directory} and its subdirectories...") + for root, dirs, files in os.walk(directory): + if "pyproject.toml" in files: + print(f"Publishing package from {root}...") + run_command("poetry build", cwd=root) + run_command( + f"poetry publish --username {username} --password {password}", + cwd=root, + ) + elif file: + location = os.path.dirname(file) + print(f"Publishing package from {location}...") + run_command("poetry build", cwd=location) + run_command( + f"poetry publish --username {username} --password {password}", + cwd=location, + ) + else: + print("Error: Either a directory or a file must be specified.", file=sys.stderr) + sys.exit(1) + + +def publish_from_dependencies(directory=None, file=None, username=None, password=None): + """ + Build and publish packages based on path dependencies defined in a pyproject.toml. + + :param directory: Directory containing the base pyproject.toml. + :param file: Specific pyproject.toml file. + :param username: PyPI username. + :param password: PyPI password. + """ + pyproject_path = file if file else os.path.join(directory, "pyproject.toml") + if not os.path.isfile(pyproject_path): + print(f"pyproject.toml not found at {pyproject_path}", file=sys.stderr) + sys.exit(1) + + base_dir = os.path.dirname(pyproject_path) + dependencies = extract_path_dependencies(pyproject_path) + print("Building and publishing packages based on path dependencies...") + for package_path in dependencies: + full_path = os.path.join(base_dir, package_path) + pyproject_file = os.path.join(full_path, "pyproject.toml") + if os.path.isdir(full_path) and os.path.isfile(pyproject_file): + print(f"Building and publishing package: {full_path}") + run_command("poetry build", cwd=full_path) + run_command( + f"poetry publish --username {username} --password {password}", + cwd=full_path, + ) + else: + print(f"Skipping {full_path}: not a valid package directory") diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/pyproject_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/pyproject_ops.py new file mode 100644 index 000000000..1c5738e77 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/pyproject_ops.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +pyproject_ops.py + +Provides functions for operating on pyproject.toml files: + - extract_path_dependencies: Retrieves dependencies that are defined with a "path". + - extract_git_dependencies: Retrieves dependencies that are defined with a "git" key. + - update_dependency_versions: For local dependencies (with a "path"), update their version in the parent + pyproject.toml and optionally in the dependency’s own pyproject.toml. + +These functions can be used independently or integrated into a larger monorepo management tool. +""" + +import os +import sys +import toml + + +def extract_path_dependencies(pyproject_path): + """ + Extract local (path) dependencies from a pyproject.toml file. + + Looks for dependencies in [tool.poetry.dependencies] that are dictionaries containing a "path" key. + + Args: + pyproject_path (str): Path to the pyproject.toml file. + + Returns: + list: A list of path strings extracted from the dependency definitions. + """ + try: + with open(pyproject_path, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_path}: {e}", file=sys.stderr) + sys.exit(1) + + dependencies = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) + path_deps = [ + value["path"] + for value in dependencies.values() + if isinstance(value, dict) and "path" in value + ] + return path_deps + + +def extract_git_dependencies(pyproject_path): + """ + Extract Git-based dependencies from a pyproject.toml file. + + Looks for dependencies in [tool.poetry.dependencies] that are dictionaries containing a "git" key. + + Args: + pyproject_path (str): Path to the pyproject.toml file. + + Returns: + dict: A dictionary mapping dependency names to their details dictionaries. + """ + try: + with open(pyproject_path, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_path}: {e}", file=sys.stderr) + sys.exit(1) + + dependencies = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) + git_deps = { + name: details + for name, details in dependencies.items() + if isinstance(details, dict) and "git" in details + } + return git_deps + + +def update_dependency_versions(pyproject_path, new_version): + """ + Update versions for local (path) dependencies in a pyproject.toml file. + + For each dependency that is defined as a table with a "path" key: + - The dependency’s version is updated to f"^{new_version}" in the parent pyproject.toml. + - Attempts to update the dependency's own pyproject.toml (if found in the given path) + by setting its version to new_version. + + Args: + pyproject_path (str): Path to the parent pyproject.toml file. + new_version (str): The new version string to set (without the caret). + + Returns: + None + """ + try: + with open(pyproject_path, "r") as f: + data = toml.load(f) + except Exception as e: + print(f"Error reading {pyproject_path}: {e}", file=sys.stderr) + sys.exit(1) + + poetry_section = data.get("tool", {}).get("poetry", {}) + dependencies = poetry_section.get("dependencies", {}) + updated_deps = {} + base_dir = os.path.dirname(pyproject_path) + + for dep_name, details in dependencies.items(): + if isinstance(details, dict) and "path" in details: + # Create a new dependency definition with an updated version. + new_dep = {"version": f"^{new_version}"} + # Preserve any additional keys (except we override version). + for key, value in details.items(): + if key != "path": + new_dep[key] = value + updated_deps[dep_name] = new_dep + + # Attempt to update the dependency's own pyproject.toml (if it exists). + dependency_path = os.path.join(base_dir, details["path"]) + dependency_pyproject = os.path.join(dependency_path, "pyproject.toml") + if os.path.isfile(dependency_pyproject): + try: + with open(dependency_pyproject, "r") as dep_file: + dep_data = toml.load(dep_file) + if "tool" in dep_data and "poetry" in dep_data["tool"]: + dep_data["tool"]["poetry"]["version"] = new_version + with open(dependency_pyproject, "w") as dep_file: + toml.dump(dep_data, dep_file) + print(f"Updated {dependency_pyproject} to version {new_version}") + else: + print(f"Invalid structure in {dependency_pyproject}", file=sys.stderr) + except Exception as e: + print(f"Error updating {dependency_pyproject}: {e}", file=sys.stderr) + else: + updated_deps[dep_name] = details + + # Write the updated dependencies back to the parent pyproject.toml. + data["tool"]["poetry"]["dependencies"] = updated_deps + try: + with open(pyproject_path, "w") as f: + toml.dump(data, f) + print(f"Updated dependency versions in {pyproject_path}") + except Exception as e: + print(f"Error writing updated file {pyproject_path}: {e}", file=sys.stderr) + sys.exit(1) + + +def main(): + """ + Provides a basic CLI for testing pyproject.toml operations. + + Usage Examples: + - Extract dependencies: + python pyproject_ops.py --pyproject path/to/pyproject.toml + - Update dependency versions (for local path dependencies): + python pyproject_ops.py --pyproject path/to/pyproject.toml --update-version 2.0.0 + """ + import argparse + + parser = argparse.ArgumentParser(description="Operate on pyproject.toml dependencies") + parser.add_argument( + "--pyproject", + required=True, + help="Path to the pyproject.toml file", + ) + parser.add_argument( + "--update-version", + help="If provided, update local dependencies to this version", + ) + args = parser.parse_args() + + print("Extracting local (path) dependencies:") + paths = extract_path_dependencies(args.pyproject) + if paths: + print(", ".join(paths)) + else: + print("No path dependencies found.") + + print("\nExtracting Git dependencies:") + git_deps = extract_git_dependencies(args.pyproject) + if git_deps: + for name, details in git_deps.items(): + print(f"{name}: {details}") + else: + print("No Git dependencies found.") + + if args.update_version: + print(f"\nUpdating dependency versions to {args.update_version} ...") + update_dependency_versions(args.pyproject, args.update_version) + + +if __name__ == "__main__": + main() diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/remote_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/remote_ops.py new file mode 100644 index 000000000..33b0a2aa0 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/remote_ops.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +remote_ops.py + +Provides functions to: + - Fetch the version from a remote GitHub repository's pyproject.toml. + - Update a local pyproject.toml by resolving Git dependencies: + - For dependencies defined with a 'git' key, + - Replace their version with an inline table containing the fetched version, + - Mark dependencies as optional. + - Write the updated pyproject.toml to a file (or overwrite the input file). + +Intended for use in a unified monorepo management CLI. +""" + +import os +from urllib.parse import urljoin + +import requests +from tomlkit import parse, dumps, inline_table + + +def fetch_remote_pyproject_version(git_url, branch="main", subdirectory=""): + """ + Fetches the version string from a remote pyproject.toml in a GitHub repository. + + Args: + git_url (str): The Git repository URL (must be a GitHub URL). + branch (str): The branch to fetch the file from (default: "main"). + subdirectory (str): The subdirectory in the repo where the pyproject.toml is located (if any). + + Returns: + str or None: The version string if found, otherwise None. + """ + try: + if "github.com" not in git_url: + raise ValueError("Only GitHub repositories are supported by this function.") + + # Remove trailing .git if present. + repo_path = git_url.split("github.com/")[1] + if repo_path.endswith(".git"): + repo_path = repo_path[:-4] + + # Build the raw URL; ensure subdirectory ends with "/" if provided. + base_url = f"https://raw.githubusercontent.com/{repo_path}/{branch}/" + if subdirectory and not subdirectory.endswith("/"): + subdirectory += "/" + pyproject_url = urljoin(base_url, f"{subdirectory}pyproject.toml") + + response = requests.get(pyproject_url) + response.raise_for_status() + doc = parse(response.text) + version = doc.get("tool", {}).get("poetry", {}).get("version") + if version is None: + print(f"Version key not found in remote pyproject.toml from {pyproject_url}") + return version + except Exception as e: + print(f"Error fetching pyproject.toml from {git_url}: {e}") + return None + + +def update_pyproject_with_versions(file_path): + """ + Reads the local pyproject.toml file and updates Git-based dependencies. + + For dependencies defined as a table with a 'git' key, it: + - Fetches the version from the remote repository. + - Creates an inline table for the dependency with the resolved version (prefixed with '^'). + - Marks the dependency as optional. + Also ensures that dependencies referenced in extras are marked as optional. + + Args: + file_path (str): Path to the local pyproject.toml file. + + Returns: + tomlkit.document.Document: The updated TOML document. + If an error occurs, prints the error and returns None. + """ + try: + with open(file_path, "r") as f: + content = f.read() + doc = parse(content) + except Exception as e: + print(f"Error reading {file_path}: {e}") + return None + + try: + tool_section = doc["tool"] + poetry_section = tool_section["poetry"] + except KeyError: + print(f"Error: Invalid pyproject.toml structure in {file_path}.", flush=True) + return None + + dependencies = poetry_section.get("dependencies", {}) + extras = poetry_section.get("extras", {}) + + for dep_name, details in dependencies.items(): + # Process only Git-based dependencies. + if isinstance(details, dict) and "git" in details: + git_url = details["git"] + branch = details.get("branch", "main") + subdirectory = details.get("subdirectory", "") + print(f"Updating dependency '{dep_name}':") + print(f" Repository: {git_url}") + print(f" Branch: {branch}") + print(f" Subdirectory: {subdirectory}") + remote_version = fetch_remote_pyproject_version(git_url, branch=branch, subdirectory=subdirectory) + if remote_version: + print(f" Fetched version: {remote_version}") + # Create an inline table with the resolved version and mark as optional. + dep_inline = inline_table() + dep_inline["version"] = f"^{remote_version}" + dep_inline["optional"] = True + dependencies[dep_name] = dep_inline + else: + print(f" Could not fetch remote version for '{dep_name}'. Marking as optional.") + # Mark as optional if version could not be fetched. + details["optional"] = True + dependencies[dep_name] = details + else: + # If the dependency appears in extras but is just a string, convert it to an inline table and mark as optional. + for extra_name, extra_deps in extras.items(): + if dep_name in extra_deps: + if isinstance(details, str): + dep_inline = inline_table() + dep_inline["version"] = details + dep_inline["optional"] = True + dependencies[dep_name] = dep_inline + elif isinstance(details, dict): + details["optional"] = True + break # Only need to update once. + + # Clean the extras section: ensure each extra only contains dependencies that exist. + for extra_name, extra_deps in extras.items(): + extras[extra_name] = [dep for dep in extra_deps if dep in dependencies] + + # Update the document. + poetry_section["dependencies"] = dependencies + poetry_section["extras"] = extras + return doc + + +def update_and_write_pyproject(input_file_path, output_file_path=None): + """ + Updates the specified pyproject.toml file with resolved versions for Git-based dependencies + and writes the updated document to a file. + + Args: + input_file_path (str): Path to the original pyproject.toml file. + output_file_path (str, optional): Path to write the updated file. + If not provided, the input file is overwritten. + + Returns: + bool: True if the update and write succeed, False otherwise. + """ + updated_doc = update_pyproject_with_versions(input_file_path) + if updated_doc is None: + print("Failed to update the pyproject.toml document.") + return False + + # Overwrite input file if output file not provided. + output_file_path = output_file_path or input_file_path + + try: + with open(output_file_path, "w") as f: + f.write(dumps(updated_doc)) + print(f"Updated pyproject.toml written to {output_file_path}") + return True + except Exception as e: + print(f"Error writing updated pyproject.toml: {e}") + return False + + +# Example usage when running this module directly. +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Update local pyproject.toml with versions fetched from Git dependencies." + ) + parser.add_argument("--input", required=True, help="Path to the local pyproject.toml to update") + parser.add_argument("--output", help="Optional output file path (if not specified, overwrites input)") + args = parser.parse_args() + + success = update_and_write_pyproject(args.input, args.output) + if not success: + exit(1) diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/test_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/test_ops.py new file mode 100644 index 000000000..e73fb8489 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/test_ops.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +test_ops.py + +Provides functions to analyze test result data from a JSON file. + +The JSON file is expected to have: + - A "summary" section with keys: "total", "passed", "failed", "skipped". + - A "tests" list, where each test contains an "outcome" (e.g., "passed", "failed", "skipped") + and a "keywords" list for tags. + +The module: + - Reads the JSON file. + - Prints a summary table with counts and percentages. + - Groups tests by tags (excluding unwanted tags such as "tests", tags starting with "test_", tags + ending with "_test.py", or empty tags). + - Checks threshold conditions for passed and skipped percentages (if provided) and exits with an error + code if the conditions are not satisfied. +""" + +import json +import sys +from collections import defaultdict +import argparse + + +def parse_arguments(args): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Analyze test results from a JSON file.") + parser.add_argument("file", help="Path to the JSON file containing test results") + parser.add_argument( + "--required-passed", + type=str, + help=( + "Required passed percentage threshold (e.g., 'gt:50', 'lt:30', 'eq:50', " + "'ge:50', 'le:50')" + ), + ) + parser.add_argument( + "--required-skipped", + type=str, + help=( + "Required skipped percentage threshold (e.g., 'gt:20', 'lt:50', 'eq:50', " + "'ge:50', 'le:50')" + ), + ) + return parser.parse_args(args) + + +def evaluate_threshold(value, threshold): + """ + Evaluate if the given value meets the specified threshold condition. + + The threshold format should be: operator:limit (e.g., "gt:50"). + Supported operators: + - gt: greater than + - lt: less than + - eq: equal to + - ge: greater than or equal to + - le: less than or equal to + + Returns: + bool: True if the condition is met, False otherwise. + """ + try: + op, limit = threshold.split(":") + limit = float(limit) + except ValueError as e: + raise ValueError( + f"Invalid threshold format '{threshold}'. Expected format: 'gt:' etc." + ) from e + + if op == "gt": + return value > limit + elif op == "lt": + return value < limit + elif op == "eq": + return value == limit + elif op == "ge": + return value >= limit + elif op == "le": + return value <= limit + else: + raise ValueError( + f"Invalid operator '{op}'. Use one of: 'gt', 'lt', 'eq', 'ge', 'le'." + ) + + +def analyze_test_file(file_path, required_passed=None, required_skipped=None): + """ + Analyzes a JSON file with test results. + + The function: + - Prints a summary table with the total count and percentage for each outcome. + - Checks if the percentage of passed or skipped tests meet the specified thresholds. + - Groups tests by tags (excluding tags that are deemed irrelevant). + - Prints detailed tag-based results. + + If thresholds are not met, the function exits with an error. + + Args: + file_path (str): Path to the JSON file. + required_passed (str, optional): Threshold for passed tests (e.g., "gt:50"). + required_skipped (str, optional): Threshold for skipped tests (e.g., "lt:20"). + """ + try: + with open(file_path, "r") as f: + data = json.load(f) + except FileNotFoundError: + print(f"Error: File not found: {file_path}") + sys.exit(1) + except json.JSONDecodeError: + print(f"Error: Could not decode JSON from {file_path}") + sys.exit(1) + except Exception as e: + print(f"Unexpected error reading {file_path}: {e}") + sys.exit(1) + + summary = data.get("summary", {}) + tests = data.get("tests", []) + if not summary or not tests: + print("No test data or summary found in the provided file.") + sys.exit(1) + + total_tests = summary.get("total", 0) + print("\nTest Results Summary:") + print(f"{'Category':<15}{'Count':<10}{'Total':<10}{'% of Total':<10}") + print("-" * 50) + for category in ["passed", "skipped", "failed"]: + count = summary.get(category, 0) + percentage = (count / total_tests) * 100 if total_tests > 0 else 0 + print(f"{category.capitalize():<15}{count:<10}{total_tests:<10}{percentage:<10.2f}") + + # Calculate percentages for threshold evaluation. + passed_pct = (summary.get("passed", 0) / total_tests) * 100 if total_tests > 0 else 0 + skipped_pct = (summary.get("skipped", 0) / total_tests) * 100 if total_tests > 0 else 0 + + threshold_error = False + if required_passed and not evaluate_threshold(passed_pct, required_passed): + print( + f"\nWARNING: Passed percentage ({passed_pct:.2f}%) does not meet the condition '{required_passed}'!" + ) + threshold_error = True + + if required_skipped and not evaluate_threshold(skipped_pct, required_skipped): + print( + f"WARNING: Skipped percentage ({skipped_pct:.2f}%) does not meet the condition '{required_skipped}'!" + ) + threshold_error = True + + # Group tests by tags. + tag_outcomes = {} + for test in tests: + outcome = test.get("outcome", "").lower() + for tag in test.get("keywords", []): + # Exclude unwanted tags. + if tag == "tests" or tag.startswith("test_") or tag.endswith("_test.py") or tag.strip() == "": + continue + if tag not in tag_outcomes: + tag_outcomes[tag] = {"passed": 0, "skipped": 0, "failed": 0, "total": 0} + tag_outcomes[tag]["total"] += 1 + if outcome == "passed": + tag_outcomes[tag]["passed"] += 1 + elif outcome == "skipped": + tag_outcomes[tag]["skipped"] += 1 + elif outcome == "failed": + tag_outcomes[tag]["failed"] += 1 + + print("\nTag-Based Results:") + header = f"{'Tag':<30}{'Passed':<10}{'Skipped':<10}{'Failed':<10}{'Total':<10}{'% Passed':<10}{'% Skipped':<10}{'% Failed':<10}" + print(header) + print("-" * len(header)) + # Sort tags by percentage passed descending then alphabetically. + sorted_tags = sorted( + tag_outcomes.items(), + key=lambda item: ( + -(item[1]["passed"] / item[1]["total"] * 100 if item[1]["total"] > 0 else 0), + item[0], + ), + ) + for tag, outcomes in sorted_tags: + total = outcomes["total"] + passed_pct = (outcomes["passed"] / total * 100) if total > 0 else 0 + skipped_pct = (outcomes["skipped"] / total * 100) if total > 0 else 0 + failed_pct = (outcomes["failed"] / total * 100) if total > 0 else 0 + print( + f"{tag:<30}{outcomes['passed']:<10}{outcomes['skipped']:<10}{outcomes['failed']:<10}" + f"{total:<10}{passed_pct:<10.2f}{skipped_pct:<10.2f}{failed_pct:<10.2f}" + ) + + # If thresholds are not met, exit with a non-zero status code. + if threshold_error: + sys.exit(1) + else: + print("\nTest analysis completed successfully.") + + +def main(): + args = parse_arguments(sys.argv[1:]) + analyze_test_file( + file_path=args.file, + required_passed=args.required_passed, + required_skipped=args.required_skipped, + ) + + +if __name__ == "__main__": + main() diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/version_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/version_ops.py new file mode 100644 index 000000000..4258ed672 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/version_ops.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +version_ops.py + +Provides functions to: + - Read the current version from pyproject.toml. + - Bump the current version (major, minor, patch) or finalize a dev release. + - Validate that a user-provided new version is not lower than the current one. + - Update the pyproject.toml with the new version. + +Intended for use in a unified monorepo management CLI. +""" + +import sys +from packaging.version import Version, InvalidVersion +from tomlkit import parse, dumps + + +def read_pyproject_version(file_path): + """ + Reads the current version from the provided pyproject.toml file. + + Args: + file_path (str): Path to the pyproject.toml file. + + Returns: + tuple: A tuple containing the current version string and the + tomlkit Document representing the file. + Raises: + KeyError: If the version key is missing. + """ + try: + with open(file_path, "r") as f: + content = f.read() + except Exception as e: + print(f"Error reading {file_path}: {e}", file=sys.stderr) + sys.exit(1) + + doc = parse(content) + try: + version = doc["tool"]["poetry"]["version"] + except KeyError: + raise KeyError("No version found under [tool.poetry] in the given pyproject.toml") + return version, doc + + +def bump_version(current_version, bump_type): + """ + Bumps the current version up using semantic versioning. + Supports: + - Bumping stable versions (major, minor, patch) which also start a dev cycle. + - Bumping within a dev cycle. + - Finalizing a dev version (removing the .dev suffix). + + Args: + current_version (str): The current version (e.g. "1.0.0" or "1.0.1.dev2"). + bump_type (str): One of "major", "minor", "patch", or "finalize". + + Returns: + str: The new version string. + Raises: + ValueError: If the current version is invalid or the bump operation cannot be performed. + """ + try: + ver = Version(current_version) + except InvalidVersion as e: + raise ValueError(f"Invalid current version '{current_version}': {e}") + + # Check if it's a dev release + is_dev = ver.dev is not None + major, minor, patch = ver.release + + if bump_type == "finalize": + if is_dev: + # Remove the dev segment + new_version = f"{major}.{minor}.{patch}" + else: + raise ValueError("Current version is stable; nothing to finalize.") + elif bump_type == "major": + major += 1 + minor = 0 + patch = 0 + new_version = f"{major}.{minor}.{patch}.dev1" + elif bump_type == "minor": + minor += 1 + patch = 0 + new_version = f"{major}.{minor}.{patch}.dev1" + elif bump_type == "patch": + if is_dev: + # Increment the dev counter if already in a dev cycle. + new_dev = ver.dev + 1 + new_version = f"{major}.{minor}.{patch}.dev{new_dev}" + else: + patch += 1 + new_version = f"{major}.{minor}.{patch}.dev1" + else: + raise ValueError("bump_type must be one of: 'major', 'minor', 'patch', or 'finalize'") + + return new_version + + +def validate_and_set_version(current_version, new_version): + """ + Validates that the new version is not lower than the current version. + + Args: + current_version (str): The current version string. + new_version (str): The target version string. + + Returns: + str: The new version if it is valid. + Raises: + ValueError: If new_version is lower than current_version. + """ + try: + cur_ver = Version(current_version) + tgt_ver = Version(new_version) + except InvalidVersion as e: + raise ValueError(f"Invalid version provided: {e}") + + if tgt_ver < cur_ver: + raise ValueError("You cannot bump the version downwards. The target version must be higher than the current version.") + + return new_version + + +def update_pyproject_version(file_path, new_version): + """ + Updates the pyproject.toml file with the new version. + + Args: + file_path (str): The path to the pyproject.toml file. + new_version (str): The new version string. + + Returns: + None + """ + try: + current_version, doc = read_pyproject_version(file_path) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + # Update the version field if it exists + if "tool" in doc and "poetry" in doc["tool"]: + doc["tool"]["poetry"]["version"] = new_version + else: + print(f"Error: Invalid pyproject.toml structure in {file_path}.", file=sys.stderr) + sys.exit(1) + + try: + with open(file_path, "w") as f: + f.write(dumps(doc)) + except Exception as e: + print(f"Error writing updated pyproject.toml: {e}", file=sys.stderr) + sys.exit(1) + + print(f"Bumped version from {current_version} to {new_version} in {file_path}.") + + +def bump_or_set_version(pyproject_file, bump=None, set_ver=None): + """ + Executes either a version bump or a direct version set on the given pyproject.toml file. + + Args: + pyproject_file (str): Path to the pyproject.toml file. + bump (str, optional): The type of bump ("major", "minor", "patch", or "finalize"). + set_ver (str, optional): A specific version string to set. + + Returns: + None + """ + try: + current_version, _ = read_pyproject_version(pyproject_file) + except Exception as e: + print(f"Error reading current version: {e}", file=sys.stderr) + sys.exit(1) + + try: + if bump: + new_version = bump_version(current_version, bump) + elif set_ver: + new_version = validate_and_set_version(current_version, set_ver) + else: + print("No version operation specified.", file=sys.stderr) + sys.exit(1) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + update_pyproject_version(pyproject_file, new_version) + + +# Example usage when running this module directly. +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Bump or set version in pyproject.toml using semantic versioning." + ) + parser.add_argument("file", help="Path to the pyproject.toml file") + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--bump", choices=["major", "minor", "patch", "finalize"], help="Type of version bump to perform") + group.add_argument("--set", dest="set_ver", help="Set the version explicitly (e.g. 1.2.3 or 1.2.3.dev1)") + + args = parser.parse_args() + bump_or_set_version(args.file, bump=args.bump, set_ver=args.set_ver) diff --git a/pkgs/tooling/monorepo_manager/pyproject.toml b/pkgs/tooling/monorepo_manager/pyproject.toml new file mode 100644 index 000000000..f72dc6a28 --- /dev/null +++ b/pkgs/tooling/monorepo_manager/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "monorepo-manager" +version = "0.1.0" +description = "A CLI for managing a Python monorepo" +authors = [ + { name="Your Name", email="you@example.com" } +] +dependencies = [ + "requests", + "tomlkit", + "toml", + "packaging", + # any other libraries you used +] + +[project.scripts] +monorepo-manager = "monorepo_manager.cli:main" From 698dbf125baad8d9772de9c9d37cfc7fd20905ff Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:26:17 -0600 Subject: [PATCH 08/16] tooling - update readme.md --- pkgs/tooling/monorepo_manager/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index 632624a98..dbffce727 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -1,8 +1,3 @@ -Below is an example of a comprehensive `README.md` for your unified monorepo management tool: - ---- - -```markdown # Monorepo Manager **Monorepo Manager** is a unified command-line tool for managing a Python monorepo that contains multiple standalone packages, each with its own `pyproject.toml`. It consolidates several common tasks—including dependency management, version bumping, remote dependency resolution, test analysis, and project configuration updates—into one robust CLI. From 85c05c5a8fa54dff4d10fcdff621f3feb13dc902 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:35:06 -0600 Subject: [PATCH 09/16] tooling - update test and analyze --- pkgs/tooling/monorepo_manager/README.md | 90 ++++++------ .../monorepo_manager/monorepo_manager/cli.py | 24 +++- .../monorepo_manager/poetry_ops.py | 130 +++--------------- 3 files changed, 85 insertions(+), 159 deletions(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index dbffce727..28d632a15 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -1,13 +1,13 @@ # Monorepo Manager -**Monorepo Manager** is a unified command-line tool for managing a Python monorepo that contains multiple standalone packages, each with its own `pyproject.toml`. It consolidates several common tasks—including dependency management, version bumping, remote dependency resolution, test analysis, and project configuration updates—into one robust CLI. +**Monorepo Manager** is a unified command-line tool for managing a Python monorepo that contains multiple standalone packages—each with its own `pyproject.toml`. It consolidates common tasks such as dependency management, version bumping, remote dependency resolution, test execution and analysis, and project configuration updates into one robust CLI. ## Features - **Dependency Management** - **Lock:** Generate a `poetry.lock` file. - **Install:** Install dependencies with options for extras and development dependencies. - - **Show Freeze:** Display installed packages using `pip freeze`. + - **Show Freeze:** (Included via internal commands) Display installed packages using `pip freeze`. - **Version Management** - **Version:** Bump (major, minor, patch, finalize) or explicitly set package versions in `pyproject.toml`. @@ -16,38 +16,39 @@ - **Remote Fetch:** Fetch the version from a remote GitHub repository’s `pyproject.toml`. - **Remote Update:** Update a local `pyproject.toml` file with version information from remote Git dependencies. -- **Test Analysis** - - **Test:** Parse a JSON file with test results, display summary statistics, and evaluate threshold conditions. +- **Testing and Analysis** + - **Test:** Run your tests using pytest. Optionally, run tests in parallel (supports [pytest‑xdist](https://pypi.org/project/pytest-xdist/)). + - **Analyze:** Analyze test results provided in a JSON file, display summary statistics, and evaluate threshold conditions for passed/skipped tests. - **Pyproject Operations** - **Pyproject:** Extract local (path) and Git-based dependencies from a `pyproject.toml` file and optionally update dependency versions. ## Installation - ```bash - pip install monorepo-manager - ``` +Install using pip: -_This installs the `monorepo-manager` CLI (provided via the entry point `monorepo-manager`) into your system PATH._ +```bash +pip install monorepo-manager +``` + +_This command installs the `monorepo-manager` CLI, which is provided via the entry point `monorepo-manager`, into your system PATH._ ## Usage -Once installed, you can invoke the CLI using: +After installation, run the following to see a list of available commands: ```bash monorepo-manager --help ``` -This displays the list of available commands. - ### Command Examples #### 1. Lock Dependencies -Generate a `poetry.lock` file for a package by specifying the directory or file path: +Generate a `poetry.lock` file by specifying either a directory or a file path containing a `pyproject.toml`: ```bash -# Lock using a directory containing pyproject.toml: +# Lock using a directory: monorepo-manager lock --directory ./packages/package1 # Lock using an explicit pyproject.toml file: @@ -56,13 +57,13 @@ monorepo-manager lock --file ./packages/package1/pyproject.toml #### 2. Install Dependencies -Install dependencies with various options: +Install dependencies with options for extras and including development dependencies: ```bash # Basic installation: monorepo-manager install --directory ./packages/package1 -# Install using an explicit pyproject.toml file: +# Using an explicit pyproject.toml file: monorepo-manager install --file ./packages/package1/pyproject.toml # Install including development dependencies: @@ -77,13 +78,13 @@ monorepo-manager install --directory ./packages/package2 --all-extras #### 3. Version Management -Bump the version or set it explicitly for a given package: +Bump or explicitly set the version in a package's `pyproject.toml`: ```bash -# Bump the patch version (e.g. from 1.2.3.dev1 to 1.2.3.dev2): +# Bump the patch version (e.g., from 1.2.3.dev1 to 1.2.3.dev2): monorepo-manager version ./packages/package1/pyproject.toml --bump patch -# Finalize a development version (remove the ".dev" suffix): +# Finalize a dev version (remove the .dev suffix): monorepo-manager version ./packages/package1/pyproject.toml --bump finalize # Set an explicit version: @@ -92,38 +93,52 @@ monorepo-manager version ./packages/package1/pyproject.toml --set 2.0.0.dev1 #### 4. Remote Operations -Fetch version information from a remote GitHub repository’s `pyproject.toml` and update your local configuration accordingly. +Fetch remote version information and update your local dependency configuration: ```bash -# Fetch the remote version: +# Fetch the version from a remote GitHub repository's pyproject.toml: monorepo-manager remote fetch --git-url https://github.com/YourOrg/YourRepo.git --branch main --subdir "src/" -# Update a local pyproject.toml with versions resolved from remote dependencies. +# Update a local pyproject.toml with remote-resolved versions: # (If --output is omitted, the input file is overwritten.) monorepo-manager remote update --input ./packages/package1/pyproject.toml --output ./packages/package1/pyproject.updated.toml ``` -#### 5. Test Analysis +#### 5. Testing and Analysis -Analyze test results provided in a JSON file, and enforce percentage thresholds for passed and skipped tests: +Run your tests using pytest and analyze test results from a JSON report: -```bash -# Analyze test results without thresholds: -monorepo-manager test test-results.json +- **Run Tests:** + Execute tests in a specified directory. Use the `--num-workers` flag to run tests in parallel (requires pytest‑xdist). -# Analyze test results with thresholds: require that passed tests are greater than 75% and skipped tests are less than 20%: -monorepo-manager test test-results.json --required-passed gt:75 --required-skipped lt:20 -``` + ```bash + # Run tests sequentially: + monorepo-manager test --directory ./tests + + # Run tests in parallel using 4 workers: + monorepo-manager test --directory ./tests --num-workers 4 + ``` + +- **Analyze Test Results:** + Analyze a JSON test report and enforce thresholds for passed and skipped tests. + + ```bash + # Analyze test results without thresholds: + monorepo-manager analyze test-results.json + + # Analyze test results with thresholds (e.g., passed tests > 75% and skipped tests < 20%): + monorepo-manager analyze test-results.json --required-passed gt:75 --required-skipped lt:20 + ``` #### 6. Pyproject Operations -Operate on the `pyproject.toml` file to extract dependency information and optionally update dependency versions: +Extract and update dependency information from a `pyproject.toml` file: ```bash # Extract local (path) and Git-based dependencies: monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml -# Update local dependency versions to 2.0.0 (updates parent file and, if possible, each dependency's own pyproject.toml): +# Update local dependency versions to 2.0.0 (updates the parent file and, if possible, each dependency's own pyproject.toml): monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml --update-version 2.0.0 ``` @@ -135,7 +150,7 @@ monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml --upda monorepo_manager/ ├── __init__.py ├── cli.py # Main CLI entry point -├── poetry_ops.py # Poetry operations (lock, install, build, publish, etc.) +├── poetry_ops.py # Poetry operations (lock, install, build, publish, run tests, etc.) ├── version_ops.py # Version bumping and setting operations ├── remote_ops.py # Remote Git dependency version fetching/updating ├── test_ops.py # Test result analysis operations @@ -144,14 +159,6 @@ pyproject.toml # Package configuration file containing metadata README.md # This file ``` -### Running Tests - -For development purposes, you can use your favorite test runner (such as `pytest`) to run tests for the CLI and modules. - -```bash -pytest -``` - ## Contributing Contributions are welcome! Feel free to open issues or submit pull requests for improvements or bug fixes. @@ -161,6 +168,3 @@ Contributions are welcome! Feel free to open issues or submit pull requests for This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. ``` ---- - -This `README.md` provides a detailed overview, installation instructions, and comprehensive usage examples for each command offered by your CLI tool. Feel free to adjust sections or add any additional details specific to your project needs. \ No newline at end of file diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py index c3c4d3196..b185ce8b7 100644 --- a/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py @@ -7,11 +7,12 @@ - Manage Poetry-based operations (lock, install, show pip-freeze, recursive build, publish) - Manage version operations (bump or set versions in pyproject.toml) - Manage remote operations (fetch/update Git dependency versions) + - Run tests using pytest (with optional parallelism) - Analyze test results from a JSON file - Operate on pyproject.toml files (extract and update dependency versions) The commands are intentionally named with simple terms (e.g. "lock" instead of "poetry lock", -"install" instead of "poetry install", and "test" instead of "test-analyze"). +"install" instead of "poetry install"). """ import argparse @@ -76,12 +77,19 @@ def main(): update_parser.add_argument("--output", help="Optional output file path (defaults to overwriting the input)") # ------------------------------------------------ - # Command: test + # Command: test (run pytest) # ------------------------------------------------ - test_parser = subparsers.add_parser("test", help="Analyze test results from a JSON file") - test_parser.add_argument("file", help="Path to the JSON file with test results") - test_parser.add_argument("--required-passed", type=str, help="Threshold for passed tests (e.g. 'gt:75')") - test_parser.add_argument("--required-skipped", type=str, help="Threshold for skipped tests (e.g. 'lt:20')") + test_parser = subparsers.add_parser("test", help="Run tests using pytest") + test_parser.add_argument("--directory", type=str, default=".", help="Directory to run tests in (default: current directory)") + test_parser.add_argument("--num-workers", type=int, default=1, help="Number of workers to use for parallel testing (requires pytest-xdist)") + + # ------------------------------------------------ + # Command: analyze (analyze test results from JSON) + # ------------------------------------------------ + analyze_parser = subparsers.add_parser("analyze", help="Analyze test results from a JSON file") + analyze_parser.add_argument("file", help="Path to the JSON file with test results") + analyze_parser.add_argument("--required-passed", type=str, help="Threshold for passed tests (e.g. 'gt:75')") + analyze_parser.add_argument("--required-skipped", type=str, help="Threshold for skipped tests (e.g. 'lt:20')") # ------------------------------------------------ # Command: pyproject @@ -127,6 +135,10 @@ def main(): sys.exit(1) elif args.command == "test": + # Run pytest (with optional parallelism if --num-workers > 1) + poetry_ops.run_pytests(test_directory=args.directory, num_workers=args.num_workers) + + elif args.command == "analyze": test_ops.analyze_test_file( file_path=args.file, required_passed=args.required_passed, diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py b/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py index 71bbb70fd..71b87ea30 100644 --- a/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/poetry_ops.py @@ -135,116 +135,6 @@ def show_pip_freeze(): run_command("pip freeze") -def set_version_in_pyproject(version, directory=None, file=None): - """ - Set the version for the package in the pyproject.toml file(s). - - :param version: The new version string to set. - :param directory: If provided, recursively update all pyproject.toml files found in the directory. - :param file: If provided, update the specific pyproject.toml file. - """ - def update_file(pyproject_file, version): - print(f"Setting version to {version} in {pyproject_file}...") - try: - with open(pyproject_file, "r") as f: - data = toml.load(f) - except Exception as e: - print(f"Error reading {pyproject_file}: {e}", file=sys.stderr) - return - - if "tool" in data and "poetry" in data["tool"]: - data["tool"]["poetry"]["version"] = version - else: - print(f"Invalid pyproject.toml structure in {pyproject_file}.", file=sys.stderr) - return - - try: - with open(pyproject_file, "w") as f: - toml.dump(data, f) - print(f"Version set to {version} in {pyproject_file}.") - except Exception as e: - print(f"Error writing {pyproject_file}: {e}", file=sys.stderr) - - if directory: - for root, _, files in os.walk(directory): - if "pyproject.toml" in files: - pyproject_file = os.path.join(root, "pyproject.toml") - update_file(pyproject_file, version) - elif file: - update_file(file, version) - else: - print("Error: A directory or a file must be specified.", file=sys.stderr) - sys.exit(1) - - -def set_dependency_versions(version, directory=None, file=None): - """ - Update versions for path dependencies found in pyproject.toml files. - - For each dependency that is defined with a "path", update its version to '^'. - Also attempts to update the dependency's own pyproject.toml. - - :param version: The new version string (without the caret). - :param directory: A directory to search for pyproject.toml files. - :param file: A specific pyproject.toml file to update. - """ - def update_file(pyproject_file, version): - print(f"Setting dependency versions to {version} in {pyproject_file}...") - try: - with open(pyproject_file, "r") as f: - data = toml.load(f) - except Exception as e: - print(f"Error reading {pyproject_file}: {e}", file=sys.stderr) - return - - dependencies = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) - updated_dependencies = {} - - for dep_name, dep_value in dependencies.items(): - if isinstance(dep_value, dict) and "path" in dep_value: - # Update the version entry while preserving other keys except 'path' - updated_dep = {"version": f"^{version}"} - for key, val in dep_value.items(): - if key != "path": - updated_dep[key] = val - updated_dependencies[dep_name] = updated_dep - - # Update the dependency's own pyproject.toml file if present. - dependency_path = os.path.join(os.path.dirname(pyproject_file), dep_value["path"]) - dependency_pyproject = os.path.join(dependency_path, "pyproject.toml") - if os.path.isfile(dependency_pyproject): - print(f"Updating version in dependency: {dependency_pyproject} to {version}") - set_version_in_pyproject(version, file=dependency_pyproject) - else: - print(f"Warning: pyproject.toml not found at {dependency_pyproject}") - else: - updated_dependencies[dep_name] = dep_value - - # Write back the updated dependencies. - if "tool" in data and "poetry" in data["tool"]: - data["tool"]["poetry"]["dependencies"] = updated_dependencies - else: - print(f"Error: Invalid pyproject.toml structure in {pyproject_file}.", file=sys.stderr) - return - - try: - with open(pyproject_file, "w") as f: - toml.dump(data, f) - print(f"Dependency versions set to {version} in {pyproject_file}.") - except Exception as e: - print(f"Error writing {pyproject_file}: {e}", file=sys.stderr) - - if directory: - for root, _, files in os.walk(directory): - if "pyproject.toml" in files: - update_file(os.path.join(root, "pyproject.toml"), version) - elif file: - update_file(file, version) - else: - print("Error: A directory or a file must be specified.", file=sys.stderr) - sys.exit(1) - - def publish_package(directory=None, file=None, username=None, password=None): """ Build and publish packages to PyPI. @@ -306,3 +196,23 @@ def publish_from_dependencies(directory=None, file=None, username=None, password ) else: print(f"Skipping {full_path}: not a valid package directory") + +def run_pytests(test_directory=".", num_workers=1): + """ + Run pytest in the specified directory. + + If num_workers is greater than 1, uses pytest‑xdist to run tests in parallel. + + :param test_directory: Directory in which to run tests (default: current directory). + :param num_workers: Number of workers to use (default: 1). Requires pytest-xdist when > 1. + """ + command = "pytest" + try: + workers = int(num_workers) + except ValueError: + print("Error: num_workers must be an integer", file=sys.stderr) + sys.exit(1) + if workers > 1: + command += f" -n {workers}" + print(f"Running tests in '{test_directory}' with command: {command}") + run_command(command, cwd=test_directory) \ No newline at end of file From c32dfa03883172fa862f4f12aef80ec77ef10795 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:41:04 -0600 Subject: [PATCH 10/16] mono - add monorepo_manager to dev dependencies --- pkgs/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/pyproject.toml b/pkgs/pyproject.toml index 4f8acfe47..c152b12ac 100644 --- a/pkgs/pyproject.toml +++ b/pkgs/pyproject.toml @@ -31,6 +31,8 @@ swm_example_community_package = { path = "./community/swm_example_community_pack pytest = "^7.0" black = "^22.3.0" toml = "^0.10.2" +monorepo_manager = { git = "https://github.com/swarmauri/swarmauri-sdk", branch = "mono/0.6.0.dev1", subdirectory="pkgs/tooling/monorepo_manager" } + [build-system] requires = ["poetry-core>=1.0.0"] From 558e4fcda81f386ac407fac050359aadb54a8dcb Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:44:23 -0600 Subject: [PATCH 11/16] Update pyproject.toml --- pkgs/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/pyproject.toml b/pkgs/pyproject.toml index c152b12ac..3e7a3ca79 100644 --- a/pkgs/pyproject.toml +++ b/pkgs/pyproject.toml @@ -31,7 +31,7 @@ swm_example_community_package = { path = "./community/swm_example_community_pack pytest = "^7.0" black = "^22.3.0" toml = "^0.10.2" -monorepo_manager = { git = "https://github.com/swarmauri/swarmauri-sdk", branch = "mono/0.6.0.dev1", subdirectory="pkgs/tooling/monorepo_manager" } +#monorepo_manager = { git = "https://github.com/swarmauri/swarmauri-sdk", branch = "mono/0.6.0.dev1", subdirectory="pkgs/tooling/monorepo_manager" } [build-system] From fb86a75a3c0e214d5829d19543e4c5bca581272d Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:49:31 -0600 Subject: [PATCH 12/16] tooling - add recursive build mechanism --- pkgs/tooling/monorepo_manager/README.md | 103 +++++++++++++++--- .../monorepo_manager/monorepo_manager/cli.py | 16 ++- 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index 28d632a15..68e2e438b 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -7,25 +7,28 @@ - **Dependency Management** - **Lock:** Generate a `poetry.lock` file. - **Install:** Install dependencies with options for extras and development dependencies. - - **Show Freeze:** (Included via internal commands) Display installed packages using `pip freeze`. + - **Show Freeze:** (Available as an internal command) Display installed packages using `pip freeze`. + +- **Build Operations** + - **Build:** Recursively build packages based on local (path) dependencies as specified in your `pyproject.toml` files. - **Version Management** - **Version:** Bump (major, minor, patch, finalize) or explicitly set package versions in `pyproject.toml`. - **Remote Operations** - **Remote Fetch:** Fetch the version from a remote GitHub repository’s `pyproject.toml`. - - **Remote Update:** Update a local `pyproject.toml` file with version information from remote Git dependencies. + - **Remote Update:** Update a local `pyproject.toml` file with version information fetched from remote Git dependencies. - **Testing and Analysis** - **Test:** Run your tests using pytest. Optionally, run tests in parallel (supports [pytest‑xdist](https://pypi.org/project/pytest-xdist/)). - - **Analyze:** Analyze test results provided in a JSON file, display summary statistics, and evaluate threshold conditions for passed/skipped tests. + - **Analyze:** Analyze test results provided in a JSON file by displaying summary statistics and evaluating threshold conditions for passed/skipped tests. - **Pyproject Operations** - - **Pyproject:** Extract local (path) and Git-based dependencies from a `pyproject.toml` file and optionally update dependency versions. + - **Pyproject:** Extract both local (path) and Git-based dependencies from a `pyproject.toml` file and, optionally, update dependency versions. ## Installation -Install using pip: +Install via pip: ```bash pip install monorepo-manager @@ -35,7 +38,7 @@ _This command installs the `monorepo-manager` CLI, which is provided via the ent ## Usage -After installation, run the following to see a list of available commands: +After installation, run the following command to see a list of available commands: ```bash monorepo-manager --help @@ -76,7 +79,19 @@ monorepo-manager install --directory ./packages/package2 --extras full monorepo-manager install --directory ./packages/package2 --all-extras ``` -#### 3. Version Management +#### 3. Build Packages + +Recursively build packages based on their local dependency paths defined in their `pyproject.toml` files: + +```bash +# Build packages using a directory containing a master pyproject.toml: +monorepo-manager build --directory . + +# Build packages using an explicit pyproject.toml file: +monorepo-manager build --file ./packages/package1/pyproject.toml +``` + +#### 4. Version Management Bump or explicitly set the version in a package's `pyproject.toml`: @@ -84,14 +99,14 @@ Bump or explicitly set the version in a package's `pyproject.toml`: # Bump the patch version (e.g., from 1.2.3.dev1 to 1.2.3.dev2): monorepo-manager version ./packages/package1/pyproject.toml --bump patch -# Finalize a dev version (remove the .dev suffix): +# Finalize a development version (remove the .dev suffix): monorepo-manager version ./packages/package1/pyproject.toml --bump finalize # Set an explicit version: monorepo-manager version ./packages/package1/pyproject.toml --set 2.0.0.dev1 ``` -#### 4. Remote Operations +#### 5. Remote Operations Fetch remote version information and update your local dependency configuration: @@ -104,12 +119,12 @@ monorepo-manager remote fetch --git-url https://github.com/YourOrg/YourRepo.git monorepo-manager remote update --input ./packages/package1/pyproject.toml --output ./packages/package1/pyproject.updated.toml ``` -#### 5. Testing and Analysis +#### 6. Testing and Analysis Run your tests using pytest and analyze test results from a JSON report: - **Run Tests:** - Execute tests in a specified directory. Use the `--num-workers` flag to run tests in parallel (requires pytest‑xdist). + Execute tests within a specified directory. Use the `--num-workers` flag for parallel execution (requires pytest‑xdist): ```bash # Run tests sequentially: @@ -120,7 +135,7 @@ Run your tests using pytest and analyze test results from a JSON report: ``` - **Analyze Test Results:** - Analyze a JSON test report and enforce thresholds for passed and skipped tests. + Analyze a JSON test report and enforce thresholds for passed and skipped tests: ```bash # Analyze test results without thresholds: @@ -130,7 +145,7 @@ Run your tests using pytest and analyze test results from a JSON report: monorepo-manager analyze test-results.json --required-passed gt:75 --required-skipped lt:20 ``` -#### 6. Pyproject Operations +#### 7. Pyproject Operations Extract and update dependency information from a `pyproject.toml` file: @@ -142,6 +157,56 @@ monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml monorepo-manager pyproject --pyproject ./packages/package1/pyproject.toml --update-version 2.0.0 ``` +## Workflow Example in GitHub Actions + +Here's an example GitHub Actions workflow that uses **monorepo_manager** to lock, build, install, test, bump the patch version, and publish: + +```yaml +name: Release Workflow + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install monorepo_manager Tools + run: pip install "monorepo_manager@git+https://github.com/YourOrg/monorepo_manager.git@main" + + - name: Lock Dependencies + run: monorepo-manager lock --directory . + + - name: Build Packages + run: monorepo-manager build --directory . + + - name: Install Dependencies + run: monorepo-manager install --directory . + + - name: Run Tests + run: monorepo-manager test --directory ./tests --num-workers 4 + + - name: Bump Patch Version + run: monorepo-manager version ./packages/package1/pyproject.toml --bump patch + + - name: Publish Packages + env: + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: monorepo-manager publish --directory . --username "$PYPI_USERNAME" --password "$PYPI_PASSWORD" +``` + ## Development ### Project Structure @@ -159,12 +224,18 @@ pyproject.toml # Package configuration file containing metadata README.md # This file ``` +### Running Tests + +For development purposes, you can run your tests using your preferred test runner (e.g., `pytest`): + +```bash +pytest +``` + ## Contributing -Contributions are welcome! Feel free to open issues or submit pull requests for improvements or bug fixes. +Contributions are welcome! Please open issues or submit pull requests for improvements or bug fixes. ## License This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. -``` - diff --git a/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py index b185ce8b7..3c8f702a6 100644 --- a/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py +++ b/pkgs/tooling/monorepo_manager/monorepo_manager/cli.py @@ -4,7 +4,7 @@ This is the main entry point for the monorepo management CLI. It provides commands to: - - Manage Poetry-based operations (lock, install, show pip-freeze, recursive build, publish) + - Manage Poetry-based operations (lock, install, build, show pip-freeze, publish) - Manage version operations (bump or set versions in pyproject.toml) - Manage remote operations (fetch/update Git dependency versions) - Run tests using pytest (with optional parallelism) @@ -12,7 +12,7 @@ - Operate on pyproject.toml files (extract and update dependency versions) The commands are intentionally named with simple terms (e.g. "lock" instead of "poetry lock", -"install" instead of "poetry install"). +"install" instead of "poetry install", and "test" instead of "test-analyze"). """ import argparse @@ -49,6 +49,13 @@ def main(): install_parser.add_argument("--dev", action="store_true", help="Include dev dependencies") install_parser.add_argument("--all-extras", action="store_true", help="Include all extras") + # ------------------------------------------------ + # Command: build + # ------------------------------------------------ + build_parser = subparsers.add_parser("build", help="Build packages recursively based on path dependencies") + build_parser.add_argument("--directory", type=str, help="Directory containing pyproject.toml") + build_parser.add_argument("--file", type=str, help="Explicit path to a pyproject.toml file") + # ------------------------------------------------ # Command: version # ------------------------------------------------ @@ -81,7 +88,7 @@ def main(): # ------------------------------------------------ test_parser = subparsers.add_parser("test", help="Run tests using pytest") test_parser.add_argument("--directory", type=str, default=".", help="Directory to run tests in (default: current directory)") - test_parser.add_argument("--num-workers", type=int, default=1, help="Number of workers to use for parallel testing (requires pytest-xdist)") + test_parser.add_argument("--num-workers", type=int, default=1, help="Number of workers for parallel testing (requires pytest-xdist)") # ------------------------------------------------ # Command: analyze (analyze test results from JSON) @@ -115,6 +122,9 @@ def main(): all_extras=args.all_extras ) + elif args.command == "build": + poetry_ops.recursive_build(directory=args.directory, file=args.file) + elif args.command == "version": version_ops.bump_or_set_version(args.pyproject_file, bump=args.bump, set_ver=args.set_ver) From e49e984048eb227fd6e879c955506fc033a6de54 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:50:41 -0600 Subject: [PATCH 13/16] Update README.md --- pkgs/tooling/monorepo_manager/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index 68e2e438b..a882004ff 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -183,7 +183,7 @@ jobs: python-version: '3.9' - name: Install monorepo_manager Tools - run: pip install "monorepo_manager@git+https://github.com/YourOrg/monorepo_manager.git@main" + run: pip install "monorepo_manager@git+https://github.com/swarmauri/monorepo_manager.git@master" - name: Lock Dependencies run: monorepo-manager lock --directory . @@ -224,14 +224,6 @@ pyproject.toml # Package configuration file containing metadata README.md # This file ``` -### Running Tests - -For development purposes, you can run your tests using your preferred test runner (e.g., `pytest`): - -```bash -pytest -``` - ## Contributing Contributions are welcome! Please open issues or submit pull requests for improvements or bug fixes. From 65940e36dfc6e31123410548dd0fc5f12c5a8e78 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:53:07 -0600 Subject: [PATCH 14/16] tooling - standardize readme.md --- pkgs/tooling/monorepo_manager/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index a882004ff..e73b1eff4 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -1,3 +1,23 @@ +![Swamauri Logo](https://res.cloudinary.com/dbjmpekvl/image/upload/v1730099724/Swarmauri-logo-lockup-2048x757_hww01w.png) + +
+ +![PyPI - Downloads](https://img.shields.io/pypi/dm/swarmauri) +![](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https://github.com/swarmauri/swarmauri-sdk&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false) +![GitHub repo size](https://img.shields.io/github/repo-size/swarmauri/swarmauri-sdk) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/swarmauri) ![PyPI - License](https://img.shields.io/pypi/l/swarmauri) + + + + + +![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_core&color=green) +![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri&color=green) +![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_community&color=yellow) +![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_experimental&color=yellow) + + + # Monorepo Manager **Monorepo Manager** is a unified command-line tool for managing a Python monorepo that contains multiple standalone packages—each with its own `pyproject.toml`. It consolidates common tasks such as dependency management, version bumping, remote dependency resolution, test execution and analysis, and project configuration updates into one robust CLI. From 60c0830cb0259f95624b228420cda9472e13dcaa Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:57:46 -0600 Subject: [PATCH 15/16] Update README.md --- pkgs/tooling/monorepo_manager/README.md | 28 +++++++++---------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index e73b1eff4..b7f7fb47a 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -1,21 +1,13 @@ -![Swamauri Logo](https://res.cloudinary.com/dbjmpekvl/image/upload/v1730099724/Swarmauri-logo-lockup-2048x757_hww01w.png) - -
- -![PyPI - Downloads](https://img.shields.io/pypi/dm/swarmauri) -![](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https://github.com/swarmauri/swarmauri-sdk&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false) -![GitHub repo size](https://img.shields.io/github/repo-size/swarmauri/swarmauri-sdk) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/swarmauri) ![PyPI - License](https://img.shields.io/pypi/l/swarmauri) - - - - - -![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_core&color=green) -![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri&color=green) -![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_community&color=yellow) -![PyPI - Version](https://img.shields.io/pypi/v/swarmauri?label=swarmauri_experimental&color=yellow) - +

+ Swamauri Logo +
+ Hits + License + PyPI - monorepo_manager Version + PyPI - monorepo_manager Downloads +
+ Python +

# Monorepo Manager From e5240e2a88b557f68bfb6d222a02fa8343c8df75 Mon Sep 17 00:00:00 2001 From: cobycloud <25079070+cobycloud@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:58:15 -0600 Subject: [PATCH 16/16] Update README.md --- pkgs/tooling/monorepo_manager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tooling/monorepo_manager/README.md b/pkgs/tooling/monorepo_manager/README.md index b7f7fb47a..66db27fad 100644 --- a/pkgs/tooling/monorepo_manager/README.md +++ b/pkgs/tooling/monorepo_manager/README.md @@ -1,7 +1,7 @@

Swamauri Logo
- Hits + Hits License PyPI - monorepo_manager Version PyPI - monorepo_manager Downloads