From fbb63d5d4eeef25f74501c52c716318f69e52a7f Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Thu, 16 Jan 2025 18:19:58 -0600 Subject: [PATCH 1/8] NEW: Migrate NVIDIA redist feedstocks to use cf-nvidia-tools --- conda_forge_tick/make_migrators.py | 7 +- conda_forge_tick/migrators/__init__.py | 1 + conda_forge_tick/migrators/nvtools.py | 245 +++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 conda_forge_tick/migrators/nvtools.py diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index cb96b85a6..d9c9f611d 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -75,6 +75,7 @@ XzLibLzmaDevelMigrator, make_from_lazy_json_data, skip_migrator_due_to_schema, + AddNVIDIATools, ) from conda_forge_tick.migrators.arch import OSXArm from conda_forge_tick.migrators.migration_yaml import ( @@ -794,7 +795,11 @@ def initialize_migrators( gx: nx.DiGraph, dry_run: bool = False, ) -> MutableSequence[Migrator]: - migrators: List[Migrator] = [] + migrators: List[Migrator] = [ + AddNVIDIATools( + check_solvable=False, + ), + ] add_arch_migrate(migrators, gx) diff --git a/conda_forge_tick/migrators/__init__.py b/conda_forge_tick/migrators/__init__.py index 13dd0adf4..03ba195c6 100644 --- a/conda_forge_tick/migrators/__init__.py +++ b/conda_forge_tick/migrators/__init__.py @@ -43,3 +43,4 @@ from .version import Version from .xz_to_liblzma_devel import XzLibLzmaDevelMigrator from .noarch_python_min import NoarchPythonMinMigrator +from .nvtools import AddNVIDIATools diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py new file mode 100644 index 000000000..a3030b52c --- /dev/null +++ b/conda_forge_tick/migrators/nvtools.py @@ -0,0 +1,245 @@ +import os.path +import logging +import copy +from typing import Any + +from conda_forge_tick.contexts import ClonedFeedstockContext +import conda_forge_tick.migrators + +from conda_forge_tick.migrators_types import AttrsTypedDict, MigrationUidTypedDict +from conda_forge_tick.utils import ( + yaml_safe_dump, + yaml_safe_load, + get_bot_run_url, +) + + +def _file_contains(filename, string): + with open(filename, "r") as f: + return string in f.read() + + +def _insert_requirements_build(meta): + insert_before: bool = True + before: list[str] = [] + after: list[str] = [] + requirements_found = False + build_found = False + with open(meta, "r") as f: + for line in f: + if build_found: + insert_before = False + elif line.startswith("requirements"): + requirements_found = True + elif requirements_found and line.lstrip().startswith("build"): + build_found = True + + if insert_before: + before.append(line) + else: + after.append(line) + + if not (build_found or requirements_found): + return + + with open(meta, "w") as f: + f.writelines(before + [" - cf-nvidia-tools # [linux]\n"] + after) + + +def _insert_build_script(meta): + insert_before: bool = True + before: list[str] = [] + after: list[str] = [] + script_found = False + build_found = False + with open(meta, "r") as f: + for line in f: + if not insert_before: + pass + elif line.startswith("build"): + build_found = True + elif build_found and line.lstrip().startswith("script"): + script_found = True + elif build_found and script_found: + if line.lstrip().startswith("-"): + # inside script + pass + else: + # empty script section? + insert_before = False + if insert_before: + before.append(line) + else: + after.append(line) + + if not (script_found or build_found): + return + + with open(meta, "w") as f: + f.writelines( + before + + [' - check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* # [linux]\n'] + + after + ) + + +class AddNVIDIATools(conda_forge_tick.migrators.Migrator): + """Add the cf-nvidia-tools package to NVIDIA redist feedstocks. + + In order to ensure that NVIDIA's redistributed binaries (redists) are being packaged + correctly, NVIDIA has created a package containing a collection of tools to perform + common actions for NVIDIA recipes. + + At this time, the package may be used to check Linux binaries for their minimum glibc + requirement in order to ensure that the correct metadata is being used in the conda + package. + + This migrator will attempt to add this glibc check to all feedstocks which download any + artifacts from https://developer.download.nvidia.com. The check involves adding + something like: + + ```bash + check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* + ``` + + to the build script after the package artifacts have been installed. + + > [!NOTE] + > A human needs to verify that the glob expression is checking all of the correct + > artifacts! + + More information about cf-nvidia-tools is available in the feedstock's + [README](https://github.com/conda-forge/cf-nvidia-tools-feedstock/tree/main/recipe). + + Please ping carterbox for questions. + """ + + name = "NVIDIA Tools Migrator" + + rerender = True + + max_solver_attempts = 3 + + migrator_version = 0 + + allow_empty_commits = False + + allowed_schema_versions = [0] + + def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool: + """If true don't act upon node + + Parameters + ---------- + attrs : dict + The node attributes + not_bad_str_start : str, optional + If the 'bad' notice starts with the string then it is not + to be excluded. For example, rebuild migrations don't need + to worry about if the upstream can be fetched. Defaults to ``''`` + + Returns + ------- + bool : + True if node is to be skipped + """ + return ( + attrs["archived"] + or "https://developer.download.nvidia.com" not in attrs["source"]["url"] + ) + + def migrate( + self, recipe_dir: str, attrs: AttrsTypedDict, **kwargs: Any + ) -> MigrationUidTypedDict: + """Perform the migration, updating the ``meta.yaml`` + + Parameters + ---------- + recipe_dir : str + The directory of the recipe + attrs : dict + The node attributes + + Returns + ------- + namedtuple or bool: + If namedtuple continue with PR, if False scrap local folder + """ + # STEP 0: Bump the build number + self.set_build_number(os.path.join(recipe_dir, "meta.yaml")) + + # STEP 1: Add cf-nvidia-tools to build requirements + meta = os.path.join(recipe_dir, "meta.yaml") + if _file_contains(meta, "cf-nvidia-tools"): + logging.debug("cf-nvidia-tools already in meta.yaml; not adding again.") + else: + _insert_requirements_build(meta) + logging.debug("cf-nvidia-tools added to meta.yaml.") + + # STEP 2: Add check-glibc to the build script + build = os.path.join(recipe_dir, "build.sh") + if os.path.isfile(build): + if _file_contains(build, "check-glibc"): + logging.debug( + "build.sh already contains check-glibc; not adding again." + ) + else: + with open(build, "a") as file: + file.write('\ncheck-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/*\n') + logging.debug("Added check-glibc to build.sh") + else: + if _file_contains(meta, "check-glibc"): + logging.debug( + "meta.yaml already contains check-glibc; not adding again." + ) + else: + _insert_build_script(meta) + logging.debug("Added check-glibc to meta.yaml") + + # STEP 3: Remove os_version keys from conda-forge.yml + config = os.path.join(recipe_dir, "..", "conda-forge.yml") + with open(config) as f: + y = yaml_safe_load(f) + y_orig = copy.deepcopy(y) + y.pop("os_version", None) + if y_orig != y: + with open(config, "w") as f: + yaml_safe_dump(y, f) + + return self.migrator_uid(attrs) + + def pr_body( + self, feedstock_ctx: ClonedFeedstockContext, add_label_text=True + ) -> str: + """Create a PR message body + + Returns + ------- + body: str + The body of the PR message + :param feedstock_ctx: + """ + body = f"{AddNVIDIATools.__doc__}\n\n" + + if add_label_text: + body += ( + "If this PR was opened in error or needs to be updated please add " + "the `bot-rerun` label to this PR. The bot will close this PR and " + "schedule another one. If you do not have permissions to add this " + "label, you can use the phrase " + "@conda-forge-admin, please rerun bot " + "in a PR comment to have the `conda-forge-admin` add it for you.\n\n" + ) + + body += ( + "" + "This PR was created by the [regro-cf-autotick-bot](https://github.com/regro/cf-scripts). " + "The **regro-cf-autotick-bot** is a service to automatically " + "track the dependency graph, migrate packages, and " + "propose package version updates for conda-forge. " + "Feel free to drop us a line if there are any " + "[issues](https://github.com/regro/cf-scripts/issues)! " + + f"This PR was generated by {get_bot_run_url()} - please use this URL for debugging." + + "" + ) + return body From cce6d6206d842a623f53472a6ca0146d3e2079cd Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Fri, 17 Jan 2025 10:47:12 -0600 Subject: [PATCH 2/8] STY: Run pre-commit on files --- conda_forge_tick/make_migrators.py | 2 +- conda_forge_tick/migrators/nvtools.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index d9c9f611d..253c3ae48 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -39,6 +39,7 @@ remove_key_for_hashmap, ) from conda_forge_tick.migrators import ( + AddNVIDIATools, ArchRebuild, Build2HostMigrator, CondaForgeYAMLCleanup, @@ -75,7 +76,6 @@ XzLibLzmaDevelMigrator, make_from_lazy_json_data, skip_migrator_due_to_schema, - AddNVIDIATools, ) from conda_forge_tick.migrators.arch import OSXArm from conda_forge_tick.migrators.migration_yaml import ( diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index a3030b52c..817e3a470 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -1,21 +1,20 @@ -import os.path -import logging import copy +import logging +import os.path from typing import Any -from conda_forge_tick.contexts import ClonedFeedstockContext import conda_forge_tick.migrators - +from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.migrators_types import AttrsTypedDict, MigrationUidTypedDict from conda_forge_tick.utils import ( + get_bot_run_url, yaml_safe_dump, yaml_safe_load, - get_bot_run_url, ) def _file_contains(filename, string): - with open(filename, "r") as f: + with open(filename) as f: return string in f.read() @@ -25,7 +24,7 @@ def _insert_requirements_build(meta): after: list[str] = [] requirements_found = False build_found = False - with open(meta, "r") as f: + with open(meta) as f: for line in f: if build_found: insert_before = False @@ -52,7 +51,7 @@ def _insert_build_script(meta): after: list[str] = [] script_found = False build_found = False - with open(meta, "r") as f: + with open(meta) as f: for line in f: if not insert_before: pass From 0cab166ef7c1c1e1732061d95afc3956acea8540 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Tue, 21 Jan 2025 14:04:08 -0600 Subject: [PATCH 3/8] REF: Refactor nvtools recipe item insertion --- conda_forge_tick/migrators/nvtools.py | 113 ++++++++++++-------------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 817e3a470..25c5c5da4 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -13,73 +13,54 @@ ) -def _file_contains(filename, string): +def _file_contains(filename: str, string: str) -> bool: + """Return whether the given file contains the given string.""" with open(filename) as f: return string in f.read() -def _insert_requirements_build(meta): - insert_before: bool = True - before: list[str] = [] - after: list[str] = [] - requirements_found = False - build_found = False - with open(meta) as f: - for line in f: - if build_found: - insert_before = False - elif line.startswith("requirements"): - requirements_found = True - elif requirements_found and line.lstrip().startswith("build"): - build_found = True - - if insert_before: - before.append(line) - else: - after.append(line) - - if not (build_found or requirements_found): - return - - with open(meta, "w") as f: - f.writelines(before + [" - cf-nvidia-tools # [linux]\n"] + after) - - -def _insert_build_script(meta): - insert_before: bool = True - before: list[str] = [] - after: list[str] = [] - script_found = False - build_found = False - with open(meta) as f: +def _insert_subsection( + filename: str, + section: str, + subsection: str, + new_item: str, +) -> None: + """Append a new item onto the end of the section.subsection of a recipe.""" + # Strategy: Read the file as a list of strings. Split the file in half at the end of the + # section.subsection section. Append the new_item to the first half. Combine the two + # file halves. Write the file back to disk. + first_half: list[str] = [] + second_half: list[str] = [] + break_located: bool = False + section_found: bool = False + subsection_found: bool = False + with open(filename) as f: for line in f: - if not insert_before: - pass - elif line.startswith("build"): - build_found = True - elif build_found and line.lstrip().startswith("script"): - script_found = True - elif build_found and script_found: - if line.lstrip().startswith("-"): - # inside script - pass - else: - # empty script section? - insert_before = False - if insert_before: - before.append(line) + if break_located: + second_half.append(line) else: - after.append(line) - - if not (script_found or build_found): + if line.startswith(section): + section_found = True + elif section_found and line.lstrip().startswith(subsection): + subsection_found = True + elif section_found and subsection_found: + if line.lstrip().startswith("-"): + # Inside section.subsection elements start with "-". We assume there + # is at least one item under section.subsection already. + first_half.append(line) + continue + else: + break_located = True + second_half.append(line) + continue + first_half.append(line) + + if not break_located: + # Don't overwrite file if we didn't find section.subsection return - with open(meta, "w") as f: - f.writelines( - before - + [' - check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* # [linux]\n'] - + after - ) + with open(filename, "w") as f: + f.writelines(first_half + [new_item] + second_half) class AddNVIDIATools(conda_forge_tick.migrators.Migrator): @@ -172,7 +153,12 @@ def migrate( if _file_contains(meta, "cf-nvidia-tools"): logging.debug("cf-nvidia-tools already in meta.yaml; not adding again.") else: - _insert_requirements_build(meta) + _insert_subsection( + meta, + "requirements", + "build", + " - cf-nvidia-tools # [linux]\n", + ) logging.debug("cf-nvidia-tools added to meta.yaml.") # STEP 2: Add check-glibc to the build script @@ -192,7 +178,12 @@ def migrate( "meta.yaml already contains check-glibc; not adding again." ) else: - _insert_build_script(meta) + _insert_subsection( + meta, + "build", + "script", + ' - check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* # [linux]\n', + ) logging.debug("Added check-glibc to meta.yaml") # STEP 3: Remove os_version keys from conda-forge.yml From 754307421d6cbcb61c17fa33fc1514ab707011d2 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Tue, 21 Jan 2025 14:17:02 -0600 Subject: [PATCH 4/8] DOC: Add note about migrator incompatability for AddNVIDIATools --- conda_forge_tick/migrators/nvtools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 25c5c5da4..3d8bf6e33 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -76,7 +76,7 @@ class AddNVIDIATools(conda_forge_tick.migrators.Migrator): This migrator will attempt to add this glibc check to all feedstocks which download any artifacts from https://developer.download.nvidia.com. The check involves adding - something like: + "cf-nvidia-tools" to the top-level build requirements and something like: ```bash check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* @@ -88,6 +88,11 @@ class AddNVIDIATools(conda_forge_tick.migrators.Migrator): > A human needs to verify that the glob expression is checking all of the correct > artifacts! + > [!NOTE] + > If the recipe does not have a top-level requirements.build section, it should be + > refactored so that the top-level package does not share a name with one of the + > outputs. i.e. The top-level package name should be something like "libcufoo-split". + More information about cf-nvidia-tools is available in the feedstock's [README](https://github.com/conda-forge/cf-nvidia-tools-feedstock/tree/main/recipe). From 6f4ffc0ca7f2ecd65d6893b3002008e5b4539cdc Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Fri, 31 Jan 2025 16:26:40 -0600 Subject: [PATCH 5/8] BUG: Fix circular import --- conda_forge_tick/migrators/nvtools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 3d8bf6e33..6cc82e447 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -3,7 +3,6 @@ import os.path from typing import Any -import conda_forge_tick.migrators from conda_forge_tick.contexts import ClonedFeedstockContext from conda_forge_tick.migrators_types import AttrsTypedDict, MigrationUidTypedDict from conda_forge_tick.utils import ( @@ -12,6 +11,8 @@ yaml_safe_load, ) +from .core import Migrator + def _file_contains(filename: str, string: str) -> bool: """Return whether the given file contains the given string.""" @@ -63,7 +64,7 @@ def _insert_subsection( f.writelines(first_half + [new_item] + second_half) -class AddNVIDIATools(conda_forge_tick.migrators.Migrator): +class AddNVIDIATools(Migrator): """Add the cf-nvidia-tools package to NVIDIA redist feedstocks. In order to ensure that NVIDIA's redistributed binaries (redists) are being packaged From ced739ddc2327a759e52e68ff13598c68ff59b0d Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Fri, 31 Jan 2025 16:28:06 -0600 Subject: [PATCH 6/8] REF: Pin cf-nvidia-tools to version 1 in migrator --- conda_forge_tick/migrators/nvtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 6cc82e447..39d0c51a3 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -163,7 +163,7 @@ def migrate( meta, "requirements", "build", - " - cf-nvidia-tools # [linux]\n", + " - cf-nvidia-tools 1 # [linux]\n", ) logging.debug("cf-nvidia-tools added to meta.yaml.") From 2e023aecf708a1773fc79b7917e0dcd321d29254 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Fri, 31 Jan 2025 16:37:53 -0600 Subject: [PATCH 7/8] REF: Add targets prefix to checked paths --- conda_forge_tick/migrators/nvtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 39d0c51a3..6310038d9 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -176,7 +176,7 @@ def migrate( ) else: with open(build, "a") as file: - file.write('\ncheck-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/*\n') + file.write('\ncheck-glibc "$PREFIX"/lib*/*.so.* "$PREFIX"/bin/* "$PREFIX"/targets/*/lib*/*.so.* "$PREFIX"/targets/*/bin/*\n') logging.debug("Added check-glibc to build.sh") else: if _file_contains(meta, "check-glibc"): @@ -188,7 +188,7 @@ def migrate( meta, "build", "script", - ' - check-glibc "$PREFIX"/lib/*.so.* "$PREFIX"/bin/* # [linux]\n', + ' - check-glibc "$PREFIX"/lib*/*.so.* "$PREFIX"/bin/* "$PREFIX"/targets/*/lib*/*.so.* "$PREFIX"/targets/*/bin/* # [linux]\n', ) logging.debug("Added check-glibc to meta.yaml") From 3fbdb6a8818dbdb666a8f31a23974766ec6dc994 Mon Sep 17 00:00:00 2001 From: Daniel Ching Date: Fri, 31 Jan 2025 16:42:19 -0600 Subject: [PATCH 8/8] STY: Run pre-commit checks --- conda_forge_tick/migrators/nvtools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/migrators/nvtools.py b/conda_forge_tick/migrators/nvtools.py index 6310038d9..ad13ad7f4 100644 --- a/conda_forge_tick/migrators/nvtools.py +++ b/conda_forge_tick/migrators/nvtools.py @@ -176,7 +176,9 @@ def migrate( ) else: with open(build, "a") as file: - file.write('\ncheck-glibc "$PREFIX"/lib*/*.so.* "$PREFIX"/bin/* "$PREFIX"/targets/*/lib*/*.so.* "$PREFIX"/targets/*/bin/*\n') + file.write( + '\ncheck-glibc "$PREFIX"/lib*/*.so.* "$PREFIX"/bin/* "$PREFIX"/targets/*/lib*/*.so.* "$PREFIX"/targets/*/bin/*\n' + ) logging.debug("Added check-glibc to build.sh") else: if _file_contains(meta, "check-glibc"):