diff --git a/.bazelrc b/.bazelrc index ada5c5a0a7..f20f38f6ee 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_local_package,examples/pip_parse_local_package/hello,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_local_package,examples/pip_parse_local_package/hello,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/examples/pip_parse_local_package/BUILD.bazel b/examples/pip_parse_local_package/BUILD.bazel new file mode 100644 index 0000000000..fbb15c884c --- /dev/null +++ b/examples/pip_parse_local_package/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_python//python:defs.bzl", "py_test") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_test( + name = "test_hello", + srcs = ["test_hello.py"], + deps = [ + "@pypi//hello", + ], +) + +py_console_script_binary( + name = "hello_parse", + pkg = "@pypi//hello", + script = "parse", +) + +sh_test( + name = "test_hello_script", + srcs = ["test_hello_script.sh"], + data = [":hello_parse"], + env = { + "HELLO_PARSE": "$(rootpaths :hello_parse)", + }, +) diff --git a/examples/pip_parse_local_package/MODULE.bazel b/examples/pip_parse_local_package/MODULE.bazel new file mode 100644 index 0000000000..987385adf0 --- /dev/null +++ b/examples/pip_parse_local_package/MODULE.bazel @@ -0,0 +1,25 @@ +module(name = "rules_python_pip_parse_local_package_example") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + # We can specify the exact version. + python_version = "3.9.13", +) + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pypi", + # If we want to expand the PROJECT_ROOT variable in the requirement lock file, + # we need to pass a label to the file defining the project root. + project_root = "//:pyproject.toml", + # We need to use the same version here as in the `python.toolchain` call. + python_version = "3.9.13", + requirements_lock = "//:requirements_lock.txt", +) +use_repo(pip, "pypi", "pypi_39_hello") diff --git a/examples/pip_parse_local_package/pyproject.toml b/examples/pip_parse_local_package/pyproject.toml new file mode 100644 index 0000000000..dee2e36534 --- /dev/null +++ b/examples/pip_parse_local_package/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "test" +version = "0.0.0" +dependencies = [ + # Test + "hello @ file://${PROJECT_ROOT}/hello", +] diff --git a/examples/pip_parse_local_package/requirements_lock.txt b/examples/pip_parse_local_package/requirements_lock.txt new file mode 100644 index 0000000000..b4fcabc39d --- /dev/null +++ b/examples/pip_parse_local_package/requirements_lock.txt @@ -0,0 +1,6 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements_lock.txt +hello @ file://${PROJECT_ROOT}/hello + # via test (pyproject.toml) +hjson==3.1.0 + # via hello diff --git a/examples/pip_parse_local_package/test_hello.py b/examples/pip_parse_local_package/test_hello.py new file mode 100644 index 0000000000..5b1df56df7 --- /dev/null +++ b/examples/pip_parse_local_package/test_hello.py @@ -0,0 +1,13 @@ +import hello + +EXAMPLE_HJSON = """ +{ + # TL;DR + human: Hjson + machine: JSON +} +""" + +res = hello.parse(EXAMPLE_HJSON) +assert res["human"] == "Hjson" +assert res["machine"] == "JSON" diff --git a/examples/pip_parse_local_package/test_hello_script.sh b/examples/pip_parse_local_package/test_hello_script.sh new file mode 100755 index 0000000000..66f5ae19c6 --- /dev/null +++ b/examples/pip_parse_local_package/test_hello_script.sh @@ -0,0 +1,9 @@ +TEST_HJSON=$(mktemp) + +echo "{ + # TL;DR + human: Hjson + machine: JSON +}" > $TEST_HJSON + +$HELLO_PARSE $TEST_HJSON diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9b150bdce0..811912264d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -241,6 +241,7 @@ def _create_whl_repos( group_deps = group_deps, group_name = group_name, pip_data_exclude = pip_attr.pip_data_exclude, + project_root = pip_attr.project_root, python_interpreter = pip_attr.python_interpreter, python_interpreter_target = python_interpreter_target, whl_patches = { @@ -732,6 +733,9 @@ find in case extra indexes are specified. """, default = True, ), + "project_root": attr.label( + doc = "Label of the file defining the project root. If present, this label will be passed to all `whl_library` created from the requirement file. It will then expanded to a path and its parent directory will be made available in the PROJECT_ROOT environment variable when building the wheel.", + ), "python_version": attr.string( mandatory = True, doc = """ diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 79a58a81f2..7a4f0767d9 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -193,6 +193,10 @@ def _whl_library_impl(rctx): # Manually construct the PYTHONPATH since we cannot use the toolchain here environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger) + # Add a PROJECT_ROOT environment variable + if rctx.attr.project_root: + environment["PROJECT_ROOT"] = str(rctx.path(rctx.attr.project_root).dirname) + whl_path = None if rctx.attr.whl_file: whl_path = rctx.path(rctx.attr.whl_file) @@ -255,6 +259,17 @@ def _whl_library_impl(rctx): logger = logger, ) + # If the requirement was a local directory, then we need to watch its content + # to recreate the repository when it is modified. + # Assume that such packages are imported using a single line requirement + # of the form [ @] file:// + # We might have to perform some substitutions in the string before searching. + subst_req = envsubst(rctx.attr.requirement, environment.keys(), lambda x, dft: environment[x]) + if subst_req.startswith("file://"): + _, path = subst_req.split("file://", 1) + logger.info(lambda: "watching tree {} for wheel library {}".format(path, rctx.name)) + rctx.watch_tree(path) + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) if not rctx.delete("whl_file.json"): fail("failed to delete the whl_file.json file") @@ -404,6 +419,9 @@ and the target that we need respectively. "group_name": attr.string( doc = "Name of the group, if any.", ), + "project_root": attr.label( + doc = "Label of the file defining the project root. If present, this label will be expanded to a path and its parent directory will be made available in the PROJECT_ROOT environment variable when building the wheel.", + ), "repo": attr.string( mandatory = True, doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 1caab23cea..23c95ace87 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -91,6 +91,7 @@ def _parse( netrc = None, parse_all_requirements_files = True, pip_data_exclude = None, + project_root = None, python_interpreter = None, python_interpreter_target = None, quiet = True, @@ -119,6 +120,7 @@ def _parse( netrc = netrc, parse_all_requirements_files = parse_all_requirements_files, pip_data_exclude = pip_data_exclude, + project_root = project_root, python_interpreter = python_interpreter, python_interpreter_target = python_interpreter_target, python_version = python_version,