diff --git a/tools/lint/buildifier-tables.json b/tools/lint/buildifier-tables.json index 53cd40177c18..ffab24cb8266 100644 --- a/tools/lint/buildifier-tables.json +++ b/tools/lint/buildifier-tables.json @@ -37,12 +37,6 @@ "git_repository.remote": 300, "git_repository.commit": 310, - "pypi_archive.package": 300, - "pypi_archive.version": 310, - "pypi_archive.sha256": 340, - "pypi_archive.strip_prefix": 350, - "pypi_archive.build_file": 360, - "new_local_repository.path": 300, "new_local_repository.build_file_content": 360, diff --git a/tools/workspace/README.md b/tools/workspace/README.md index e429ada1b0b3..4ace93469f17 100644 --- a/tools/workspace/README.md +++ b/tools/workspace/README.md @@ -185,19 +185,6 @@ this will be susceptible to Ubuntu vs macOS differences, so please opt-in to the macOS build(s) in Jenkins before merging, using the instructions at https://drake.mit.edu/jenkins.html#running-an-on-demand-build. -## Updating pypi_archive software versions - -To lock in a new version, change the `version` argument of the `pypi_archive` -call, comment out the `sha256` argument, and then run `bazel build`. Bazel's -fetch step will attempt to download the new version but then complain about a -checksum mismatch. Paste the new checksum into the `sha256` argument and -remove its commenting-out. Then, `bazel build` should succeed. - -Commit and pull-request the changed lines to Drake as usual. Many changes like -this will be susceptible to Ubuntu vs macOS differences, so please opt-in to -the macOS build(s) in Jenkins before merging, using the instructions at -https://drake.mit.edu/jenkins.html#running-an-on-demand-build. - ## Updating pkg_config_repository software versions Most `pkg_config_repository` calls refer to libraries provided by the host diff --git a/tools/workspace/mirrors.bzl b/tools/workspace/mirrors.bzl index 4c885e11c37f..aa98c731f5cd 100644 --- a/tools/workspace/mirrors.bzl +++ b/tools/workspace/mirrors.bzl @@ -50,16 +50,6 @@ DEFAULT_MIRRORS = { # N.B. ibiblio doesn't offer https. "http://maven.ibiblio.org/maven2/{fulljar}", ], - "pypi": [ - "https://files.pythonhosted.org/packages/source/{p}/{package}/{package}-{version}.tar.gz", # noqa - "https://drake-mirror.csail.mit.edu/pypi/{package}/{package}-{version}.tar.gz", # noqa - "https://s3.amazonaws.com/drake-mirror/pypi/{package}/{package}-{version}.tar.gz", # noqa - ], - "pypi_wheel": [ - "https://files.pythonhosted.org/packages/{blake2_256_01}/{blake2_256_23}/{blake2_256_4p}/{package}-{version}-{tag}.whl", # noqa - "https://drake-mirror.csail.mit.edu/pypi_wheel/{package}/{package}-{version}-{tag}.tar.gz", # noqa - "https://s3.amazonaws.com/drake-mirror/pypi_wheel/{package}/{package}-{version}-{tag}.tar.gz", # noqa - ], # On 2024-01-01 upon completion of @vtk deprecation, remove this stanza. "vtk": [ "https://drake-packages.csail.mit.edu/vtk/{archive}", diff --git a/tools/workspace/new_release.py b/tools/workspace/new_release.py index 734e4e62778a..1b8e9803ac8d 100644 --- a/tools/workspace/new_release.py +++ b/tools/workspace/new_release.py @@ -206,11 +206,6 @@ def _check_for_upgrades(gh, args, metadata): # For details, see drake/tools/workspace/crate_universe/README.md. print(f"Ignoring {workspace_name} from rules_rust") continue - elif key in ["pypi", "pypi_wheel"]: - # TODO(jwnimmer-tri) Implement for real. - print("{} version {} needs manual inspection".format( - workspace_name, data["version"])) - continue elif workspace_name == "buildifier": assert key == "manual" old_commit, new_commit = _handle_buildifier(gh, data) diff --git a/tools/workspace/pypi.bzl b/tools/workspace/pypi.bzl deleted file mode 100644 index 3b4fee60f3cb..000000000000 --- a/tools/workspace/pypi.bzl +++ /dev/null @@ -1,255 +0,0 @@ -load( - "@bazel_tools//tools/build_defs/repo:utils.bzl", - "workspace_and_buildfile", -) -load("//tools/workspace:metadata.bzl", "generate_repository_metadata") - -def pypi_archive( - name, - package = None, - version = None, - sha256 = None, - strip_prefix = "", - build_file = None, - build_file_content = None, - workspace_file = None, - workspace_file_content = None, - mirrors = None): - """Downloads and unpacks a PyPI package archive and adds it to the - WORKSPACE as an external. - - Example: - Download and use the "foo" package, version 1.2.3, hosted on PyPI at - https://files.pythonhosted.org/packages/source/f/foo/foo-1.2.3.tar.gz. - - WORKSPACE: - load("//tools/workspace:pypi.bzl", "pypi_archive") - pypi_archive( - name = "foo", - version = "1.2.3", - build_file = "foo.BUILD", - sha256 = "0123456789abcdef...", - ) - - foo.BUILD: - load("//tools/skylark:py.bzl", "py_library") - py_library( - name = "foo", - srcs = [ - "foo/__init__.py", - "foo/bar.py", - ], - visibility = ["//visibility:public"], - ) - - BUILD: - load("//tools/skylark:py.bzl", "py_binary") - py_binary( - name = "foobar", - deps = ["@foo//:foo"], - srcs = ["foobar.py"], - ) - - Arguments: - name: A unique name for this rule. This argument will be used for the - package name if the "package" argument is omitted [Name; required]. - - package: The name of the PyPI package to download. The "name" argument - will be used if this argument is omitted [String; optional]. - - version: The version of the PyPI package to be downloaded - [String; required]. - - sha256: The expected SHA-256 hash of the archive to download. This - argument must match the SHA-256 hash of the downloaded archive. - The download will fail if omitted, but the checksum-mismatch error - message will offer a suggestion for the correct value of this - argument [String; required]. - - strip_prefix: A directory prefix to strip from the extracted files - [String; optional]. - - build_file: The file to use as the BUILD file for this repository. - This argument is an absolute label. Either build_file or - build_file_content must be specified, but not both - [Label; optional]. - - build_file_content: The content for the BUILD file for this repository. - Either build_file or build_file_content must be specified, but not - both [Label; optional]. - - workspace_file: The file to use as the WORKSPACE file for this - repository. This argument is an absolute label. Either - workspace_file or workspace_file_content may be specified, or - neither, but not both [Label; optional]. - - workspace_file_content: The content for the WORKSPACE file for this - repository. Either workspace_file, workspace_file_content or - neither may be specified, but not both [Label; optional]. - - mirrors: A dict from string to list-of-string with key "pypi", where - the list-of-strings are URLs to use, formatted using {package}, - {version}, and {p} (where {p} is the first letter of {package}). - """ - if not package: - package = name - - if not version: - fail("The version argument to pypi_archive is required.") - - if not build_file and not build_file_content: - fail("Either the build_file or build_file_content argument to " + - "pypi_archive is required.") - - _pypi_archive( - name = name, - package = package, - version = version, - sha256 = sha256, - strip_prefix = strip_prefix, - build_file = build_file, - build_file_content = build_file_content, - workspace_file = workspace_file, - workspace_file_content = workspace_file_content, - mirrors = mirrors, - ) - -def setup_pypi_repository(repository_ctx): - """Downloads and unpacks a PyPI package archive and adds it to the - WORKSPACE as an external. - - Args: - repository_ctx: context of a Bazel repository rule. - """ - pypi_download_and_extract( - repository_ctx, - package = repository_ctx.attr.package, - version = repository_ctx.attr.version, - mirrors = repository_ctx.attr.mirrors, - sha256 = repository_ctx.attr.sha256, - strip_prefix = repository_ctx.attr.strip_prefix, - ) - - workspace_and_buildfile(repository_ctx) - - return struct(error = None) - -def pypi_download_and_extract( - repository_ctx, - package, - version, - mirrors, - output = "", - sha256 = None, - strip_prefix = ""): - """Downloads an archive of the provided PyPI package and version to the - output path and extracts it. - - Args: - repository_ctx: context of a Bazel repository rule. - package: PyPI package name. - version: version for which the archive should be downloaded. - mirrors: dictionary of mirrors, see mirrors.bzl in this directory for - an example. - output: path to the directory where the archive will be unpacked, - relative to the Bazel repository directory. - sha256: expected SHA-256 hash of the archive downloaded. Fallback to - an incorrect default value to prevent the hash check from being - disabled, but allow the first download attempt to fail and print - the correct SHA-256 hash. - strip_prefix: additional directory prefix to strip from the extracted - files. - """ - urls = _urls(package, version, mirrors) - - repository_ctx.download_and_extract( - urls, - output = output, - sha256 = _sha256(sha256), - type = "tar.gz", - stripPrefix = _strip_prefix(package, version, strip_prefix), - ) - - generate_repository_metadata( - repository_ctx, - repository_rule_type = "pypi", - package = package, - version = version, - sha256 = sha256, - urls = urls, - ) - -def _sha256(sha256): - """Fallback to an incorrect default value of SHA-256 hash to prevent the - hash check from being disabled, but allow the first download attempt of an - archive to fail and print the correct hash. - - Args: - sha256: expected SHA-256 hash of the archive to be downloaded. - """ - if not sha256: - sha256 = "0" * 64 - - return sha256 - -def _strip_prefix(package, version, strip_prefix = ""): - """Computes the strip prefix for a downloaded archive of the provided - PyPI package and version. - - Args: - package: PyPI package name. - version: version for which the archive should be downloaded. - strip_prefix: additional directory prefix to strip from the extracted - files. - """ - if strip_prefix: - return "{0}-{1}/{2}".format(package, version, strip_prefix) - - return "{0}-{1}".format(package, version) - -def _urls(package, version, mirrors): - """Computes the urls from which an archive of the provided PyPI package and - version should be downloaded. - - Args: - package: PyPI package name. - version: version for which the archive should be downloaded. - mirrors: dictionary of mirrors, see mirrors.bzl in this directory for - an example. - """ - return [ - x.format( - p = package[:1], - package = package, - version = version, - ) - for x in mirrors.get("pypi") - ] - -def _pypi_archive_impl(repository_ctx): - result = setup_pypi_repository(repository_ctx) - - if result.error: - fail("Unable to complete setup for @{} repository: {}".format( - repository_ctx.name, - result.error, - )) - -_pypi_archive = repository_rule( - attrs = { - "package": attr.string(), - "version": attr.string(), - "sha256": attr.string(), - "strip_prefix": attr.string(), - "build_file": attr.label(allow_single_file = True), - "build_file_content": attr.string(), - "workspace_file": attr.label(), - "workspace_file_content": attr.string(), - "mirrors": attr.string_list_dict( - mandatory = True, - allow_empty = False, - ), - }, - environ = ["BAZEL_SH"], - implementation = _pypi_archive_impl, -) diff --git a/tools/workspace/pypi_wheel.bzl b/tools/workspace/pypi_wheel.bzl deleted file mode 100644 index 88e21308a884..000000000000 --- a/tools/workspace/pypi_wheel.bzl +++ /dev/null @@ -1,165 +0,0 @@ -load("//tools/workspace:os.bzl", "determine_os") -load("//tools/workspace:metadata.bzl", "generate_repository_metadata") - -def download_and_extract_pypi_wheel( - repository_ctx, - package = None, - version = None, - version_pin = None, - pypi_tag = None, - blake2_256 = None, - sha256 = None, - mirrors = None): - """Downloads and unpacks a PyPI wheel. - - Arguments: - package: The name of the PyPI package to download [String; required]. - - version: The version of the PyPI package to download - [String; required]. - - version_pin: True iff the wheel should remain at the same version - indefinitely, eschewing automated upgrades to newer versions. - [Bool; optional]. - - pypi_tag: The tag of the PyPI package to download, e.g., - "cp36-cp36m-manylinux1_x86_64". - This is seen on the PyPI webpage under "Download Files" then "View" - [String; required]. - - blake2_256: The expected BLAKE2-256 hash of the archive to download. - This is seen on the PyPI webpage under "Download Files" then "View" - [String; required]. - - sha256: The expected SHA-256 hash of the archive to download. This - argument must match the SHA-256 hash of the downloaded archive. - The download will fail if omitted, but the checksum-mismatch error - message will offer a suggestion for the correct value of this - argument [String; required]. - - mirrors: A dict from string to list-of-string with key "pypi_wheel", - where the list-of-strings are URLs to use, formatted using - {package}, {version}, {tag}, {sha256}, {blake2_256}, - {blake2_256_01}, {blake2_256_23}, {blake2_256_4p} where the final - three are respectively the first two characters of {blake2_256}, - second two characters of {blake2_256}, and remaining characters - of {blake2_256}. - """ - - # Validate our args. - package or fail("Missing required package") - version or fail("Missing required version") - pypi_tag or fail("Missing required pypi_tag") - blake2_256 or fail("Missing required blake2_256") - mirrors or fail("Missing required mirrors") - sha256 = sha256 or "0" * 64 - - # Download and extract the wheel. - urls = [ - pattern.format( - package = package, - version = version, - tag = pypi_tag, - blake2_256 = blake2_256, - blake2_256_01 = blake2_256[:2], - blake2_256_23 = blake2_256[2:4], - blake2_256_4p = blake2_256[4:], - sha256 = sha256, - ) - for pattern in mirrors["pypi_wheel"] - ] - repository_ctx.download_and_extract( - url = urls, - sha256 = sha256, - type = "zip", - ) - - # Wheel files can store things at the top level in a usable way, or in a - # separate data directory. If we have the separate data directory, move - # its contents to the repository root. - data_dir = "{}-{}.data".format(package, version) - if repository_ctx.path(data_dir).exists: - data_subdirs = repository_ctx.path(data_dir).readdir() - for data_subdir in data_subdirs: - for to_move in data_subdir.readdir(): - result = repository_ctx.execute(["mv", to_move, "."]) - if result.return_code: - fail("{} error moving file {}".format( - repository_ctx.name, - to_move, - )) - - # Tell our mirroring scripts what we downloaded. - generate_repository_metadata( - repository_ctx, - repository_rule_type = "pypi_wheel", - package = package, - version = version, - version_pin = version_pin, - pypi_tag = pypi_tag, - blake2_256 = blake2_256, - sha256 = sha256, - urls = urls, - ) - - return struct(error = None) - -def setup_pypi_wheel( - repository_ctx, - package = None, - version = None, - version_pin = None, - pypi_tag = None, - blake2_256 = None, - sha256 = None, - mirrors = None, - deps = [], - imports = [], - data = []): - """Downloads and unpacks a PyPI wheel and adds it to the WORKSPACE as an - external. - - Arguments: - package, version, pypi_tag, blake2_256, sha256, mirrors: Arguments to - be passed to download_and_extract_pypi_wheel() [Required]. - - version_pin: Arguments to be passed to - download_and_extract_pypi_wheel() [Optional]. - - deps, imports, data: Additional attrs for the target in the BUILD file - [Optional]. - """ - - # Download and extract the wheel. - download_and_extract_pypi_wheel( - repository_ctx, - package = package, - version = version, - version_pin = version_pin, - pypi_tag = pypi_tag, - blake2_256 = blake2_256, - sha256 = sha256, - mirrors = mirrors, - ) - - # Create the BUILD file. - BUILD = """ -load("@drake//tools/skylark:py.bzl", "py_library") - -py_library( - name = "{name}", - srcs = glob(["**/*.py"]), - data = glob(["**/*.so", "**/*.so.*"]) + {extra_data}, - deps = {deps}, - imports = {imports}, - visibility = ["//visibility:public"], -) - """.format( - name = repository_ctx.name, - extra_data = data, - deps = [str(x) for x in deps], - imports = imports, - ) - repository_ctx.file("BUILD.bazel", BUILD) - - return struct(error = None)