diff --git a/CMakeLists.txt b/CMakeLists.txt index a2f336f3ce97..92a2dad6d3ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,8 @@ project(drake # (e.g., `-DCMAKE_BUILD_TYPE=Release`) and install Drake using those settings. # # We'll do that by converting the settings to generated Bazel inputs: -# - a `WORKSPACE.bazel` file that specifies dependencies; and +# - a generated `MODULE.bazel` that depends on the Drake module and customizes +# the toolchain selection. # - a `.bazelrc` file that specifies configuration choices. # and then running the `@drake//:install` program from that temporary workspace. @@ -77,9 +78,8 @@ else() endif() endif() -# The version passed to find_package(Bazel) should match the -# minimum_bazel_version value in the call to versions.check() in WORKSPACE. -set(MINIMUM_BAZEL_VERSION 7.4) +# This version number should match bazel_compatibility in MODULE.bazel. +set(MINIMUM_BAZEL_VERSION 7.4.1) find_package(Bazel ${MINIMUM_BAZEL_VERSION} MODULE) if(NOT Bazel_FOUND) set(Bazel_EXECUTABLE "${PROJECT_SOURCE_DIR}/third_party/com_github_bazelbuild_bazelisk/bazelisk.py") @@ -355,17 +355,10 @@ function(symlink_external_repository_libs NAME TARGET) file(CREATE_LINK "${location}" "${workspace}/${NAME}/lib/${other_basename}" SYMBOLIC) endfunction() -set(BAZEL_WORKSPACE_EXTRA) -set(BAZEL_WORKSPACE_EXCLUDES) - -# Our cmake/WORKSPACE.bzlmod always provides @python. -list(APPEND BAZEL_WORKSPACE_EXCLUDES "python") - macro(override_repository NAME) set(repo "${CMAKE_CURRENT_BINARY_DIR}/external/workspace/${NAME}") - string(APPEND BAZEL_WORKSPACE_EXTRA - "local_repository(name = '${NAME}', path = '${repo}')\n") - list(APPEND BAZEL_WORKSPACE_EXCLUDES "${NAME}") + string(APPEND BAZEL_REPO_ENV + " --override_repository=+drake_dep_repositories+${NAME}=${repo}") endmacro() option(WITH_USER_EIGEN "Use user-provided Eigen3" OFF) @@ -507,7 +500,7 @@ endif() # N.B. If you are testing the CMake API and making changes to `installer.py`, # you can change this target to something more lightweight, such as # `//tools/install/dummy:install`. -set(BAZEL_INSTALL_TARGET //:install) +set(BAZEL_INSTALL_TARGET "@drake//:install") if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/install" CACHE STRING @@ -553,10 +546,8 @@ endforeach() # however, that the macOS wheel builds also need to know this path, so if it # ever changes, tools/wheel/macos/build-wheel.sh will also need to be updated. configure_file(cmake/bazel.rc.in drake_build_cwd/.bazelrc @ONLY) -configure_file(cmake/WORKSPACE.bzlmod.in drake_build_cwd/WORKSPACE.bzlmod @ONLY) +configure_file(cmake/MODULE.bazel.in drake_build_cwd/MODULE.bazel @ONLY) file(CREATE_LINK "${PROJECT_SOURCE_DIR}/.bazeliskrc" drake_build_cwd/.bazeliskrc SYMBOLIC) -file(CREATE_LINK "${PROJECT_SOURCE_DIR}/MODULE.bazel" drake_build_cwd/MODULE.bazel SYMBOLIC) -file(CREATE_LINK "${PROJECT_SOURCE_DIR}/WORKSPACE" drake_build_cwd/WORKSPACE SYMBOLIC) find_package(Git) diff --git a/MODULE.bazel b/MODULE.bazel index 983a7c83c987..c7a22c7dc2f9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,7 +4,13 @@ # This file lists Drake's external dependencies as known to bzlmod. It is used # in concert with WORKSPACE.bzlmod (which has the workspace-style externals). -module(name = "drake") +module( + name = "drake", + # This version number should match MINIMUM_BAZEL_VERSION in CMakeLists.txt. + bazel_compatibility = [">=7.4.1"], +) + +# Add starlark rules. bazel_dep(name = "apple_support", version = "1.17.1", repo_name = "build_bazel_apple_support") # noqa bazel_dep(name = "bazel_features", version = "1.22.0") @@ -17,13 +23,197 @@ bazel_dep(name = "rules_python", version = "0.40.0") bazel_dep(name = "rules_rust", version = "0.56.0") bazel_dep(name = "rules_shell", version = "0.3.0") +# Customize our toolchains. + cc_configure = use_extension( "@rules_cc//cc:extensions.bzl", "cc_configure_extension", ) use_repo(cc_configure, "local_config_cc") -# TODO(#20731) Move all of our dependencies from WORKSPACE.bzlmod into this -# file, so that downstream projects can consume Drake exclusively via bzlmod -# (and so that we can delete our WORKSPACE files prior to Bazel 9 which drops -# suppose for it). +# Load dependencies which are "public", i.e., made available to downstream +# projects. +# +# Downstream projects may load the same `drake_dep_repositories` module +# extension shown below and call its `use_repo` with whatever list of +# repositories they desire to cite from their project. It's safe to call +# `use_repo` on a subset of this list, or not call it at all downstream. +# Its only effect on a downstream project is to make the repository name +# visible to BUILD rules; Drake's own use of the repository is unaffected. + +drake_dep_repositories = use_extension( + "@drake//tools/workspace:default.bzl", + "drake_dep_repositories", +) +use_repo( + drake_dep_repositories, + "blas", + "buildifier", + "drake_models", + "eigen", + "fmt", + "gflags", + "glib", + "glx", + "gtest", + "gurobi", + "lapack", + "lcm", + "libblas", + "liblapack", + "meshcat", + "mosek", + "opencl", + "opengl", + "pybind11", + "pycodestyle", + "python", + "snopt", + "spdlog", + "styleguide", + "x11", + "zlib", +) + +# Load dependencies which are "private", i.e., not available for use by +# downstream projects. These are all "internal use only". +# +# TODO(jwnimmer-tri) By historical accident, not all of the repository names +# here end with "internal". We should work on improving the consistency, either +# by switching them to use BCR modules instead (e.g., nasm) or renaming them +# with any necessary deprecations (e.g., statsjs). + +internal_repositories = use_extension( + "@drake//tools/workspace:default.bzl", + "internal_repositories", +) +use_repo( + internal_repositories, + "abseil_cpp_internal", + "bazelisk", + "ccd_internal", + "clang_cindex_python3_internal", + "clarabel_cpp_internal", + "clp_internal", + "coinutils_internal", + "com_jidesoft_jide_oss", + "common_robotics_utilities_internal", + "commons_io", + "conex_internal", + "csdp_internal", + "curl_internal", + "dm_control_internal", + "doxygen", + "fcl_internal", + "gfortran", + "github3_py_internal", + "gklib_internal", + "googlebenchmark", + "gymnasium_py", + "gz_math_internal", + "gz_utils_internal", + "highway_internal", + "ipopt_internal", + "libjpeg_turbo_internal", + "libpng_internal", + "libtiff_internal", + "metis_internal", + "mpmath_py_internal", + "msgpack_internal", + "mujoco_menagerie_internal", + "mumps_internal", + "mypy_extensions_internal", + "mypy_internal", + "nanoflann_internal", + "nasm", + "net_sf_jchart2d", + "nlohmann_internal", + "nlopt_internal", + "onetbb_internal", + "openusd_internal", + "org_apache_xmlgraphics_commons", + "osqp_internal", + "picosha2_internal", + "poisson_disk_sampling_internal", + "qdldl_internal", + "qhull_internal", + "ros_xacro_internal", + "rules_python_drake_constants", + "scs_internal", + "sdformat_internal", + "spgrid_internal", + "spral_internal", + "stable_baselines3_internal", + "statsjs", + "stduuid_internal", + "suitesparse_internal", + "sympy_py_internal", + "tinygltf_internal", + "tinyobjloader_internal", + "tinyxml2_internal", + "tomli_internal", + "typing_extensions_internal", + "uritemplate_py_internal", + "usockets_internal", + "uwebsockets_internal", + "voxelized_geometry_tools_internal", + "vtk_internal", + "xmlrunner_py", + "yaml_cpp_internal", +) + +internal_crate_universe_repositories = use_extension( + "//tools/workspace:default.bzl", + "internal_crate_universe_repositories", +) +use_repo( + internal_crate_universe_repositories, + "crate__amd-0.2.2", + "crate__autocfg-1.4.0", + "crate__blas-0.22.0", + "crate__blas-sys-0.7.1", + "crate__cfg-if-1.0.0", + "crate__clarabel-0.9.0", + "crate__darling-0.14.4", + "crate__darling_core-0.14.4", + "crate__darling_macro-0.14.4", + "crate__derive_builder-0.11.2", + "crate__derive_builder_core-0.11.2", + "crate__derive_builder_macro-0.11.2", + "crate__either-1.13.0", + "crate__enum_dispatch-0.3.13", + "crate__equivalent-1.0.1", + "crate__fnv-1.0.7", + "crate__hashbrown-0.15.2", + "crate__ident_case-1.0.1", + "crate__indexmap-2.7.0", + "crate__itertools-0.11.0", + "crate__itoa-1.0.14", + "crate__lapack-0.19.0", + "crate__lapack-sys-0.14.0", + "crate__lazy_static-1.5.0", + "crate__libc-0.2.169", + "crate__memchr-2.7.4", + "crate__num-complex-0.4.6", + "crate__num-traits-0.2.19", + "crate__once_cell-1.19.0", + "crate__paste-1.0.15", + "crate__proc-macro2-1.0.92", + "crate__quote-1.0.38", + "crate__ryu-1.0.18", + "crate__serde-1.0.217", + "crate__serde_derive-1.0.217", + "crate__serde_json-1.0.134", + "crate__strsim-0.10.0", + "crate__syn-1.0.109", + "crate__syn-2.0.94", + "crate__thiserror-1.0.69", + "crate__thiserror-impl-1.0.69", + "crate__unicode-ident-1.0.14", +) + +# TODO(#20731) More improvements are still needed to our MODULE organization: +# - Switch public API dependencies (e.g., eigen) to use modules. +# - Provide better configuration options for choosing dependencies. +# - Adjust the wheel build to build more dependencies as Bazel modules. +# - Deprecate non-bzlmod use of Drake downstream. diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index df38712b8dd8..20f61a63a8fc 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -1,22 +1,13 @@ # -*- bazel -*- # -# This file lists Drake's workspace-style external dependencies. It is used in -# concert with MODULE.bazel (which has the module-style externals). +# This file is only ever used by drake/tools/clion/bazel_wrapper (for Drake +# Developers who develop using CLion; see https://drake.mit.edu/clion.html). +# +# Do not add any other new dependencies into this file. workspace(name = "drake") -load("//tools/workspace:default.bzl", "add_default_workspace") - -add_default_workspace(bzlmod = True) - # Add some special heuristic logic for using CLion with Drake. load("//tools/clion:repository.bzl", "drake_clion_environment") drake_clion_environment() - -load("@bazel_skylib//lib:versions.bzl", "versions") - -# This needs to be in WORKSPACE or a repository rule for native.bazel_version -# to actually be defined. The minimum_bazel_version value should match the -# version passed to the find_package(Bazel) call in the root CMakeLists.txt. -versions.check(minimum_bazel_version = "7.4") diff --git a/cmake/MODULE.bazel.in b/cmake/MODULE.bazel.in new file mode 100644 index 000000000000..48a1daf76e48 --- /dev/null +++ b/cmake/MODULE.bazel.in @@ -0,0 +1,25 @@ +module(name = "drake_cmake") + +bazel_dep(name = "rules_python", version = "0.40.0") + +bazel_dep(name = "drake") +local_path_override( + module_name = "drake", + path = "@PROJECT_SOURCE_DIR@", +) + +python_repository = use_repo_rule( + "@drake//tools/workspace/python:repository.bzl", + "python_repository", +) + +# Use Drake's python repository rule to interrogate the interpreter chosen by +# the CMake find_program stanza, in support of compiling our C++ bindings. +python_repository( + name = "python", + linux_interpreter_path = "@Python_EXECUTABLE@", + macos_interpreter_path = "@Python_EXECUTABLE@", + requirements_flavor = "build", +) + +register_toolchains("@python//:all") diff --git a/cmake/WORKSPACE.bzlmod.in b/cmake/WORKSPACE.bzlmod.in deleted file mode 100644 index 6e95de09c039..000000000000 --- a/cmake/WORKSPACE.bzlmod.in +++ /dev/null @@ -1,23 +0,0 @@ -workspace(name = "drake") - -load("//:cmake/external/workspace/conversion.bzl", "split_cmake_list") -load("//tools/workspace/python:repository.bzl", "python_repository") -load("//tools/workspace:default.bzl", "add_default_workspace") - -# Use Drake's python repository rule to interrogate the interpreter chosen by -# the CMake find_program stanza, in support of compiling our C++ bindings. -python_repository( - name = "python", - linux_interpreter_path = "@Python_EXECUTABLE@", - macos_interpreter_path = "@Python_EXECUTABLE@", - requirements_flavor = "build", -) - -# Custom repository rules injected by CMake. -@BAZEL_WORKSPACE_EXTRA@ - -# For anything not already overridden, use Drake's default externals. -add_default_workspace( - repository_excludes = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@"), - bzlmod = True, -) diff --git a/cmake/bazel.rc.in b/cmake/bazel.rc.in index b3a139abc9b2..987d46c6e583 100644 --- a/cmake/bazel.rc.in +++ b/cmake/bazel.rc.in @@ -10,17 +10,10 @@ startup --output_base="@BAZEL_OUTPUT_BASE@" # Environment variables to be used in repository rules (if any). common @BAZEL_REPO_ENV@ -# Use the Python interpreter from our cmake/WORKSPACE.bzlmod.in. -build --extra_toolchains=@python//:all - # Disable the "convenience symlinks" intended for Bazel users; they only add # confusion for the CMake use case. build --symlink_prefix=/ -# Use the source code and BUILD files from the Drake source tree. -# The WORKSPACE lives in the build directory, so by has no packages otherwise. -build --package_path="%workspace%:@PROJECT_SOURCE_DIR@" - # Fix macOS per https://github.com/bazelbuild/bazel/issues/14294. build --notrim_test_configuration diff --git a/doc/_pages/stable.md b/doc/_pages/stable.md index 72c1a88a0383..ee7a563ece27 100644 --- a/doc/_pages/stable.md +++ b/doc/_pages/stable.md @@ -107,20 +107,31 @@ part of the Stable API. For Drake's dependencies: -* The `add_default_...` macros defined in `@drake//tools/workspace:default.bzl` - are all part of the Stable API. - * For any Bazel external loaded by these functions (e.g., `"@eigen"`), we - will deprecate it prior to removing our definition of the dependency. - * Excluding any items documented as "internal use only". - * Excluding any items documented with an "experimental" warning. +* When using Bazel to depend on Drake as a Bazel Module (i.e., using bzlmod): + * The extension module + `use_extension("@drake//tools/workspace:default.bzl", "drake_dep_repositories")` + is part of the Stable API, including the names of the repositories it offers + as extensions (e.g., `"eigen"`). + * For any repository provided by the extension, we will deprecate + it prior to removing it. +* When using Bazel to depend on Drake via `WORKSPACE.bazel` (i.e., without + bzlmod): + * The `add_default_...` macros defined in + `@drake//tools/workspace:default.bzl` are all part of the Stable API. + * For any Bazel external loaded by these functions (e.g., `"@eigen"`), we + will deprecate it prior to removing our definition of the dependency. + * Excluding any items documented as "internal use only". + * Excluding any items documented with an "experimental" warning. We may upgrade any of our dependencies to a newer version without prior notice. If you require an older version, you will need to rebuild Drake from source and -pin your own WORKSPACE to refer to the older version of the dependency. +customize your own `WORKSPACE.bazel` or `MODULE.bazel` file to refer to the +older version of the dependency. We may add new dependencies without prior notice. All of our dependencies will either be installed via the host system via our `install_prereqs` scripts, -and/or downloaded at build-time via our `add_default_...` macros, and/or +and/or downloaded at build-time via our `add_default_...` macros (when not +using `bzlmod`) or our `MODULE.bazel` file (when using bzlmod), and/or specified via packaging metadata in the case of `apt` or `pip`. ## LCM messages diff --git a/tools/bazel.rc b/tools/bazel.rc index 8b9210670a6c..54517618874d 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,6 +1,3 @@ -# TODO(#20731) Stop using WORKSPACE (and WORKSPACE.bzlmod). -common --enable_workspace=true - # Require all rules to be loaded in MODULE.bazel -- don't allow any legacy # implicit loads built-in to Bazel itself to take effect. common --incompatible_autoload_externally= diff --git a/tools/clion/bazel_wrapper b/tools/clion/bazel_wrapper index 39d1c3a079ca..fb65ceb0ff9c 100755 --- a/tools/clion/bazel_wrapper +++ b/tools/clion/bazel_wrapper @@ -51,6 +51,7 @@ def main(argv, execvp, write_stderr, popen): new_magic = "--aspects=@drake//tools/clion:aspect.bzl%intellij_info_aspect" if old_magic in args: args[args.index(old_magic)] = new_magic + args.insert(0, "--enable_workspace=true") # If stream editing is disabled, just delegate everything to Bazel. nostream_magic = "--nodrake_error_rewriting" diff --git a/tools/clion/test/bazel_wrapper_test.py b/tools/clion/test/bazel_wrapper_test.py index 517d683042d3..f7a734f732d8 100644 --- a/tools/clion/test/bazel_wrapper_test.py +++ b/tools/clion/test/bazel_wrapper_test.py @@ -112,7 +112,7 @@ def test_no_rewriting(self): self.assertEqual(detail.exception.name, "bazel") self.assertSequenceEqual( detail.exception.args, - ["bazel", new_magic, "dummy_arg"]) + ["bazel", "--enable_workspace=true", new_magic, "dummy_arg"]) self.assertEqual("", self._stderr.decode("utf-8")) def test_subprocess(self): diff --git a/tools/install/install.bzl b/tools/install/install.bzl index 18a4f4168d5c..e006230a0a5c 100644 --- a/tools/install/install.bzl +++ b/tools/install/install.bzl @@ -22,17 +22,19 @@ InstalledTestInfo = provider() def _workspace(ctx): """Compute name of current workspace.""" - # Check for override + # Check for override. if hasattr(ctx.attr, "workspace"): if len(ctx.attr.workspace): return ctx.attr.workspace - # Check for meaningful workspace_root + # Check for meaningful workspace_root (using the apparent repository name + # for brevity, not the canonical repository name with "+" symbols). workspace = ctx.label.workspace_root.split("/")[-1] + workspace = workspace.split("+")[-1] if len(workspace): return workspace - # If workspace_root is empty, assume we are the root workspace + # If workspace_root is empty, assume we are the root workspace. return ctx.workspace_name def _rename(file_dest, rename): @@ -524,11 +526,13 @@ def _install_impl(ctx): ) # Generate install script. + installer_binary = ctx.attr._installer[DefaultInfo] ctx.actions.write( output = ctx.outputs.executable, content = "\n".join([ "#!/bin/bash", - "tools/install/installer --actions '{}' \"$@\"".format( + "{} --actions '{}' \"$@\"".format( + installer_binary.files.to_list()[0].short_path, actions_file.short_path, ), ]), @@ -552,7 +556,7 @@ def _install_impl(ctx): ) # Return actions. - installer_runfiles = ctx.attr._installer[DefaultInfo].default_runfiles + installer_runfiles = installer_binary.default_runfiles action_runfiles = ctx.runfiles(files = ( [a.src for a in actions if not hasattr(a, "main_class")] + [i.src for i in installed_tests] + diff --git a/tools/install/libdrake/header_lint.bzl b/tools/install/libdrake/header_lint.bzl index e6f2c7729db6..e3dacddaef05 100644 --- a/tools/install/libdrake/header_lint.bzl +++ b/tools/install/libdrake/header_lint.bzl @@ -10,13 +10,12 @@ load("//tools/skylark:sh.bzl", "sh_test") # without consulting Drake's build system maintainers (see #7451). Keep this # list in sync with test/header_dependency_test.py. _ALLOWED_EXTERNALS = [ - "eigen", - "fmt", - "lcm", - "spdlog", - - # The entries that follow are defects; we should work to remove them. - "zlib", + "+drake_dep_repositories+eigen", + "+drake_dep_repositories+fmt", + "+drake_dep_repositories+spdlog", + # N.B. LCM is not allowed by the header_dependency_test; our allowed use + # of LCM is only for linking to, not for direct inclusion in our headers. + "+drake_dep_repositories+lcm", ] # Drake's allowed list of public preprocessor definitions. The only things diff --git a/tools/skylark/pybind.bzl b/tools/skylark/pybind.bzl index 339d8cb40e31..cfc6d3e2c887 100644 --- a/tools/skylark/pybind.bzl +++ b/tools/skylark/pybind.bzl @@ -70,7 +70,7 @@ def pybind_py_library( copts = cc_copts + EXTRA_PYBIND_COPTS, # Always link to pybind11. deps = [ - "@pybind11", + "@drake//tools/workspace/pybind11", ] + cc_deps, **kwargs ) diff --git a/tools/wheel/wheel_builder/common.py b/tools/wheel/wheel_builder/common.py index be44287c9d89..5a0ac7e38d29 100644 --- a/tools/wheel/wheel_builder/common.py +++ b/tools/wheel/wheel_builder/common.py @@ -92,7 +92,8 @@ def create_snopt_tgz(*, snopt_path, output): output_base = subprocess.check_output( command, cwd=resource_root, stderr=subprocess.DEVNULL, encoding='utf-8').strip() - bazel_snopt = os.path.join(output_base, 'external/snopt') + bazel_snopt = os.path.join( + output_base, 'external/+internal_repositories+snopt') # Ask Bazel to fetch SNOPT from its default git pin. command = [ diff --git a/tools/workspace/crate_universe/upgrade.sh b/tools/workspace/crate_universe/upgrade.sh index 92f31806482e..607814bae0b8 100755 --- a/tools/workspace/crate_universe/upgrade.sh +++ b/tools/workspace/crate_universe/upgrade.sh @@ -33,7 +33,8 @@ perl -pi -e ' s/^/# This file is automatically generated by upgrade.sh.\n/; ' lock/archives.bzl -# Sanity check and then clean up. If the defs.bzl split into archives.bzl ended -# up with errors, the defs.bzl.orig file will hang around as a reference. -bazel test --config=lint //... -rm -f lock/details/defs.bzl.orig +# TODO(jwnimmer-tri) It would be nice to automate the MODULE.bazel edits, +# but we anticipate ditching this whole thing in the next couple months, +# so probably not worth it. +echo "*** NOTE ***" +echo "You must also fix the version numbers in MODULE.bazel by hand now." diff --git a/tools/workspace/default.bzl b/tools/workspace/default.bzl index b54374715e79..6e5eb3df99be 100644 --- a/tools/workspace/default.bzl +++ b/tools/workspace/default.bzl @@ -111,40 +111,21 @@ load("//tools/workspace/xmlrunner_py:repository.bzl", "xmlrunner_py_repository") load("//tools/workspace/yaml_cpp_internal:repository.bzl", "yaml_cpp_internal_repository") # noqa load("//tools/workspace/zlib:repository.bzl", "zlib_repository") -# This is the list of modules that our MODULE.bazel already incorporates. -# It is cross-checked by the workspace_bzlmod_sync_test.py test. -REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [ - "build_bazel_apple_support", - "bazel_features", - "bazel_skylib", - "platforms", - "rust_toolchain", - "rules_cc", - "rules_java", - "rules_license", - "rules_python", - "rules_rust", - "rules_shell", -] +# ============================================================================= +# For Bazel projects using Drake as a dependency via the WORKSPACE mechanism. +# ============================================================================= -def add_default_repositories( - excludes = [], - mirrors = DEFAULT_MIRRORS, - *, - bzlmod = False): +def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS): """Declares workspace repositories for all externals needed by drake (other - than those built into Bazel, of course). This is intended to be loaded and - called from a WORKSPACE file. + than those built into Bazel, of course). For users, this is intended to be + loaded and called from a WORKSPACE file. (Drake also calls it internally + in service of our module extension infrastructure.) Args: excludes: list of string names of repositories to exclude; this can be useful if a WORKSPACE file has already supplied its own external of a given name. - bzlmod: when True, skips repositories declared in our MODULE.bazel; - set this to True if you are using bzlmod. """ - if bzlmod: - excludes = excludes + REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES if "abseil_cpp_internal" not in excludes: abseil_cpp_internal_repository(name = "abseil_cpp_internal", mirrors = mirrors) # noqa if "bazelisk" not in excludes: @@ -377,24 +358,14 @@ def add_default_repositories( if "zlib" not in excludes: zlib_repository(name = "zlib") -def add_default_toolchains( - excludes = [], - *, - bzlmod = False): +def add_default_toolchains(excludes = []): """Register toolchains for each language (e.g., "py") not explicitly excluded and/or not using an automatically generated toolchain. Args: excludes: List of languages for which a toolchain should not be registered. - bzlmod: when True, skips toolchains declared in our MODULE.bazel; - set this to True if you are using bzlmod. """ - if bzlmod: - # The cc toolchain is in MODULE.bazel already. - # The py toolchain is in tools/bazel.rc already. - return - if "py" not in excludes: native.register_toolchains("@python//:all") if "rust" not in excludes: @@ -403,9 +374,7 @@ def add_default_toolchains( def add_default_workspace( repository_excludes = [], toolchain_excludes = [], - mirrors = DEFAULT_MIRRORS, - *, - bzlmod = False): + mirrors = DEFAULT_MIRRORS): """Declare repositories in this WORKSPACE for each dependency of @drake (e.g., "eigen") that is not explicitly excluded, and register toolchains for each language (e.g., "py") not explicitly excluded and/or not using an @@ -419,16 +388,120 @@ def add_default_workspace( mirrors: Dictionary of mirrors from which to download repository files. See mirrors.bzl file in this directory for the file format and default values. - bzlmod: when True, skips repositories and toolchains declared in our - MODULE.bazel; set this to True if you are using bzlmod. """ - add_default_repositories( - excludes = repository_excludes, - mirrors = mirrors, - bzlmod = bzlmod, - ) - add_default_toolchains( - excludes = toolchain_excludes, - bzlmod = bzlmod, + add_default_repositories(excludes = repository_excludes, mirrors = mirrors) + add_default_toolchains(excludes = toolchain_excludes) + +# ============================================================================= +# For Bazel projects using Drake as a dependency via the MODULE mechanism. +# ============================================================================= + +# This is the list of modules that our MODULE.bazel already incorporates. +# It is cross-checked by the workspace_bzlmod_sync_test.py test. +REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [ + "build_bazel_apple_support", + "bazel_features", + "bazel_skylib", + "platforms", + "rust_toolchain", + "rules_cc", + "rules_java", + "rules_license", + "rules_python", + "rules_rust", + "rules_shell", +] + +# This is the list of repositories that Drake provides as a module extension +# for downstream projects; see comments in drake/MODULE.bazel for details. +# It is cross-checked by the workspace_bzlmod_sync_test.py test. +REPOS_EXPORTED = [ + "blas", + "buildifier", + "drake_models", + "eigen", + "fmt", + "gflags", + "glib", + "glx", + "gtest", + "gurobi", + "lapack", + "lcm", + "libblas", + "liblapack", + "meshcat", + "mosek", + "opencl", + "opengl", + "pybind11", + "pycodestyle", + "python", + "snopt", + "spdlog", + "styleguide", + "x11", + "zlib", +] + +def _drake_dep_repositories_impl(module_ctx): + # This sequence should match REPOS_EXPORTED exactly. + # Mismatches will be reported as errors by Bazel. + mirrors = DEFAULT_MIRRORS + blas_repository(name = "blas") + buildifier_repository(name = "buildifier", mirrors = mirrors) + drake_models_repository(name = "drake_models", mirrors = mirrors) + eigen_repository(name = "eigen") + fmt_repository(name = "fmt", mirrors = mirrors) + gflags_repository(name = "gflags", mirrors = mirrors) + glib_repository(name = "glib") + glx_repository(name = "glx") + gtest_repository(name = "gtest", mirrors = mirrors) + gurobi_repository(name = "gurobi") + lapack_repository(name = "lapack") + lcm_repository(name = "lcm", mirrors = mirrors) + libblas_repository(name = "libblas") + liblapack_repository(name = "liblapack") + meshcat_repository(name = "meshcat", mirrors = mirrors) + mosek_repository(name = "mosek", mirrors = mirrors) + opencl_repository(name = "opencl") + opengl_repository(name = "opengl") + pybind11_repository(name = "pybind11", mirrors = mirrors) + pycodestyle_repository(name = "pycodestyle", mirrors = mirrors) + python_repository(name = "python") + snopt_repository(name = "snopt") + spdlog_repository(name = "spdlog", mirrors = mirrors) + styleguide_repository(name = "styleguide", mirrors = mirrors) + x11_repository(name = "x11") + zlib_repository(name = "zlib") + +drake_dep_repositories = module_extension( + implementation = _drake_dep_repositories_impl, + doc = """(Stable API) Provides access to Drake's dependencies for use by + downstream projects. See comments in drake/MODULE.bazel for details.""", +) + +def _internal_repositories_impl(module_ctx): + excludes = ( + REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES + + REPOS_EXPORTED + + ["crate_universe"] ) + add_default_repositories(excludes = excludes) + +internal_repositories = module_extension( + implementation = _internal_repositories_impl, + doc = """(Internal use only) Wraps the add_default_repositories repository + rule into a bzlmod module extension, excluding repositories that are + already covered by modules, drake_dep_repositories, and crate_universe.""", +) + +def _internal_crate_universe_repositories_impl(module_ctx): + crate_universe_repositories(mirrors = DEFAULT_MIRRORS) + +internal_crate_universe_repositories = module_extension( + implementation = _internal_crate_universe_repositories_impl, + doc = """(Internal use only) Wraps the crate_universe repository rules to + be usable as a bzlmod module extension.""", +) diff --git a/tools/workspace/drake_models/test/parse_test.py b/tools/workspace/drake_models/test/parse_test.py index 42f8a20cf339..8e9b6eb6c320 100644 --- a/tools/workspace/drake_models/test/parse_test.py +++ b/tools/workspace/drake_models/test/parse_test.py @@ -19,7 +19,7 @@ def _runfiles_inventory() -> Iterator[tuple[str, Path]]: manifest = runfiles.Create() inventory = Path(manifest.Rlocation( "drake/tools/workspace/drake_models/inventory.txt")) - repo_name = "drake_models/" + repo_name = "+drake_dep_repositories+drake_models/" for line in inventory.read_text(encoding="utf-8").splitlines(): assert line.startswith(repo_name), line filename = line[len(repo_name):].strip() diff --git a/tools/workspace/java.bzl b/tools/workspace/java.bzl index 061c0bb6e4ed..a06abb0250b5 100644 --- a/tools/workspace/java.bzl +++ b/tools/workspace/java.bzl @@ -30,7 +30,7 @@ package(default_visibility = ["//visibility:public"]) else: is_local = False name = "jar" - actual = "@drake_java_internal_maven_{}//jar".format(repo_ctx.name) + actual = "@{}//jar".format(repo_ctx.attr.maven_name) build_content += "alias(name = {name}, actual = {actual})\n".format( name = repr(name), actual = repr(actual), @@ -60,6 +60,7 @@ _internal_drake_java_import = repository_rule( "licenses": attr.string_list(mandatory = True), "local_os_targets": attr.string_list(mandatory = True), "local_jar": attr.string(mandatory = True), + "maven_name": attr.string(mandatory = True), }, implementation = _impl, ) @@ -79,8 +80,9 @@ def drake_java_import( the jar. Otherwise, the maven_jar will be used. The recognized values for OSs in the list of targets are either "linux" or "osx". """ + maven_name = "drake_java_internal_maven_{}".format(name) java_import_external( - name = "drake_java_internal_maven_{}".format(name), + name = maven_name, licenses = licenses, jar_urls = [ x.format(fulljar = maven_jar) @@ -94,4 +96,5 @@ def drake_java_import( licenses = licenses, local_os_targets = local_os_targets, local_jar = local_jar, + maven_name = maven_name, ) diff --git a/tools/workspace/lcm/package.BUILD.bazel b/tools/workspace/lcm/package.BUILD.bazel index 193a18217c2b..0f25b1cca2dc 100644 --- a/tools/workspace/lcm/package.BUILD.bazel +++ b/tools/workspace/lcm/package.BUILD.bazel @@ -200,41 +200,42 @@ cc_binary( ], ) -# Downstream users of lcm-python expect to say "import lcm". However, in the -# sandbox the python package is located at lcm/lcm-python/lcm/__init__.py to -# match the source tree structure of LCM; without any special help the import -# would fail. +# Downstream users of lcm-python expect to say "import lcm". However, in the +# sandbox the python package is located at lcm-python/lcm/__init__.py to match +# the source tree structure of LCM; without declaring an `imports = [...]` path +# the import would fail. # # Normally we'd add `imports = ["lcm-python"]` to establish a PYTHONPATH at the -# correct subdirectory, and that almost works. However, because the external -# is named "lcm", Bazel's auto-generated empty "lcm/__init__.py" at the root of -# the sandbox is found first, and prevents the lcm-python subdirectory from -# ever being found. +# correct subdirectory, and that almost works -- except that the native code's +# RUNPATH entries are not quite correct. Even though `./_lcm.so` (the glue) +# resolves correctly in the sandbox, it needs to then load the main library +# `liblcm.so` to operate. That happens via its RUNPATH, but because the RUNPATH +# is relative, when the lcm module is loaded from the wrong sys.path entry, the +# RUNPATH no longer works. # -# To repair this, we provide our own init file at the root of the sandbox that -# overrides the Bazel empty default. Its implementation just delegates to the -# lcm-python init file. (Note that this __init__.py shim is neither used nor -# present in the installed copy of Drake; once Drake is installed, the paths -# are standard and there is no aliasing confusion.) +# To repair this, we'll generate our own init file that pre-loads the shared +# library (using python ctypes with the realpath to the shared library) before +# calling the upstream __init__. # -# Relatedly, within the upstream __init__.py there is a `from ._lcm import` -# statement that loads the compiled C code for LCM python support. Even though -# the `./_lcm.so` (the glue) resolves correctly in the sandbox, it needs to -# then load the main library `liblcm.so` to operate. That happens via its -# RUNPATH, but because the RUNPATH is relative, when the lcm module is loaded -# from the wrong sys.path entry, the RUNPATH no longer works. To work around -# that, we pre-load the shared library before calling the upstream __init__, -# using python ctypes with the realpath to the shared library. +# Note that this generated __init__.py shim is neither used nor present in the +# installed copy of Drake; once Drake is installed, the paths are standard and +# there is no aliasing confusion. generate_file( - name = "__init__.py", + name = "gen/lcm/__init__.py", content = """ import ctypes -import os.path -ctypes.cdll.LoadLibrary(os.path.realpath(__path__[0] + '/_lcm.so')) -_filename = __path__[0] + \"/lcm-python/lcm/__init__.py\" -with open(_filename) as f: - _code = compile(f.read(), _filename, 'exec') - exec(_code) +from pathlib import Path +# The base_dir refers to the base of our package.BUILD.bazel. +_base_dir = Path(__path__[0]).resolve().parent.parent +# Load the native code. +ctypes.cdll.LoadLibrary(_base_dir / '_lcm.so') +# We need to tweak the upstream __init__ before we run it. +_filename = _base_dir / 'lcm-python/lcm/__init__.py' +_text = _filename.read_text(encoding='utf-8') +# Respell where the native code comes from. +_text = _text.replace('from lcm import _lcm', 'import _lcm') +_text = _text.replace('from lcm._lcm import', 'from _lcm import') +exec(compile(_text, _filename, 'exec')) """, visibility = ["//visibility:private"], ) @@ -248,7 +249,8 @@ py_library( py_library( name = "lcm-python", - srcs = ["__init__.py"], # Shim, from the genrule above. + srcs = ["gen/lcm/__init__.py"], # Shim, from the genrule above. + imports = ["gen"], deps = [":lcm-python-upstream"], ) diff --git a/tools/workspace/lcm/test/no_lcm_warnings_test.py b/tools/workspace/lcm/test/no_lcm_warnings_test.py index 8067cd13d240..993b32fa61a6 100644 --- a/tools/workspace/lcm/test/no_lcm_warnings_test.py +++ b/tools/workspace/lcm/test/no_lcm_warnings_test.py @@ -1,7 +1,8 @@ +import os import unittest import warnings -from lcm import LCM +from lcm import EventLog, LCM class Test(unittest.TestCase): @@ -14,3 +15,11 @@ def test_publish(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) lcm.publish("TEST_CHANNEL", b"") + + def test_event_log(self): + """ + Ensures no crashes on construction / destruction. + """ + dut = EventLog(path=f"{os.environ['TEST_TMPDIR']}/lcm.log", mode="w") + dut.close() + del dut diff --git a/tools/workspace/pkg_config.BUILD.tpl b/tools/workspace/pkg_config.BUILD.tpl index e30075603e62..7742157a5618 100644 --- a/tools/workspace/pkg_config.BUILD.tpl +++ b/tools/workspace/pkg_config.BUILD.tpl @@ -9,7 +9,7 @@ licenses(%{licenses}) package(default_visibility = ["//visibility:public"]) cc_library( - name = %{name}, + name = %{library_name}, srcs = %{srcs}, hdrs = %{hdrs}, copts = %{copts}, diff --git a/tools/workspace/pkg_config.bzl b/tools/workspace/pkg_config.bzl index 738190f88bd3..3187686f87f3 100644 --- a/tools/workspace/pkg_config.bzl +++ b/tools/workspace/pkg_config.bzl @@ -59,6 +59,12 @@ def setup_pkg_config_repository(repository_ctx): pkg_config_paths.insert(0, "/opt/drake-dependencies/share/pkgconfig") pkg_config_paths.insert(0, "/opt/drake-dependencies/lib/pkgconfig") + # Convert the canonical name (e.g., "+_repo_rules+eigen") to its apparent + # name (e.g., "eigen") so that when a BUILD file uses a label which omits + # the target name (e.g., deps = ["@eigen"]) the unabbreviated label (e.g., + # "@eigen//:eigen") will match what we provide here. + library_name = repository_ctx.name.split("+")[-1] + # Check if we can find the required *.pc file of any version. result = _run_pkg_config(repository_ctx, args, pkg_config_paths) if result.error != None: @@ -73,12 +79,12 @@ def setup_pkg_config_repository(repository_ctx): """ load("@drake//tools/skylark:cc.bzl", "cc_library") cc_library( - name = {name}, + name = {library_name}, srcs = ["pkg_config_failed.cc"], visibility = ["//visibility:public"], ) """.format( - name = repr(repository_ctx.name), + library_name = repr(library_name), ), ) return struct(value = True, error = None) @@ -264,8 +270,8 @@ cc_library( "%{licenses}": repr( getattr(repository_ctx.attr, "licenses", []), ), - "%{name}": repr( - repository_ctx.name, + "%{library_name}": repr( + library_name, ), "%{srcs}": repr( getattr(repository_ctx.attr, "extra_srcs", []), diff --git a/tools/workspace/pybind11/BUILD.bazel b/tools/workspace/pybind11/BUILD.bazel index d94f84327827..f9af2b7e8c5d 100644 --- a/tools/workspace/pybind11/BUILD.bazel +++ b/tools/workspace/pybind11/BUILD.bazel @@ -15,6 +15,14 @@ load( "generate_pybind_documentation_header", ) +# This alias provides a single point of control for defining which pybind11 +# library our tools/skylark/pybind.bzl macro should use. +alias( + name = "pybind11", + actual = "@pybind11", + visibility = ["//visibility:public"], +) + exports_files( [ "pybind11-config.cmake", diff --git a/tools/workspace/workspace_bzlmod_sync_test.py b/tools/workspace/workspace_bzlmod_sync_test.py index 3313c44284da..9bd7443e8cb9 100644 --- a/tools/workspace/workspace_bzlmod_sync_test.py +++ b/tools/workspace/workspace_bzlmod_sync_test.py @@ -12,10 +12,11 @@ def _read(self, respath): path = Path(manifest.Rlocation(respath)) return path.read_text(encoding="utf-8") - def _parse_modules(self, content): - """Given the contents of MODULE.bazel, returns a dictionary mapping - from module_name to module_version. + def _parse_modules(self): + """Parses MODULE.bazel to return a dictionary mapping from module_name + to module_version. """ + content = self._read(f"drake/MODULE.bazel") result = {} for line in content.splitlines(): # Only match bazel_dep lines. @@ -32,10 +33,12 @@ def _parse_modules(self, content): result[kwargs["name"]] = kwargs["version"] return result - def _parse_repo_rule_version(self, content): - """Given the contents of a repository.bzl that calls 'github_archive', - returns the version number it pins to. + def _parse_repo_rule_version(self, repo_name): + """Parses tools/workspace/{repo_name}/repository.bzl to find the call + to 'github_archive' and returns the version number it pins to. """ + content = self._read( + f"drake/tools/workspace/{repo_name}/repository.bzl") assert "github_archive" in content, content for line in content.splitlines(): line = line.strip() @@ -61,7 +64,7 @@ def test_version_sync(self): and WORKSPACE. This test ensures that the versions pinned in each file are correctly synchronized. """ - modules = self._parse_modules(self._read(f"drake/MODULE.bazel")) + modules = self._parse_modules() # Don't check modules that are known to be module-only. del modules["bazel_features"] @@ -73,22 +76,22 @@ def test_version_sync(self): self.assertTrue(modules) for module_name, module_version in modules.items(): repo_name = self._module_name_to_repo_name(module_name) - workspace_version = self._parse_repo_rule_version(self._read( - f"drake/tools/workspace/{repo_name}/repository.bzl")) + workspace_version = self._parse_repo_rule_version(repo_name) self.assertEqual(workspace_version, module_version) - def _parse_workspace_already_provided(self, content): - """Given the contents of default.bzl, returns the list of - REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES. + def _parse_workspace_list_constant(self, name): + """Returns the contents of the list constant named `name` in our + tools/workspace/default.bzl. """ + content = self._read("drake/tools/workspace/default.bzl") result = None for line in content.splitlines(): line = line.strip() - if line == "REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [": + if line == f"{name} = [": result = list() continue if result is None: - # We haven't seen the REPOS_ALREADY_... line yet. + # We haven't seen the opening line yet. continue if line == "]": break @@ -103,18 +106,49 @@ def test_default_exclude_sync(self): provided by MODULE.bazel. This test ensures that the list is correctly synchronized. """ - modules = self._parse_modules(self._read(f"drake/MODULE.bazel")) + modules = self._parse_modules() # These workspace-only repositories are irrelevant for bzlmod. modules["rust_toolchain"] = None - # Check that default.bzl's constant matches the inventory of modules. - repo_names = sorted([ + repo_names_in_module = sorted([ self._module_name_to_repo_name(module_name) for module_name in modules.keys() ]) - self.assertEqual(repo_names, self._parse_workspace_already_provided( - self._read("drake/tools/workspace/default.bzl"))) + repo_names_in_default = self._parse_workspace_list_constant( + name="REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES") + self.assertEqual(repo_names_in_module, repo_names_in_default) + + def _parse_module_drake_dep_repositories(self): + """Parses MODULE.bazel to return the list of drake_dep_repositories. + """ + content = self._read(f"drake/MODULE.bazel") + result = None + for line in content.splitlines(): + line = line.strip() + if line == "drake_dep_repositories,": + result = list() + continue + if result is None: + # We haven't seen the opening line yet. + continue + if line == ")": + break + assert line.startswith('"'), line + assert line.endswith('",'), line + result.append(line[1:-2]) + assert result, content + return sorted(result) + + def test_default_exported_sync(self): + """Our default.bzl has a list of REPOS_EXPORTED that must match the + drake_dep_repositories listed in MODULE.bazel. This test ensures that + the lists are correctly synchronized. + """ + repo_names_in_module = self._parse_module_drake_dep_repositories() + repo_names_in_default = self._parse_workspace_list_constant( + name="REPOS_EXPORTED") + self.assertEqual(repo_names_in_module, repo_names_in_default) assert __name__ == '__main__'