From 99f8aea4478752eaab9904c1c0238a44688e68de Mon Sep 17 00:00:00 2001 From: Jeremy Nimmer Date: Mon, 25 Nov 2024 07:21:30 -0800 Subject: [PATCH] [setup] Rewrite Ubuntu install_prereqs from the ground up The existing from-source setup scripts are deprecated and will be removed after 2025-06-01: - setup/ubuntu/install_prereqs.sh - setup/ubuntu/source_distribution/install_prereqs_user_environment.sh Instead, users and developers should run setup/install_prereqs (not as root). Usability changes: - Fine-tuned log output, with option for --verbose (e.g., in CI). - Skip unnecessary steps; incremental runs are typically sub-second. Maintainability changes: - Always use files, not heredocs. (The pkg-config etc in a heredoc were documented as unwanted when using Drake's Debian packages, but that was wrong; we always want those deps even in Debian packages.) Website changes: - Major updates to from_source that explain how to install from CMake. A few cleanups macOS come along for the ride: - Remove useless binary_distribution_called_update shenanigans. - Remove vestigial i386 rcfile. - Abandon all hope of supporting 'brew install drake'. --- doc/_pages/bazel.md | 126 ++-- doc/_pages/from_source.md | 171 +++-- setup/BUILD.bazel | 47 +- setup/install_prereqs | 16 + .../binary_distribution/install_prereqs.sh | 6 - setup/mac/install_prereqs.sh | 1 - .../source_distribution/install_prereqs.sh | 9 - .../binary_distribution/install_prereqs.sh | 82 --- setup/ubuntu/install_prereqs | 648 ++++++++++++++++++ setup/ubuntu/install_prereqs.sh | 105 +-- setup/ubuntu/packages-bazelisk.json | 18 + ...es-jammy.txt => packages-jammy-binary.txt} | 3 + ...ges-jammy.txt => packages-jammy-build.txt} | 1 + .../packages-jammy-clang.txt | 0 ...-only.txt => packages-jammy-developer.txt} | 0 ...my-doc-only.txt => packages-jammy-doc.txt} | 0 ...only.txt => packages-jammy-maintainer.txt} | 2 + ...es-noble.txt => packages-noble-binary.txt} | 5 +- ...ges-noble.txt => packages-noble-build.txt} | 1 + .../packages-noble-clang.txt | 0 ...-only.txt => packages-noble-developer.txt} | 0 ...le-doc-only.txt => packages-noble-doc.txt} | 0 ...only.txt => packages-noble-maintainer.txt} | 2 + .../source_distribution/install_bazelisk.sh | 63 -- .../source_distribution/install_prereqs.sh | 148 ---- .../install_prereqs_user_environment.sh | 24 +- tools/macos-arch-i386.bazelrc | 7 - tools/workspace/bazelisk/repository.bzl | 2 +- 28 files changed, 880 insertions(+), 607 deletions(-) create mode 100755 setup/install_prereqs mode change 100755 => 100644 setup/mac/source_distribution/install_prereqs.sh delete mode 100755 setup/ubuntu/binary_distribution/install_prereqs.sh create mode 100755 setup/ubuntu/install_prereqs create mode 100644 setup/ubuntu/packages-bazelisk.json rename setup/ubuntu/{binary_distribution/packages-jammy.txt => packages-jammy-binary.txt} (91%) rename setup/ubuntu/{source_distribution/packages-jammy.txt => packages-jammy-build.txt} (96%) rename setup/ubuntu/{source_distribution => }/packages-jammy-clang.txt (100%) rename setup/ubuntu/{source_distribution/packages-jammy-test-only.txt => packages-jammy-developer.txt} (100%) rename setup/ubuntu/{source_distribution/packages-jammy-doc-only.txt => packages-jammy-doc.txt} (100%) rename setup/ubuntu/{source_distribution/packages-jammy-maintainer-only.txt => packages-jammy-maintainer.txt} (81%) rename setup/ubuntu/{binary_distribution/packages-noble.txt => packages-noble-binary.txt} (91%) rename setup/ubuntu/{source_distribution/packages-noble.txt => packages-noble-build.txt} (96%) rename setup/ubuntu/{source_distribution => }/packages-noble-clang.txt (100%) rename setup/ubuntu/{source_distribution/packages-noble-test-only.txt => packages-noble-developer.txt} (100%) rename setup/ubuntu/{source_distribution/packages-noble-doc-only.txt => packages-noble-doc.txt} (100%) rename setup/ubuntu/{source_distribution/packages-noble-maintainer-only.txt => packages-noble-maintainer.txt} (81%) delete mode 100755 setup/ubuntu/source_distribution/install_bazelisk.sh delete mode 100755 setup/ubuntu/source_distribution/install_prereqs.sh delete mode 100644 tools/macos-arch-i386.bazelrc diff --git a/doc/_pages/bazel.md b/doc/_pages/bazel.md index f896c5fcf628..944e367afb43 100644 --- a/doc/_pages/bazel.md +++ b/doc/_pages/bazel.md @@ -1,42 +1,79 @@ --- -title: Bazel build system +title: Developing Drake using Bazel --- -Drake's primary build system is Bazel. For more information about Bazel, see -[https://bazel.build/](https://bazel.build/). +# New Users -Drake also offers a CMake build system wrapper that invokes Bazel under the -hood. +For first-time users, we strongly suggest using one of the pre-compiled binaries +described on our [installation](/installation.html) page. -# Bazel Installation +Or, if you need to install Drake from source please refer to the +[CMake instructions](from_source.html). Bazel is only used for development; +there is no way to install Drake from Bazel. -Follow Drake's -[platform-specific setup instructions](/from_source.html#mandatory-platform-specific-instructions) -to install bazelisk at ``/usr/bin/bazel``, which will then automatically -download the correct version of Bazel necessary for the build. +This page describes how Drake Developers (and contributors making pull requests) +should develop and test their changes locally. -# Drake clone and platform setup +# Introduction -* Start with a **git clone** of drake, per the [Getting Drake](/from_source.html#getting-drake) - instructions. -* Continue with the *"Mandatory platform-specific instructions"* on the same - page. +Drake's primary build system is [Bazel](https://bazel.build/), which our +developers use to build and test locally and in CI. Bazel enables speedy +development via fine-grained caching of all actions, including tests. + +For end users, Drake also offers a CMake build system wrapper that invokes Bazel +as a subprocess and maps CMake build options into Bazel build options, so that +users can install Drake via standard tools without knowing anything about Bazel. +See the [CMake instructions](from_source.html) for details. + +# Getting Drake + +Run: + +```bash +# Get the sources. +git clone --filter=blob:none https://github.com/RobotLocomotion/drake.git + +# Install the developer dependencies. +drake/setup/install_prereqs --developer +``` + +We suggest you keep the default clone directory name (``drake``) and not rename +it (e.g., ``drake2``). The CLion integration will suffer if the checkout is not +named ``drake``. (See [CLion IDE setup](clion.html) for details.) + +## Using a fork of Drake + +The above ``git clone`` command will configure Drake's primary repository as a +remote called ``origin``. If you plan to fork Drake for development, we +recommend that you configure your fork of Drake's primary repository as the +``origin`` remote and Drake's primary repository as the ``upstream`` +remote. This can be done by executing the following commands: + +```bash +cd drake +git remote set-url origin git@github.com:[your github user name]/drake.git +git remote add upstream https://github.com/RobotLocomotion/drake.git +git remote set-url --push upstream no_push +``` + +We recommend that you +[setup SSH access to github.com](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/) +to avoid needing to type your password each time you access it. # Developing Drake using Bazel -To build or test Drake, run **bazel build** or **bazel test** with the desired -target label (and optional configuration options if desired). We give some -typical examples below; for more reading about target patterns, see: -[https://docs.bazel.build/versions/main/user-manual.html#target-patterns](https://docs.bazel.build/versions/main/user-manual.html#target-patterns). +To build or test Drake, run `bazel build` or `bazel test` with the desired +target label (and optional configuration options if desired). We give some +typical examples below; for more reading about target patterns, see +[target patterns](https://bazel.build/docs/user-manual#target-patterns). -On Ubuntu, the default compiler is the first ``gcc`` compiler in the ``PATH``. -On macOS, the default compiler is the Apple LLVM compiler. To use Clang on -Ubuntu, add ``--config=clang`` after any **bazel build**, **bazel test** or any -other **bazel** commands. +On Ubuntu, the default compiler is GCC. On macOS, the default compiler is the +Apple LLVM compiler. To use Clang on Ubuntu, add ``--config=clang`` after any +`bazel` command. Cheat sheet for operating on the entire project: -``` +```bash cd /path/to/drake bazel build //... # Build the entire project. bazel test //... # Build and test the entire project. @@ -46,17 +83,12 @@ bazel test --config=clang //... # Build and test using Clang on Ubuntu. ``` * The "``//``" means "starting from the root of the project". -* The "``...``" means "everything including the subdirectories' ``BUILD`` files". - * Contrast with, e.g., the "``bazel build common:*``" explained below, where - only targets declared *directly* in ``drake/common/BUILD`` are compiled, - and not the targets in ``drake/common/trajectories/BUILD``. The "``*``" - matches targets in that directory; the "``...``" also matches down into - subdirectories. +* The "``...``" means "everything, recursing into subdirectories". You may use relative pathnames if your shell's working directory is not at the project root: -``` +```bash cd /path/to/drake/common bazel build ... # Build everything in common and its child subdirectories. bazel test ... # Test everything in common and its child subdirectories. @@ -64,7 +96,7 @@ bazel build //... # Build the entire project. bazel test //... # Build and test the entire project. ``` -* As before, the "``...``" above means "everything including subdirectories". +* As before, the "``...``" means "everything, recursing into subdirectories". * In the first two lines we did not precede "``...``" with "``//``", so the search begins in the current directory (``common``) and not from the ``drake`` root. @@ -74,17 +106,17 @@ bazel test //... # Build and test the entire project. Cheat sheet for operating on specific portions of the project: -``` +```bash cd /path/to/drake bazel build common/... # Build everything in common and its child subdirectories. bazel build common # Build libcommon. bazel build common:polynomial # Build libpolynomial. -bazel build common:* # Build everything in common but NOT its children. +bazel build common:all # Build everything in common but NOT its children. bazel test common:polynomial_test # Run one test. bazel test -c dbg common:polynomial_test # Run one test in debug mode. bazel test --config=memcheck common:polynomial_test # Run one test under memcheck (valgrind). -bazel test --config=fastmemcheck common:* # Run common's tests under memcheck, with minimal recompiling. +bazel test --config=fastmemcheck common:all # Run common's tests under memcheck, with minimal recompiling. bazel test --config=kcov common:polynomial_test # Run one test under kcov (see instructions below). bazel build -c dbg common:polynomial_test && \ gdb bazel-bin/common/polynomial_test # Run one test under gdb. @@ -96,11 +128,10 @@ bazel test --config lint //... # Only run style checks; do * The "``:``" syntax separates target names from the directory path of the ``BUILD`` file they appear in. In this case, for example, - ``drake/common/BUILD`` specifies ``cc_test(name = "polynomial_test")``. + ``drake/common/BUILD`` specifies ``drake_cc_test(name = "polynomial_test")``. * Note that the configuration switches (``-c`` and ``--config``) influence the entire command. For example, running a test in ``dbg`` mode means that its prerequisite libraries are also compiled and linked in ``dbg`` mode. -* For the definitions of the "``--config``" options see ``drake/tools/bazel.rc``. ## Running with Flags @@ -109,7 +140,7 @@ bazel test --config lint //... # Only run style checks; do In general, to figure out what binary-specific arguments are available, add "``-- --help``" to your ``bazel run`` command. An an example, -``` +```bash bazel run //examples/acrobot:run_passive -- --help ``` @@ -122,31 +153,34 @@ For running tests, you may pass custom arguments to the test program via For a C++ unittest that uses ``drake_cc_googletest``, for example: -``` -bazel test multibody/plant:multibody_plant_test --test_output=streamed --nocache_test_results --test_arg=--gtest_filter='*SimpleModelCreation*' +```bash +bazel test //multibody/plant:multibody_plant_test --test_output=streamed --nocache_test_results --test_arg=--gtest_filter='*SimpleModelCreation*' ``` For a Python unittest that uses ``drake_py_unittest``, for example: -``` -bazel test bindings/pydrake:py/symbolic_test --test_output=streamed --nocache_test_results --test_arg=--trace=user --test_arg=TestSymbolicVariable +```bash +bazel test //bindings/pydrake:py/symbolic_test --test_output=streamed --nocache_test_results --test_arg=--trace=user --test_arg=TestSymbolicVariable ``` # Updating BUILD files -Please use the "``buildifier``" tool to format edits to ``BUILD`` files (in the -same spirit as ``clang-format`` formatting C++ code): +We use the "``buildifier``" tool to auto-format our ``BUILD`` files (in the same +spirit as ``clang-format`` formatting C++ code): -``` +```bash cd /path/to/drake bazel-bin/tools/lint/buildifier --all # Reformat all Bazel files. bazel-bin/tools/lint/buildifier common/BUILD # Only reformat one file. ``` +If a BUILD file is mis-formatted, you we see a test failure with instructions +how to fix the problem. + In most cases the ``bazel-bin/tools/lint/buildifier`` will already be compiled by the time you need it. In case it's absent, you can compile it via: -``` +```bash cd /path/to/drake bazel build //tools/lint:buildifier ``` diff --git a/doc/_pages/from_source.md b/doc/_pages/from_source.md index 41f617c71069..57b7be62195d 100644 --- a/doc/_pages/from_source.md +++ b/doc/_pages/from_source.md @@ -2,10 +2,16 @@ title: Source Installation --- +# New Users + +For first-time users, we strongly suggest using one of the pre-compiled binaries +described on our [installation](/installation.html) page. This page explains how +to build Drake form source, which is somewhat more challenging. + # Supported Configurations The following table shows the configurations and platforms that Drake -officially supports: +officially supports when building from source: @@ -39,113 +45,93 @@ setup steps. ⁽³⁾ Drake requires a compiler running in C++20 (or greater) mode. -# Getting Drake - -Run: +# Building with CMake -``` -git clone --filter=blob:none https://github.com/RobotLocomotion/drake.git -``` +For sample projects that show how to import Drake as a CMake external project +(either by building Drake from source, or by downloading a pre-compiled Drake +release) please see our gallery of +[drake-external-examples](https://github.com/RobotLocomotion/drake-external-examples). -Note: we suggest you keep the default clone directory name (``drake``) and not -rename it (such as ``drake2``). The CLion integration will suffer if the -checkout directory is not named ``drake``. (See [CLion IDE setup](clion.html) for details.) +Otherwise, you can run a build from source by hand like this: -Note: the build process may encounter problems if you have unusual characters -like parentheses in the absolute path to the drake directory -(see [#394](https://github.com/RobotLocomotion/drake/issues/394)). - -## Using a fork of Drake - -The above ``git clone`` command will configure Drake's primary repository as a -remote called ``origin``. If you plan to fork Drake for development, we -recommend that you configure your fork of Drake's primary repository as the -``origin`` remote and Drake's primary repository as the ``upstream`` -remote. This can be done by executing the following commands: - -``` -cd drake -git remote set-url origin git@github.com:[your github user name]/drake.git -git remote add upstream https://github.com/RobotLocomotion/drake.git -git remote set-url --push upstream no_push -``` - -We recommend that you -[setup SSH access to github.com](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/) -to avoid needing to type your password each time you access it. - -# Mandatory platform-specific instructions - -Before running the build, you must follow some one-time platform-specific -setup steps. - -*Ubuntu:* - -``` -sudo ./setup/ubuntu/install_prereqs.sh -``` - -*macOS:* - -We assume that you have already installed Xcode -([from the Mac App Store](https://itunes.apple.com/us/app/xcode/id497799835)). - -After that, run: - -``` -./setup/mac/install_prereqs.sh -``` - -# Build with Bazel - -For instructions, jump to -[Developing Drake using Bazel](/bazel.html#developing-drake-using-bazel), -or check out the full details at: - -* [Bazel build system](/bazel.html) - -## Building the Python Bindings +```bash +# Get the sources. +git clone --filter=blob:none https://github.com/RobotLocomotion/drake.git -To use the Python bindings from Drake externally, we recommend using CMake. -As an example: +# Install the build dependencies. +drake/setup/install_prereqs -```bash -git clone https://github.com/RobotLocomotion/drake.git +# Build and install using standard CMake commands. mkdir drake-build cd drake-build cmake ../drake make install ``` -Note that a concurrency limit passed to `make` (e.g., `make -j 2`) has almost no -effect on the Drake build. You might need to add a bazel configuration dotfile +To change the build options, you can run one of the standard CMake GUIs (e.g., +`ccmake` or `cmake-gui`) or specify command-line options with `-D` to `cmake`. + +## CMake options which are Drake-specific + +These options can be set using `-DFOO=bar` on the CMake command line, or in one +of the CMake GUIs. + +Adjusting open-source dependencies: + +* WITH_USER_EIGEN (default OFF). When ON, uses `find_package(Eigen3)` + to locate a user-provided `Eigen3::Eigen` library + instead of hard-coding to the operating system version. +* WITH_USER_FMT (default OFF). When ON, uses `find_package(fmt)` + to locate a user-provided `fmt::fmt` library + instead of hard-coding to the operating system version. +* WITH_USER_SPDLOG (default OFF). When ON, uses `find_package(spdlog)` + to locate a user-provided `spdlog::spdlog` library + instead of hard-coding to the operating system version. + +Adjusting closed-source (commercial) software dependencies: + +* WITH_GUROBI (default OFF). When ON, enables the `GurobiSolver` in the build. + * When enabled, you must download and install Gurobi 10.0 yourself prior to + running Drake's CMake configure script; Drake does not automatically + download Gurobi. If Gurobi is not installed to its standard location, you + must also specify `export GUROBI_HOME=${...GUROBI_UNZIP_PATH...}/linux64` + in your terminal so that `find_package(Gurobi)` will be able to find it. +* WITH_MOSEK (default OFF). When ON, enables the `MosekSolver` in the build. + * When enabled, Drake automatically downloads the MOSEK™ software from + `mosek.com` and installs it as part of the Drake build. The selected + version is hard-coded in Drake and cannot be configured. +* WITH_SNOPT (default OFF). When ON, enables the `SnoptSolver` in the build. + * This option is mutally exclusive with `WITH_ROBOTLOMOTION_SNOPT`. +* SNOPT_PATH (no default). When `WITH_SNOPT` is ON, this must be set to a SNOPT + source code archive path (e.g., `/home/user/Downloads/snopt7.4.tar.gz`) with + SNOPT version 7.4 (recommended) or version 7.6. + * Drake does not support using a SNOPT binary release (i.e., shared library); + it requires a source archive (i.e., the Fortran code). +* WITH_ROBOTLOMOTION_SNOPT (default OFF). When ON, enables the `SnoptSolver` + in the build, using a hard-coded and access-controlled download of SNOPT. + This option is only valid for MIT- or TRI-affiliated Drake developers. + * This option is mutally exclusive with `WITH_SNOPT`. + +## CMake caveats + +Note that a concurrency limit passed to `make` (e.g., `make -j 2`) for a Drake +build has almost no effect. You might need to add a bazel configuration dotfile to your home directory if your build is running out of memory. See the [troubleshooting](/troubleshooting.html#build-oom) page for details. Be aware that repeatedly running `make install` will install the recompiled version of Drake *on top of* the prior version. This will lead to disaster unless the set of installed filenames is exactly the same (because old files -will be hanging around polluting your PYTHONPATH). It is safe if you are merely -tweaking a source code file and repeatedly installing, without any changes to -the build system. For any kind of larger change (e.g., upgrading to a newer -Drake), we strongly advise that you delete the prior tree (within the `install` -sub-directory) before running `make`. - -Please note the additional CMake options which affect the Python bindings: - -* ``-DWITH_GUROBI={ON, [OFF]}`` - Build with Gurobi enabled. -* ``-DWITH_MOSEK={ON, [OFF]}`` - Build with MOSEK™ enabled. -* ``-DWITH_SNOPT={ON, [OFF]}`` - Build with SNOPT enabled. - -``{...}`` means a list of options, and the option surrounded by ``[...]`` is -the default option. An example of building ``pydrake`` with both Gurobi and -MOSEK™, without building tests: +will be hanging around, e.g., polluting your PYTHONPATH). It is safe if you are +merely tweaking a source code file and repeatedly installing, without any +changes to the build system. For any kind of larger change (e.g., upgrading to a +newer Drake), we strongly advise that you delete the prior tree (within the +`install` sub-directory) before running `make`. -```bash -cmake -DWITH_GUROBI=ON -DWITH_MOSEK=ON ../drake -``` +## Running the Python Bindings after a CMake install -You will also need to have your ``PYTHONPATH`` configured correctly. +To run the installed copy of `pydrake`, you will also need to have your +``PYTHONPATH`` configured correctly. *Ubuntu 22.04 (Jammy):* @@ -167,3 +153,10 @@ export PYTHONPATH=${PWD}/install/lib/python3.12/site-packages:${PYTHONPATH} cd drake-build export PYTHONPATH=${PWD}/install/lib/python3.12/site-packages:${PYTHONPATH} ``` + +# Making changes to Drake + +Drake developers use Bazel (not CMake) for development. Refer to our [Bazel +instructions](/bazel.html) for details. + +Bazel is only used for development; there is no way to install Drake from Bazel. diff --git a/setup/BUILD.bazel b/setup/BUILD.bazel index 9a4875c840ba..259526055a08 100644 --- a/setup/BUILD.bazel +++ b/setup/BUILD.bazel @@ -3,30 +3,13 @@ load("//tools/lint:lint.bzl", "add_lint_tests") package(default_visibility = ["//visibility:public"]) -# Make our package requirements (but not manual scripts) available to -# downstream projects. -exports_files([ - "mac/binary_distribution/Brewfile", - "mac/binary_distribution/requirements.txt", - "mac/source_distribution/Brewfile", - "mac/source_distribution/Brewfile-doc-only", - "mac/source_distribution/Brewfile-maintainer-only", - "mac/source_distribution/requirements.txt", - "mac/source_distribution/requirements-maintainer-only.txt", - "mac/source_distribution/requirements-test-only.txt", - "ubuntu/binary_distribution/packages-jammy.txt", - "ubuntu/source_distribution/packages-jammy.txt", - "ubuntu/source_distribution/packages-jammy-clang.txt", - "ubuntu/source_distribution/packages-jammy-doc-only.txt", - "ubuntu/source_distribution/packages-jammy-maintainer-only.txt", - "ubuntu/source_distribution/packages-jammy-test-only.txt", - "ubuntu/binary_distribution/packages-noble.txt", - "ubuntu/source_distribution/packages-noble.txt", - "ubuntu/source_distribution/packages-noble-clang.txt", - "ubuntu/source_distribution/packages-noble-doc-only.txt", - "ubuntu/source_distribution/packages-noble-maintainer-only.txt", - "ubuntu/source_distribution/packages-noble-test-only.txt", -]) +# Make our prerequisites data (but not scripts) available to downstream +# projects. +exports_files(glob([ + "**/Brewfile*", + "**/*.json", + "**/*.txt", +])) filegroup( name = "deepnote", @@ -49,19 +32,25 @@ install_files( "//tools/cc_toolchain:linux": [ "deepnote/install_nginx", "deepnote/nginx-meshcat-proxy.conf", - "ubuntu/binary_distribution/install_prereqs.sh", - "ubuntu/binary_distribution/packages-jammy.txt", - "ubuntu/binary_distribution/packages-noble.txt", + "ubuntu/install_prereqs", + "ubuntu/packages-jammy-binary.txt", + "ubuntu/packages-noble-binary.txt", ], "//conditions:default": [], }), strip_prefix = [ "mac/binary_distribution", - "ubuntu/binary_distribution", + "ubuntu", ], rename = { + # TODO(jwnimmer-tri) This is only for macOS; once we rewrite macOS to + # also not be a bash script, we can remove this. "share/drake/setup/install_prereqs.sh": "install_prereqs", }, ) -add_lint_tests() +add_lint_tests( + python_lint_extra_srcs = [ + "ubuntu/install_prereqs", + ], +) diff --git a/setup/install_prereqs b/setup/install_prereqs new file mode 100755 index 000000000000..d0a54d2886c7 --- /dev/null +++ b/setup/install_prereqs @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +case "$OSTYPE" in + darwin*) + exec "${BASH_SOURCE%/*}/mac/install_prereqs" "$@" + ;; + linux*) + exec "${BASH_SOURCE%/*}/ubuntu/install_prereqs" "$@" + ;; + *) + echo 'ERROR: Unsupported OS' >&2 + exit 1 + ;; +esac diff --git a/setup/mac/binary_distribution/install_prereqs.sh b/setup/mac/binary_distribution/install_prereqs.sh index d01e0eb5dace..2a90dcdf4f9c 100755 --- a/setup/mac/binary_distribution/install_prereqs.sh +++ b/setup/mac/binary_distribution/install_prereqs.sh @@ -53,16 +53,10 @@ export HOMEBREW_NO_AUTO_UPDATE=1 # completes or wait for the next automatic cleanup if necessary. export HOMEBREW_NO_INSTALL_CLEANUP=1 -binary_distribution_called_update=0 - if [[ "${with_update}" -eq 1 ]]; then # Note that brew update uses git, so HOMEBREW_CURL_RETRIES does not take # effect. brew update || (sleep 30; brew update) - - # Do NOT call brew update again when installing prerequisites for source - # distributions. - binary_distribution_called_update=1 fi brew bundle --file="${BASH_SOURCE%/*}/Brewfile" --no-lock diff --git a/setup/mac/install_prereqs.sh b/setup/mac/install_prereqs.sh index 8827b92d12e1..f5290225cb41 100755 --- a/setup/mac/install_prereqs.sh +++ b/setup/mac/install_prereqs.sh @@ -19,7 +19,6 @@ while [ "${1:-}" != "" ]; do # Do NOT call brew update during execution of this script. --without-update) binary_distribution_args+=(--without-update) - source_distribution_args+=(--without-update) ;; *) echo 'Invalid command line argument' >&2 diff --git a/setup/mac/source_distribution/install_prereqs.sh b/setup/mac/source_distribution/install_prereqs.sh old mode 100755 new mode 100644 index dfc8d410861b..1d5bf93f9f84 --- a/setup/mac/source_distribution/install_prereqs.sh +++ b/setup/mac/source_distribution/install_prereqs.sh @@ -8,7 +8,6 @@ set -euxo pipefail with_test_only=1 -with_update=1 while [ "${1:-}" != "" ]; do case "$1" in @@ -18,10 +17,6 @@ while [ "${1:-}" != "" ]; do --without-test-only) with_test_only=0 ;; - # Do NOT call brew update during execution of this script. - --without-update) - with_update=0 - ;; *) echo 'Invalid command line argument' >&2 exit 5 @@ -39,10 +34,6 @@ if ! command -v brew &>/dev/null; then exit 4 fi -if [[ "${with_update}" -eq 1 && "${binary_distribution_called_update:-0}" -ne 1 ]]; then - brew update || (sleep 30; brew update) -fi - brew bundle --file="${BASH_SOURCE%/*}/Brewfile" --no-lock if [[ "${with_test_only}" -eq 1 ]]; then diff --git a/setup/ubuntu/binary_distribution/install_prereqs.sh b/setup/ubuntu/binary_distribution/install_prereqs.sh deleted file mode 100755 index 58aaeee82be0..000000000000 --- a/setup/ubuntu/binary_distribution/install_prereqs.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# -# Install development and runtime prerequisites for binary distributions of -# Drake on Ubuntu. - -set -euo pipefail - -with_update=1 -with_asking=1 - -while [ "${1:-}" != "" ]; do - case "$1" in - # Do NOT call apt-get update during execution of this script. - --without-update) - with_update=0 - ;; - # Pass -y along to apt-get. - -y) - with_asking=0 - ;; - *) - echo 'Invalid command line argument' >&2 - exit 3 - esac - shift -done - -if [[ "${EUID}" -ne 0 ]]; then - echo 'ERROR: This script must be run as root' >&2 - exit 1 -fi - -if [[ "${with_asking}" -eq 0 ]]; then - maybe_yes='-y' -else - maybe_yes='' -fi - - -if command -v conda &>/dev/null; then - echo 'NOTE: Drake is not tested regularly with Anaconda, so you may experience compatibility hiccups; when asking for help, be sure to mention that Conda is involved.' >&2 -fi - -binary_distribution_called_update=0 - -if [[ "${with_update}" -eq 1 ]]; then - apt-get update || (sleep 30; apt-get update) || (cat <&2 - exit 2 -fi - -apt-get install ${maybe_yes} --no-install-recommends $(cat < /dev/null; then + echo "Installing python3 ..." + if [[ "${EUID}" -ne 0 ]]; then + SUDO=sudo + else + SUDO= + fi + if ! ${SUDO} apt-get -y install python3; then + ${SUDO} apt-get update || (sleep 30; apt-get update) + ${SUDO} apt-get -y install python3 + fi +fi +exec python3 -B $0 "$@" +""" + +__doc__ = """ +Drake setup program to install necessary prerequisites. +""" + +import argparse +import enum +import functools +import json +import hashlib +import logging +import os +import pathlib +import shlex +import shutil +import sys +import subprocess +import tempfile +import textwrap +import time +import urllib.parse +import urllib.request + + +# Our text assets are located in the same directory as this script. +_MY_DIR = pathlib.Path(__file__).parent.resolve() + + +class Flavor(enum.Enum): + """Drake users (and developers) can choose a particular flavor of + prerequisites to install, depending on which operations they need to + accomplish.""" + + # These are listed in order from narrowest to widest prerequisites. + # Each flavor adds more prerequisites atop the prior flavor(s). + BINARY = "to run a precompiled binary release" + BUILD = "to build and install using CMake and GCC" + CLANG = "to build and install using Clang" + DEVELOPER = "to build and test with Bazel" + DOC = "to generate the website" + MAINTAINER = "to perform scheduled maintainence and releases" + + def __init__(self, description): + self.description = description + + def __ge__(self, other): + everything = list(Flavor) + lhs = everything.index(self) + rhs = everything.index(other) + return lhs >= rhs + + +def _warn(message: str) -> None: + """Logs a warning. For convenience, lines are unwrapped.""" + message = textwrap.dedent(message).replace("\n", " ").strip() + logging.warning(message) + + +def _error(message: str) -> None: + """Logs an error and exits. For convenience, lines are unwrapped.""" + message = textwrap.dedent(message).replace("\n", " ").strip() + logging.error(message) + sys.exit(1) + + +def _maybe_warn_conda() -> None: + """Warns if conda is on the $PATH.""" + if shutil.which("conda") is None: + return + _warn( + """Conda was detected on your $PATH. Drake is not tested regularly with + Anaconda, so you may experience compatibility hiccups; when asking for + help, be sure to mention that Conda is involved.""" + ) + + +@functools.cache +def _check_sudo() -> None: + """Checks that 'sudo' has sufficient credentials.""" + subprocess.check_call(["sudo", "-v"]) + + +def _run( + *, + args: list, + cwd: str = None, + check: bool = True, + superuser: bool = False, + flaky: bool = False, + quiet: bool = False, + interactive: bool = False, +) -> None: + """Runs a subprocess command given by `args`. When `check` is true, failure + of the command is an `_error`. When `superuser` is true, the command will + be run under 'sudo' unless the euid is already root. When `flaky` is true, + the command will be retried a couple times when it fails. When `quiet` is + true, the command line will not be printed by default. When `interactive` + is true, input is allowed and output is unbuffered.""" + command = args[0] + if superuser and os.geteuid() != 0: + _check_sudo() + args = ["sudo"] + args + logging.log( + msg=f"Running: {shlex.join(args)} ...", + level=logging.DEBUG if quiet else logging.INFO, + ) + num_attempts = 3 if flaky else 1 + for i in range(num_attempts): + if i > 0: + logging.info("... failed; waiting 30 sec before trying again ...") + time.sleep(30) + process = subprocess.run( + args, + cwd=cwd, + stdin=subprocess.DEVNULL if not interactive else None, + stdout=subprocess.PIPE if not interactive else None, + stderr=subprocess.STDOUT if not interactive else None, + text=True, + ) + problem = check and (process.returncode != 0) + if not problem: + break + if process.stdout is not None: + for line in process.stdout.splitlines(): + logging.log( + msg=f"... from {command}: {line}", + level=logging.INFO if problem else logging.DEBUG, + ) + logging.debug(f"... finished {command}.") + if problem: + _error(f"{command} failed with returncode {process.returncode}") + return process + + +def _check_lsb_release(): + """Returns the lsb_release details as a dict. If necessary, installs + lsb-release before calling it. The dict keys will typically be: + - 'Distributor ID' (example value 'Ubuntu') + - 'Description' (example value 'Ubuntu 12.34.5 LTS') + - 'Release' (example value '12.34') + - 'Codename' (example value 'foobar') + The 'Codename' is always guaranteed to exist, but the rest are optional.""" + if shutil.which("lsb_release") is None: + _run( + args=[ + "apt-get", + "install", + "--no-install-recommends", + "--yes", + "lsb-release", + ], + superuser=True, + ) + else: + logging.debug("lsb-release is already installed") + result = dict() + process = _run(args=["lsb_release", "-idrc"], quiet=True) + for line in process.stdout.splitlines(): + tokens = line.split(":", maxsplit=1) + if len(tokens) == 2: + key, value = tokens + result[key.strip()] = value.strip() + logging.debug(f"lsb_release = {result}") + if "Codename" not in result: + _error(f"Missing required information from lsb_release.") + return result + + +@functools.cache +def _apt_update() -> None: + """Runs 'apt-get update' to refresh available packages.""" + return + process = _run( + args=["apt-get", "update"], + superuser=True, + flaky=True, + ) + if process.returncode == 0: + return + _error( + """Drake is unable to run 'sudo apt-get update', probably because this + computer contains incorrect entries in its sources.list files, or + possibly because an internet service is down. Run 'sudo apt-get update' + and try to resolve whatever problems it reports. Do not try to set up + Drake until that command succeeds. This is not a bug in Drake. Do not + contact the Drake team for help.""" + ) + + +def _apt_fix_broken() -> None: + """Runs 'apt-get' with '--fix-broken' to recover from missing packages.""" + _apt_update() + _run( + args=["apt-get", "install", "--fix-broken", "-q"], + superuser=True, + quiet=True, + ) + + +def _get_dpkg_versions(package_names: list[str]) -> dict[str, str]: + """Returns the installed version of packages. The input is a list of + package names, and the return value is a dict mapping all of those + names to their installed version (or None, if not installed).""" + assert package_names + result = {} + for name in package_names: + result[name] = None + process = _run( + args=[ + "dpkg-query", + "--show", + "--showformat=${Package} ${db:Status-Abbrev} ${Version}\n", + ] + + package_names, + check=False, + quiet=True, + ) + for line in process.stdout.splitlines(): + tokens = line.split() + if len(tokens) != 3: + continue + name, status, version = tokens + if status == "ii": + result[name] = version + logging.debug(f"dpkg_versions = {result}") + return result + + +def _get_dpkg_version(package_name: str) -> str: + """Like _get_dpkg_versions but for a single package.""" + return _get_dpkg_versions([package_name])[package_name] + + +def _dpkg_download_install( + *, temp_dir: pathlib.Path, urls: list[str], sha256: str +) -> None: + """Downloads and installs a '*.deb' file, verifying the checksum.""" + assert urls and sha256 + # Try each url in turn. + success = False + errors = [] + for url in urls: + logging.debug(f"Trying {url} ...") + basename = urllib.parse.urlparse(url).path.split("/")[-1] + temp_filename = temp_dir / basename + hasher = hashlib.sha256() + with temp_filename.open("wb") as f: + try: + with urllib.request.urlopen(url=url, timeout=30) as response: + while True: + data = response.read(4096) + if not data: + break + hasher.update(data) + f.write(data) + except OSError as e: + errors.append(f"Candidate {url} failed:\n{e}") + continue + download_sha256 = hasher.hexdigest() + if download_sha256 == sha256: + success = True + break + errors.append( + f"Candidate {url} failed:\n" + f"Checksum mismatch; was {download_sha256} but wanted {sha256}." + ) + + # Report in case no downloads succeeded. + if not success: + messages = "\n\n".join(errors) + _error(f"All downloads failed:\n\n{messages}") + + # Install the deb (even though we're possibly missing its dependencies); + # the apt_fix_broken call should be able to add the missing dependencies. + _run( + args=["dpkg", "--install", str(temp_filename)], + superuser=True, + quiet=True, + ) + _apt_fix_broken() + + +def _apt_install(*, package_names: list[str], yes: bool) -> None: + """Installs the given packages using 'apt-get'. + The `yes` flag is passed along to apt as `--yes`.""" + assert package_names + _apt_update() + args = [ + "apt-get", + "install", + "--no-install-recommends", + ] + if yes: + args.append("--yes") + args.extend(package_names) + process = _run(args=args, superuser=True, check=yes) + if process.returncode == 0: + return + # We can only reach here when yes=False (i.e., check=False). The apt-get + # command didn't work, and the most likely reason is it needs Y/n input + # from the user, so we'll try it again allowing for user input. + _run(args=args, superuser=True, interactive=True, quiet=True) + + +def _apt_install_flavor( + *, lsb_release: dict[str, str], flavor: Flavor, yes: bool +) -> None: + """Installs the apt packages from Ubuntu required for the given `flavor` + and all of its antecedent flavors. The filenames of the package lists are + governed by the `lsb_release` data. The `yes` flag is passed along to apt + as `--yes`.""" + assert isinstance(flavor, Flavor) + codename = lsb_release["Codename"] + description = lsb_release.get("Description", "") + packages = [] + for item in Flavor: + txt_file = _MY_DIR / f"packages-{codename}-{item.name.lower()}.txt" + if not txt_file.exists(): + _warn(f"No such file {str(txt_file)}.") + _error(f"This script does not support {description}.") + logging.debug(f"Reading {txt_file}.") + for line in txt_file.read_text(encoding="utf-8").splitlines(): + line = line.split("#")[0].strip() + if not line: + continue + packages.append(line) + if item == flavor: + break + packages = sorted(packages) + installed = _get_dpkg_versions(packages) + missing_packages = [ + name for name, version in installed.items() if version is None + ] + if missing_packages: + _apt_install(package_names=missing_packages, yes=yes) + else: + logging.debug("All selected packages-*.txt are already installed.") + + +def _dpkg_install_bazelisk(): + """Installs bazelisk using dpkg.""" + arch = _run( + args=["dpkg", "--print-architecture"], + quiet=True, + ).stdout.strip() + + # Load the json metadata. + content = (_MY_DIR / f"packages-bazelisk.json").read_text(encoding="utf-8") + specs = json.loads(content) + name = specs["name"] + version = specs["version"] + sha256 = specs["arch"][arch]["sha256"] + urls = specs["arch"][arch]["urls"] + + # Bail out when already installed at the correct version. + installed = _get_dpkg_version(name) + if installed == version: + logging.debug(f"{name} is already at version {version}") + return + + # Bail out when already installed at a newer version. + if installed is not None: + comparison = _run( + args=[ + "dpkg", + "--compare-versions", + installed, + "gt", + version, + ], + quiet=True, + check=False, + ) + if comparison.returncode == 0: + logging.info( + f"{name} version {installed} is already installed; " + f"not downgrading to Drake's nominal version {version}." + ) + return + + # Install. + with tempfile.TemporaryDirectory(prefix="drake_prereqs_") as temp: + logging.info(f"Installing {name} {version} ...") + _dpkg_download_install( + temp_dir=pathlib.Path(temp), + urls=urls, + sha256=sha256, + ) + + +def _setup_usr_bin_python(*, yes): + """Ensure that /usr/bin/python exists.""" + if pathlib.Path("/usr/bin/python").exists(): + logging.debug("/usr/bin/python is already installed") + return + _apt_install(package_names=["python-is-python3"], yes=yes) + + +def _setup_locales(): + """Ensures that we have available a locale that supports UTF-8 for + generating a C++ header containing Python API documentation during + the build.""" + required_locale = "en_US.utf8" + for line in _run(args=["locale", "-a"], quiet=True).stdout.splitlines(): + if line.strip() == required_locale: + logging.debug(f"The {required_locale} locale already exists.") + return + _run( + args=["locale-gen", required_locale], + superuser=True, + ) + + +def _maybe_setup_gcc12(*, lsb_release: dict[str, str], yes: bool): + """Corrects a common gcc12 installation mistake. + On Jammy, Drake doesn't install anything related to GCC 12, but if the user + has chosen to install some GCC 12 libraries but has failed to install all + of them correctly as a group, Drake's documentation header file parser will + fail with a libclang-related complaint. Therefore, we'll help the user + clean up their mess, to avoid apparent Drake build errors.""" + if lsb_release["Codename"] != "jammy": + # This fixup is only necessary on Jammy. + return + if _get_dpkg_version("libgcc-12-dev") is None: + # GCC 12 is not installed, so we don't need to fix anything. + return + for package in ["libstdc++-12-dev", "libgfortran-12-dev"]: + if _get_dpkg_version(package) is not None: + continue + _apt_install(package_names=[package], yes=yes) + + +def _setup_user_environment(*, lsb_release): + """Writes user environment prerequisites for source builds of Drake.""" + # Write out the bazel configuration. + workspace_dir = _MY_DIR.parent.parent + (workspace_dir / "gen").mkdir(exist_ok=True) + bazelrc = workspace_dir / "gen/environment.bazelrc" + logging.info(f"Writing {str(bazelrc)} ...") + bazelrc.write_text( + textwrap.dedent( + f"""\ + import %workspace%/tools/ubuntu.bazelrc + import %workspace%/tools/ubuntu-{lsb_release["Codename"]}.bazelrc + """ + ), + encoding="utf-8", + ) + + # If the user mistakenly ran 'install_prereqs' under sudo, fix ownership + # of the the generated file to match their actual user. + if os.geteuid() == 0 and os.environ.get("SUDO_USER") is not None: + stat = os.stat(__file__) + os.chown(path=bazelrc, uid=stat.st_uid, gid=stat.st_gid) + + # Prefetch the bazelisk download of bazel. (This is especially helpful for + # the "Provisioned" images in CI.) + if os.geteuid() == 0: + logging.warning("Not pre-fetching bazel for root user.") + else: + _run(args=["bazel", "version"], cwd=str(workspace_dir), quiet=True) + + +def _read_flavor_cookie(*, flavor_lookup) -> str: + """Returns the most recently-installed flavor cookie (reading from disk), + or None when unknown.""" + workspace_dir = _MY_DIR.parent.parent + cookie = workspace_dir / "gen/install_prereqs.flavor" + try: + text = cookie.read_text(encoding="utf-8").strip() + except IOError: + logging.debug("No flavor cookie found.") + return None + # Confirm the prior text is still a valid flavor. + if text not in flavor_lookup: + logging.warning(f"Ignoring no-longer-valid flavor '{text}'") + return None + logging.info(f"Assuming --flavor={text} per most recent run.") + return text + + +def _update_flavor_cookie(*, flavor: Flavor) -> None: + """Updates (on disk) the most recently-installed flavor cookie. + For developers this writes a cookie file with the selected flavor. + For users, this deletes the cookie.""" + workspace_dir = _MY_DIR.parent.parent + cookie = workspace_dir / "gen/install_prereqs.flavor" + if flavor >= Flavor.DEVELOPER: + logging.debug(f"Writing flavor cookie to {cookie}") + flavor_lower = flavor.name.lower() + cookie.write_text(f"{flavor_lower}\n") + else: + try: + cookie.unlink() + logging.debug(f"Removed flavor cookie {cookie}") + except IOError: + pass + + +def main(): + # Decide which flavors to offer. + if _MY_DIR.name == "ubuntu": + # We are in the source tree; all flavors are valid. + flavor_lookup = dict((item.name.lower(), item) for item in Flavor) + default_flavor = "build" + else: + # Not the source tree. We're a binary package install. + flavor_lookup = {"binary": Flavor.BINARY} + default_flavor = "binary" + + # Create the help string for flavors. + help_details = "" + if len(flavor_lookup) > 1: + help_details = "\nThe available flavors of prerequisites are:\n" + help_details += "\n".join( + [ + f"- {name}: {value.description}." + for name, value in flavor_lookup.items() + ] + ) + help_details += "\n\n" + help_details += "Each flavor also incoroporates all prior flavors." + + parser = argparse.ArgumentParser( + description=__doc__ + help_details, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "-y", + action="store_true", + dest="yes", + help="Install without prompting for confirmation.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable verbosity.", + ) + parser.add_argument( + "--__only_user_environment", action="store_true", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--flavor", + choices=flavor_lookup.keys(), + default=None, + help=( + "Which set of prerequisites to install. See descriptions above." + if len(flavor_lookup) > 1 + else argparse.SUPPRESS + ), + ) + for lower in flavor_lookup: + parser.add_argument( + f"--{lower}", + action="store_const", + const=lower, + dest="flavor", + help=( + f"Shortcut for --flavor={lower}." + if len(flavor_lookup) > 1 + else argparse.SUPPRESS + ), + ) + args = parser.parse_args() + is_root = os.geteuid() == 0 + + # Choose which flavor to install: + # (1) The command line always takes priority. + # (2) When not given on the command line, and not running as root, and + # not running from a binary install, a per-user cookie file is used + # for developers to make the choice be "sticky". + # (3) Otherwise, a nominal value is selected. + flavor_lower = args.flavor + if flavor_lower is None and not is_root and len(flavor_lookup) > 1: + flavor_lower = _read_flavor_cookie(flavor_lookup=flavor_lookup) + if flavor_lower is None: + logging.info(f"Using default --flavor={default_flavor}.") + flavor_lower = default_flavor + args.flavor = flavor_lookup[flavor_lower] + + # Prepare for installation. + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + _maybe_warn_conda() + if args.flavor >= Flavor.DEVELOPER and is_root: + logging.warning( + "Do NOT run install_prereqs as root or under sudo when using " + "--flavor=developer or greater." + ) + lsb_release = _check_lsb_release() + + # Deprecation shim until 2025-06-01. + if args.__only_user_environment: + _setup_user_environment(lsb_release=lsb_release) + return + + # Install the prerequisites. + _apt_install_flavor( + lsb_release=lsb_release, + flavor=args.flavor, + yes=args.yes, + ) + if args.flavor >= Flavor.DEVELOPER: + _dpkg_install_bazelisk() + + # Configure the prerequisites. + if args.flavor >= Flavor.BUILD: + _setup_usr_bin_python(yes=args.yes) + _setup_locales() + _maybe_setup_gcc12(lsb_release=lsb_release, yes=args.yes) + if args.flavor >= Flavor.DEVELOPER: + _setup_user_environment(lsb_release=lsb_release) + + # Finished. + if not is_root: + _update_flavor_cookie(flavor=args.flavor) + logging.info(f"Successfully installed --flavor={flavor_lower} prereqs.") + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: %(message)s", + ) + main() diff --git a/setup/ubuntu/install_prereqs.sh b/setup/ubuntu/install_prereqs.sh index e712116f4075..a81ec8a7d416 100755 --- a/setup/ubuntu/install_prereqs.sh +++ b/setup/ubuntu/install_prereqs.sh @@ -1,108 +1,7 @@ #!/bin/bash -# -# Install development and runtime prerequisites for both binary and source -# distributions of Drake on Ubuntu. set -euo pipefail -# Check for existence of `SUDO_USER` so that this may be used in Docker -# environments. -if [[ -n "${SUDO_USER:+D}" && $(id -u ${SUDO_USER}) -eq 0 ]]; then - cat <&2 -It appears that this script is running under sudo, but it was the root user -who ran sudo. That use is not supported; when already running as root, do not -use sudo when calling this script. -eof - exit 1 -fi +echo 'WARNING: This script is deprecated and will be removed on or after 2025-06-01; instead, run drake/setup/install_prereqs' >&2 -at_exit () { - echo "${me} has experienced an error on line ${LINENO}" \ - "while running the command ${BASH_COMMAND}" -} - -me='The Drake source distribution prerequisite setup script' - -trap at_exit EXIT - -binary_distribution_args=() -source_distribution_args=() - -while [ "${1:-}" != "" ]; do - case "$1" in - # Install prerequisites that are only needed to build documentation, - # i.e., those prerequisites that are dependencies of bazel run //doc:build. - --with-doc-only) - source_distribution_args+=(--with-doc-only) - ;; - # Install bazelisk from a deb package. - --with-bazel) - source_distribution_args+=(--with-bazel) - ;; - # Do NOT install bazelisk. - --without-bazel) - source_distribution_args+=(--without-bazel) - ;; - # Install prerequisites that are only needed for --config clang, i.e., - # opts-in to the ability to compile Drake's C++ code using Clang. - --with-clang) - source_distribution_args+=(--with-clang) - ;; - # Do NOT install prerequisites that are only needed for --config clang, - # i.e., opts-out of the ability to compile Drake's C++ code using Clang. - --without-clang) - source_distribution_args+=(--without-clang) - ;; - # Install prerequisites that are only needed to run select maintainer - # scripts. Most developers will not need to install these dependencies. - --with-maintainer-only) - source_distribution_args+=(--with-maintainer-only) - ;; - # Do NOT install prerequisites that are only needed to build and/or run - # unit tests, i.e., those prerequisites that are not dependencies of - # bazel { build, run } //:install. - --without-test-only) - source_distribution_args+=(--without-test-only) - ;; - # Do NOT call apt-get update during execution of this script. - --without-update) - binary_distribution_args+=(--without-update) - source_distribution_args+=(--without-update) - ;; - -y) - binary_distribution_args+=(-y) - source_distribution_args+=(-y) - ;; - *) - echo 'Invalid command line argument' >&2 - exit 1 - esac - shift -done - -# Dependencies that are installed by the following sourced script that are -# needed when developing with binary distributions are also needed when -# developing with source distributions. -# -# Note that the list of packages in binary_distribution/packages.txt is used to -# generate the dependencies of the drake .deb package, so does not include -# development dependencies such as build-essential and cmake. - -source "${BASH_SOURCE%/*}/binary_distribution/install_prereqs.sh" \ - "${binary_distribution_args[@]}" - -# The following additional dependencies are only needed when developing with -# source distributions. -source "${BASH_SOURCE%/*}/source_distribution/install_prereqs.sh" \ - "${source_distribution_args[@]}" - -# Configure user environment, executing as user if we're under `sudo`. -user_env_script="${BASH_SOURCE%/*}/source_distribution/install_prereqs_user_environment.sh" -if [[ -n "${SUDO_USER:+D}" ]]; then - sudo -u "${SUDO_USER}" bash "${user_env_script}" -else - source "${user_env_script}" -fi - -trap : EXIT # Disable exit reporting. -echo 'install_prereqs: success' +exec "${BASH_SOURCE%/*}/../install_prereqs" --verbose --developer "$@" diff --git a/setup/ubuntu/packages-bazelisk.json b/setup/ubuntu/packages-bazelisk.json new file mode 100644 index 000000000000..96f40894a672 --- /dev/null +++ b/setup/ubuntu/packages-bazelisk.json @@ -0,0 +1,18 @@ +{ + "name": "bazelisk", + "version": "1.22.1", + "arch": { + "amd64": { + "sha256": "26723ac5cc5753717ba873281c9abf180ce355fe42aa3219472c24b17bfdd3d6", + "urls": [ + "https://github.com/bazelbuild/bazelisk/releases/download/v1.22.1/bazelisk-amd64.deb" + ] + }, + "arm64": { + "sha256": "86b416e4664d4b4eef2ef44bb0256a4357f5e4e62e4838c45c036cb60b2b1334", + "urls": [ + "https://github.com/bazelbuild/bazelisk/releases/download/v1.22.1/bazelisk-arm64.deb" + ] + } + } +} diff --git a/setup/ubuntu/binary_distribution/packages-jammy.txt b/setup/ubuntu/packages-jammy-binary.txt similarity index 91% rename from setup/ubuntu/binary_distribution/packages-jammy.txt rename to setup/ubuntu/packages-jammy-binary.txt index d57761cd592a..f9cb2b161967 100644 --- a/setup/ubuntu/binary_distribution/packages-jammy.txt +++ b/setup/ubuntu/packages-jammy-binary.txt @@ -1,3 +1,5 @@ +build-essential +cmake default-jre jupyter-notebook libblas-dev @@ -15,6 +17,7 @@ libpython3.10 libspdlog-dev libx11-6 ocl-icd-libopencl1 +pkg-config python3 python3-ipywidgets python3-matplotlib diff --git a/setup/ubuntu/source_distribution/packages-jammy.txt b/setup/ubuntu/packages-jammy-build.txt similarity index 96% rename from setup/ubuntu/source_distribution/packages-jammy.txt rename to setup/ubuntu/packages-jammy-build.txt index f5ef370cb2c6..350820e78053 100644 --- a/setup/ubuntu/source_distribution/packages-jammy.txt +++ b/setup/ubuntu/packages-jammy-build.txt @@ -9,6 +9,7 @@ liblapack-dev libmumps-seq-dev libopengl-dev libx11-dev +locales nasm ocl-icd-opencl-dev opencl-headers diff --git a/setup/ubuntu/source_distribution/packages-jammy-clang.txt b/setup/ubuntu/packages-jammy-clang.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-jammy-clang.txt rename to setup/ubuntu/packages-jammy-clang.txt diff --git a/setup/ubuntu/source_distribution/packages-jammy-test-only.txt b/setup/ubuntu/packages-jammy-developer.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-jammy-test-only.txt rename to setup/ubuntu/packages-jammy-developer.txt diff --git a/setup/ubuntu/source_distribution/packages-jammy-doc-only.txt b/setup/ubuntu/packages-jammy-doc.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-jammy-doc-only.txt rename to setup/ubuntu/packages-jammy-doc.txt diff --git a/setup/ubuntu/source_distribution/packages-jammy-maintainer-only.txt b/setup/ubuntu/packages-jammy-maintainer.txt similarity index 81% rename from setup/ubuntu/source_distribution/packages-jammy-maintainer-only.txt rename to setup/ubuntu/packages-jammy-maintainer.txt index 271e4919495d..993db7b0091c 100644 --- a/setup/ubuntu/source_distribution/packages-jammy-maintainer-only.txt +++ b/setup/ubuntu/packages-jammy-maintainer.txt @@ -1,4 +1,5 @@ alien +ca-certificates diffstat fakeroot patchutils @@ -6,3 +7,4 @@ python3-boto3 python3-git python3-jwcrypto python3-jwt +wget diff --git a/setup/ubuntu/binary_distribution/packages-noble.txt b/setup/ubuntu/packages-noble-binary.txt similarity index 91% rename from setup/ubuntu/binary_distribution/packages-noble.txt rename to setup/ubuntu/packages-noble-binary.txt index a6c99c489038..f32ad4692d5c 100644 --- a/setup/ubuntu/binary_distribution/packages-noble.txt +++ b/setup/ubuntu/packages-noble-binary.txt @@ -1,3 +1,5 @@ +build-essential +cmake default-jre jupyter-notebook libblas-dev @@ -11,11 +13,12 @@ libjchart2d-java liblapack3 libmumps-seq-5.6 libopengl0 -libquadmath0 libpython3.12 +libquadmath0 libspdlog-dev libx11-6 ocl-icd-libopencl1 +pkg-config python3 python3-ipywidgets python3-matplotlib diff --git a/setup/ubuntu/source_distribution/packages-noble.txt b/setup/ubuntu/packages-noble-build.txt similarity index 96% rename from setup/ubuntu/source_distribution/packages-noble.txt rename to setup/ubuntu/packages-noble-build.txt index f5ef370cb2c6..350820e78053 100644 --- a/setup/ubuntu/source_distribution/packages-noble.txt +++ b/setup/ubuntu/packages-noble-build.txt @@ -9,6 +9,7 @@ liblapack-dev libmumps-seq-dev libopengl-dev libx11-dev +locales nasm ocl-icd-opencl-dev opencl-headers diff --git a/setup/ubuntu/source_distribution/packages-noble-clang.txt b/setup/ubuntu/packages-noble-clang.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-noble-clang.txt rename to setup/ubuntu/packages-noble-clang.txt diff --git a/setup/ubuntu/source_distribution/packages-noble-test-only.txt b/setup/ubuntu/packages-noble-developer.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-noble-test-only.txt rename to setup/ubuntu/packages-noble-developer.txt diff --git a/setup/ubuntu/source_distribution/packages-noble-doc-only.txt b/setup/ubuntu/packages-noble-doc.txt similarity index 100% rename from setup/ubuntu/source_distribution/packages-noble-doc-only.txt rename to setup/ubuntu/packages-noble-doc.txt diff --git a/setup/ubuntu/source_distribution/packages-noble-maintainer-only.txt b/setup/ubuntu/packages-noble-maintainer.txt similarity index 81% rename from setup/ubuntu/source_distribution/packages-noble-maintainer-only.txt rename to setup/ubuntu/packages-noble-maintainer.txt index 271e4919495d..993db7b0091c 100644 --- a/setup/ubuntu/source_distribution/packages-noble-maintainer-only.txt +++ b/setup/ubuntu/packages-noble-maintainer.txt @@ -1,4 +1,5 @@ alien +ca-certificates diffstat fakeroot patchutils @@ -6,3 +7,4 @@ python3-boto3 python3-git python3-jwcrypto python3-jwt +wget diff --git a/setup/ubuntu/source_distribution/install_bazelisk.sh b/setup/ubuntu/source_distribution/install_bazelisk.sh deleted file mode 100755 index 3976aec3ebb6..000000000000 --- a/setup/ubuntu/source_distribution/install_bazelisk.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# -# On Ubuntu, installs bazelisk at /usr/bin/bazel{,isk}. -# -# This script does not accept any command line arguments. - -set -euo pipefail - -dpkg_install_from_wget() { - package="$1" - version="$2" - url="$3" - checksum="$4" - - # Skip the install if we're already at the exact version. - installed=$(dpkg-query --showformat='${Version}\n' --show "${package}" 2>/dev/null || true) - if [[ "${installed}" == "${version}" ]]; then - echo "${package} is already at the desired version ${version}" - return - fi - - # If installing our desired version would be a downgrade, ask the user first. - if dpkg --compare-versions "${installed}" gt "${version}"; then - echo "This system has ${package} version ${installed} installed." - echo "Drake suggests downgrading to version ${version}, our supported version." - read -r -p 'Do you want to downgrade? [Y/n] ' reply - if [[ ! "${reply}" =~ ^([yY][eE][sS]|[yY])*$ ]]; then - echo "Skipping ${package} ${version} installation." - return - fi - fi - - # Download and verify. - tmpdeb="/tmp/${package}_${version}-amd64.deb" - wget -O "${tmpdeb}" "${url}" - if echo "${checksum} ${tmpdeb}" | sha256sum -c -; then - echo # Blank line between checkout output and dpkg output. - else - echo "ERROR: The ${package} deb does NOT have the expected SHA256. Not installing." >&2 - exit 2 - fi - - # Install. - dpkg -i "${tmpdeb}" - rm "${tmpdeb}" -} - -# If bazel.deb is already installed, we'll need to remove it first because -# the Debian package of bazelisk will take over the `/usr/bin/bazel` path. -apt-get remove bazel || true - -# Install bazelisk. -if [[ $(arch) = "aarch64" ]]; then - dpkg_install_from_wget \ - bazelisk 1.22.1 \ - https://github.com/bazelbuild/bazelisk/releases/download/v1.22.1/bazelisk-arm64.deb \ - 86b416e4664d4b4eef2ef44bb0256a4357f5e4e62e4838c45c036cb60b2b1334 -else - dpkg_install_from_wget \ - bazelisk 1.22.1 \ - https://github.com/bazelbuild/bazelisk/releases/download/v1.22.1/bazelisk-amd64.deb \ - 26723ac5cc5753717ba873281c9abf180ce355fe42aa3219472c24b17bfdd3d6 -fi diff --git a/setup/ubuntu/source_distribution/install_prereqs.sh b/setup/ubuntu/source_distribution/install_prereqs.sh deleted file mode 100755 index 91422e16e090..000000000000 --- a/setup/ubuntu/source_distribution/install_prereqs.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# -# Install development prerequisites for source distributions of Drake on -# Ubuntu. -# -# The development and runtime prerequisites for binary distributions should be -# installed before running this script. - -set -euo pipefail - -with_doc_only=0 -with_maintainer_only=0 -with_bazel=1 -with_clang=1 -with_test_only=1 -with_update=1 -with_asking=1 - -# TODO(jwnimmer-tri) Eventually we should default to with_clang=0. - -while [ "${1:-}" != "" ]; do - case "$1" in - # Install prerequisites that are only needed to build documentation, - # i.e., those prerequisites that are dependencies of bazel run //doc:build. - --with-doc-only) - with_doc_only=1 - ;; - # Install bazelisk from a deb package. - --with-bazel) - with_bazel=1 - ;; - # Do NOT install bazelisk. - --without-bazel) - with_bazel=0 - ;; - # Install prerequisites that are only needed for --config clang, i.e., - # opts-in to the ability to compile Drake's C++ code using Clang. - --with-clang) - with_clang=1 - ;; - # Do NOT install prerequisites that are only needed for --config clang, - # i.e., opts-out of the ability to compile Drake's C++ code using Clang. - --without-clang) - with_clang=0 - ;; - # Install prerequisites that are only needed to run select maintainer - # scripts. Most developers will not need to install these dependencies. - --with-maintainer-only) - with_maintainer_only=1 - ;; - # Do NOT install prerequisites that are only needed to build and/or run - # unit tests, i.e., those prerequisites that are not dependencies of - # bazel { build, run } //:install. - --without-test-only) - with_test_only=0 - ;; - # Do NOT call apt-get update during execution of this script. - --without-update) - with_update=0 - ;; - # Pass -y along to apt-get. - -y) - with_asking=0 - ;; - *) - echo 'Invalid command line argument' >&2 - exit 3 - esac - shift -done - -if [[ "${EUID}" -ne 0 ]]; then - echo 'ERROR: This script must be run as root' >&2 - exit 1 -fi - -if [[ "${with_asking}" -eq 0 ]]; then - maybe_yes='-y' -else - maybe_yes='' -fi - -if [[ "${with_update}" -eq 1 && "${binary_distribution_called_update:-0}" -ne 1 ]]; then - apt-get update || (sleep 30; apt-get update) -fi - -apt-get install ${maybe_yes} --no-install-recommends $(cat </dev/null || true) - if [[ "${status}" == "ii " ]]; then - status_stdcxx=$(dpkg-query --show --showformat='${db:Status-Abbrev}' libstdc++-12-dev 2>/dev/null || true) - status_fortran=$(dpkg-query --show --showformat='${db:Status-Abbrev}' libgfortran-12-dev 2>/dev/null || true) - if [[ "${status_stdcxx}" != "ii " || "${status_fortran}" != "ii " ]]; then - apt-get install ${maybe_yes} --no-install-recommends libgcc-12-dev libstdc++-12-dev libgfortran-12-dev - fi - fi -fi diff --git a/setup/ubuntu/source_distribution/install_prereqs_user_environment.sh b/setup/ubuntu/source_distribution/install_prereqs_user_environment.sh index 367474daea09..d7594ffb1997 100755 --- a/setup/ubuntu/source_distribution/install_prereqs_user_environment.sh +++ b/setup/ubuntu/source_distribution/install_prereqs_user_environment.sh @@ -1,27 +1,7 @@ #!/bin/bash -# -# Write user environment prerequisites for source distributions of Drake on -# Ubuntu. set -euo pipefail -# Check for existence of `SUDO_USER` so that this may be used in Docker -# environments. -if [[ "${EUID}" -eq 0 && -n "${SUDO_USER:+D}" ]]; then - echo 'This script must NOT be run through sudo as root' >&2 - exit 1 -fi +echo 'WARNING: This script is deprecated and will be removed on or after 2025-06-01; instead, run drake/setup/install_prereqs' >&2 -workspace_dir="$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd)" -bazelrc="${workspace_dir}/gen/environment.bazelrc" -codename=$(lsb_release -sc) - -mkdir -p "$(dirname "${bazelrc}")" -cat > "${bazelrc}" <