[pull] master from NixOS:master

.git-blame-ignore-revs

@@ -159,3 +159,6 @@ fbdcdde04a7caa007e825a8b822c75fab9adb2d6
 
 # step-cli: format package.nix with nixfmt (#331629)
 fc7a83f8b62e90de5679e993d4d49ca014ea013d
+
+# darwin.stdenv: format with nixfmt-rfc-style (#333962)
+93c10ac9e561c6594d3baaeaff2341907390d9b8

.github/CODEOWNERS

@@ -369,16 +369,13 @@ nixos/modules/services/web-apps/pretalx.nix @mweinelt
 nixos/tests/web-apps/pretix.nix @mweinelt
 nixos/tests/web-apps/pretalx.nix @mweinelt
 
-# incus/lxc/lxd
-nixos/maintainers/scripts/lxd/          @adamcstephens
+# incus/lxc
+nixos/maintainers/scripts/incus/        @adamcstephens
 nixos/modules/virtualisation/incus.nix  @adamcstephens
 nixos/modules/virtualisation/lxc*       @adamcstephens
-nixos/modules/virtualisation/lxd*       @adamcstephens
 nixos/tests/incus/                      @adamcstephens
-nixos/tests/lxd/                        @adamcstephens
 pkgs/by-name/in/incus/                  @adamcstephens
 pkgs/by-name/lx/lxc*                    @adamcstephens
-pkgs/by-name/lx/lxd*                    @adamcstephens
 
 # ExpidusOS, Flutter
 /pkgs/development/compilers/flutter @RossComputerGuy

doc/README.md

@@ -251,25 +251,42 @@ You, as the writer of documentation, are still in charge of its content.
   For example:
 
   ```markdown
-  # pkgs.coolFunction
+  # pkgs.coolFunction {#pkgs.coolFunction}
 
-  Description of what `coolFunction` does.
+  `pkgs.coolFunction` *`name`* *`config`*
 
-  ## Inputs
+  Description of what `callPackage` does.
 
-  `coolFunction` expects a single argument which should be an attribute set, with the following possible attributes:
 
-  `name` (String)
+  ## Inputs {#pkgs-coolFunction-inputs}
+
+  If something's special about `coolFunction`'s general argument handling, you can say so here.
+  Otherwise, just describe the single argument or start the arguments' definition list without introduction.
+
+  *`name`* (String)
 
   : The name of the resulting image.
 
-  `tag` (String; _optional_)
+  *`config`* (Attribute set)
+
+  : Introduce the parameter. Maybe you have a test to make sure `{ }` is a sensible default; then you can say: these attributes are optional; `{ }` is a valid argument.
 
-  : Tag of the generated image.
+    `outputHash` (String; _optional_)
 
-    _Default:_ the output path's hash.
+    : A brief explanation including when and when not to pass this attribute.
+
+    : _Default:_ the output path's hash.
   ```
 
+  Checklist:
+  - Start with a synopsis, to show the order of positional arguments.
+  - Metavariables are in emphasized code spans: ``` *`arg1`* ```. Metavariables are placeholders where users may write arbitrary expressions. This includes positional arguments.
+  - Attribute names are regular code spans: ``` `attr1` ```. These identifiers can _not_ be picked freely by users, so they are _not_ metavariables.
+  - _optional_ attributes have a _`Default:`_ if it's easily described as a value.
+  - _optional_ attributes have a _`Default behavior:`_ if it's not easily described using a value.
+  - Nix types aren't in code spans, because they are not code
+  - Nix types are capitalized, to distinguish them from the camelCase [Module System](#module-system) types, which _are_ code and behave like functions.
+
 #### Examples
 
 To define a referenceable figure use the following fencing:

doc/languages-frameworks/python.section.md

@@ -316,13 +316,7 @@ python3Packages.buildPythonApplication rec {
 }
 ```
 
-This is then added to `all-packages.nix` just as any other application would be.
-
-```nix
-{
-  luigi = callPackage ../applications/networking/cluster/luigi { };
-}
-```
+This is then added to `pkgs/by-name` just as any other application would be.
 
 Since the package is an application, a consumer doesn't need to care about
 Python versions or modules, which is why they don't go in `python3Packages`.
@@ -331,25 +325,27 @@ Python versions or modules, which is why they don't go in `python3Packages`.
 
 A distinction is made between applications and libraries, however, sometimes a
 package is used as both. In this case the package is added as a library to
-`python-packages.nix` and as an application to `all-packages.nix`. To reduce
+`python-packages.nix` and as an application to `pkgs/by-name`. To reduce
 duplication the `toPythonApplication` can be used to convert a library to an
 application.
 
 The Nix expression shall use [`buildPythonPackage`](#buildpythonpackage-function) and be called from
-`python-packages.nix`. A reference shall be created from `all-packages.nix` to
+`python-packages.nix`. A reference shall be created from `pkgs/by-name` to
 the attribute in `python-packages.nix`, and the `toPythonApplication` shall be
 applied to the reference:
 
 ```nix
 {
-  youtube-dl = with python3Packages; toPythonApplication youtube-dl;
-}
+  python3Packages,
+}:
+
+python3Packages.toPythonApplication python3Packages.youtube-dl
 ```
 
 #### `toPythonModule` function {#topythonmodule-function}
 
 In some cases, such as bindings, a package is created using
-[`stdenv.mkDerivation`](#sec-using-stdenv) and added as attribute in `all-packages.nix`. The Python
+[`stdenv.mkDerivation`](#sec-using-stdenv) and added as attribute in `pkgs/by-name` or in `all-packages.nix`. The Python
 bindings should be made available from `python-packages.nix`. The
 `toPythonModule` function takes a derivation and makes certain Python-specific
 modifications.
@@ -481,7 +477,6 @@ are used in [`buildPythonPackage`](#buildpythonpackage-function).
   See [example usage](#using-pythonrelaxdepshook).
 - `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder.
 - `setuptoolsBuildHook` to build a wheel using `setuptools`.
-- `setuptoolsCheckHook` to run tests with `python setup.py test`.
 - `sphinxHook` to build documentation and manpages using Sphinx.
 - `venvShellHook` to source a Python 3 `venv` at the `venvDir` location. A
   `venv` is created if it does not yet exist. `postVenvCreation` can be used to

doc/languages-frameworks/r.section.md

@@ -104,24 +104,27 @@ directory and executed as follows:
 ```bash
 nix-shell generate-shell.nix
 
-Rscript generate-r-packages.R cran  > cran-packages.nix.new
-mv cran-packages.nix.new cran-packages.nix
+Rscript generate-r-packages.R cran  > cran-packages.json.new
+mv cran-packages.json.new cran-packages.json
 
-Rscript generate-r-packages.R bioc  > bioc-packages.nix.new
-mv bioc-packages.nix.new bioc-packages.nix
+Rscript generate-r-packages.R bioc  > bioc-packages.json.new
+mv bioc-packages.json.new bioc-packages.json
 
-Rscript generate-r-packages.R bioc-annotation > bioc-annotation-packages.nix.new
-mv bioc-annotation-packages.nix.new bioc-annotation-packages.nix
+Rscript generate-r-packages.R bioc-annotation > bioc-annotation-packages.json.new
+mv bioc-annotation-packages.json.new bioc-annotation-packages.json
 
-Rscript generate-r-packages.R bioc-experiment > bioc-experiment-packages.nix.new
-mv bioc-experiment-packages.nix.new bioc-experiment-packages.nix
+Rscript generate-r-packages.R bioc-experiment > bioc-experiment-packages.json.new
+mv bioc-experiment-packages.json.new bioc-experiment-packages.json
 ```
 
-`generate-r-packages.R <repo>` reads  `<repo>-packages.nix`, therefore
+`generate-r-packages.R <repo>` reads  `<repo>-packages.json`, therefore
 the renaming.
 
+The contents of a generated `*-packages.json` file will be used to
+create a package derivation for each R package listed in the file.
+
 Some packages require overrides to specify external dependencies or other
 patches and special requirements. These overrides are specified in the
-`pkgs/development/r-modules/default.nix` file. As the `*-packages.nix`
+`pkgs/development/r-modules/default.nix` file. As the `*-packages.json`
 contents are automatically generated it should not be edited and broken
 builds should be addressed using overrides.

doc/stdenv/stdenv.chapter.md

@@ -1538,6 +1538,16 @@ Adds the `-fPIE` compiler and `-pie` linker options. Position Independent Execut
 Static libraries need to be compiled with `-fPIE` so that executables can link them in with the `-pie` linker option.
 If the libraries lack `-fPIE`, you will get the error `recompile with -fPIE`.
 
+#### `shadowstack` {#shadowstack}
+
+Adds the `-fcf-protection=return` compiler option. This enables the Shadow Stack feature supported by some newer processors, which maintains a user-inaccessible copy of the program's stack containing only return-addresses. When returning from a function, the processor compares the return-address value on the two stacks and throws an error if they do not match, considering it a sign of corruption and possible tampering. This should significantly increase the difficulty of ROP attacks.
+
+For the Shadow Stack to be enabled at runtime, all code linked into a process must be built with Shadow Stack enabled, so this is probably only useful to enable on a wide scale, so that all of a packages dependencies also have the feature enabled.
+
+This is currently only supported on some newer Intel and AMD processors as part of the Intel CET set of features. However, the generated code should continue to work on older processors which will simply omit any of this checking.
+
+This breaks some code that does advanced stack management or exception handling. If enabling this hardening flag it is important to test the result on a system that has known working and enabled CET support, so that any such breakage can be discovered.
+
 #### `trivialautovarinit` {#trivialautovarinit}
 
 Adds the `-ftrivial-auto-var-init=pattern` compiler option. This causes "trivially-initializable" uninitialized stack variables to be forcibly initialized with a nonzero value that is likely to cause a crash (and therefore be noticed). Uninitialized variables generally take on their values based on fragments of previous program state, and attackers can carefully manipulate that state to craft malicious initial values for these variables.
@@ -1554,6 +1564,14 @@ sorry, unimplemented: __builtin_clear_padding not supported for variable length
 
 This flag adds the `-fstack-clash-protection` compiler option, which causes growth of a program's stack to access each successive page in order. This should force the guard page to be accessed and cause an attempt to "jump over" this guard page to crash.
 
+#### `pacret` {#pacret}
+
+This flag adds the `-mbranch-protection=pac-ret` compiler option on aarch64-linux targets. This uses ARM v8.3's Pointer Authentication feature to sign function return pointers before adding them to the stack. The pointer's authenticity is then validated before returning to its destination. This dramatically increases the difficulty of ROP exploitation techniques.
+
+This may cause problems with code that does advanced stack manipulation, and debugging/stack-unwinding tools need to be pac-ret aware to work correctly when these features are in operation.
+
+Pre-ARM v8.3 processors will ignore Pointer Authentication instructions, so code built with this flag will continue to work on older processors, though without any of the intended protections. If enabling this flag, it is recommended to ensure the resultant packages are tested against an ARM v8.3+ linux system with known-working Pointer Authentication support so that any breakage caused by this feature is actually detected.
+
 [^footnote-stdenv-ignored-build-platform]: The build platform is ignored because it is a mere implementation detail of the package satisfying the dependency: As a general programming principle, dependencies are always *specified* as interfaces, not concrete implementation.
 [^footnote-stdenv-native-dependencies-in-path]: Currently, this means for native builds all dependencies are put on the `PATH`. But in the future that may not be the case for sake of matching cross: the platforms would be assumed to be unique for native and cross builds alike, so only the `depsBuild*` and `nativeBuildInputs` would be added to the `PATH`.
 [^footnote-stdenv-propagated-dependencies]: Nix itself already takes a package’s transitive dependencies into account, but this propagation ensures nixpkgs-specific infrastructure like [setup hooks](#ssec-setup-hooks) also are run as if it were a propagated dependency.

flake.nix

@@ -77,6 +77,10 @@
         }).nixos.manual;
       };
 
+      devShells = forAllSystems (system: {
+        default = import ./shell.nix { inherit system; };
+      });
+
       # The "legacy" in `legacyPackages` doesn't imply that the packages exposed
       # through this attribute are "legacy" packages. Instead, `legacyPackages`
       # is used here as a substitute attribute name for `packages`. The problem

lib/tests/modules.sh

@@ -13,13 +13,44 @@ set -o errexit -o noclobber -o nounset -o pipefail
 shopt -s failglob inherit_errexit
 
 # https://stackoverflow.com/a/246128/6605742
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 cd "$DIR"/modules
 
 pass=0
 fail=0
 
+# loc
+#   prints the location of the call of to the function that calls it
+# loc n
+#   prints the location n levels up the call stack
+loc() {
+    local caller depth
+    depth=1
+    if [[ $# -gt 0 ]]; then
+        depth=$1
+    fi
+    # ( lineno fnname file ) of the caller
+    caller=( $(caller $depth) )
+    echo "${caller[2]}:${caller[0]}"
+}
+
+line() {
+    echo "----------------------------------------"
+}
+logStartFailure() {
+    line
+}
+logEndFailure() {
+    line
+    echo
+}
+
+logFailure() {
+    # bold red
+    printf '\033[1;31mTEST FAILED\033[0m at %s\n' "$(loc 2)"
+}
+
 evalConfig() {
     local attr=$1
     shift
@@ -31,7 +62,7 @@ reportFailure() {
     local attr=$1
     shift
     local script="import ./default.nix { modules = [ $* ];}"
-    echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only --json"
+    echo "$ nix-instantiate -E '$script' -A '$attr' --eval-only --json"
     evalConfig "$attr" "$@" || true
     ((++fail))
 }
@@ -42,8 +73,12 @@ checkConfigOutput() {
     if evalConfig "$@" 2>/dev/null | grep -E --silent "$outputContains" ; then
         ((++pass))
     else
-        echo 2>&1 "error: Expected result matching '$outputContains', while evaluating"
+        logStartFailure
+        echo "ACTUAL:"
         reportFailure "$@"
+        echo "EXPECTED: result matching '$outputContains'"
+        logFailure
+        logEndFailure
     fi
 }
 
@@ -52,14 +87,22 @@ checkConfigError() {
     local err=""
     shift
     if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then
-        echo 2>&1 "error: Expected error code, got exit code 0, while evaluating"
+        logStartFailure
+        echo "ACTUAL: exit code 0, output:"
         reportFailure "$@"
+        echo "EXPECTED: non-zero exit code"
+        logFailure
+        logEndFailure
     else
         if echo "$err" | grep -zP --silent "$errorContains" ; then
             ((++pass))
         else
-            echo 2>&1 "error: Expected error matching '$errorContains', while evaluating"
+            logStartFailure
+            echo "ACTUAL:"
             reportFailure "$@"
+            echo "EXPECTED: error matching '$errorContains'"
+            logFailure
+            logEndFailure
         fi
     fi
 }