From ac177dff933cfa8de4ba87a7367e0efe80877958 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 13:33:50 +0000 Subject: [PATCH 01/15] lib.fetchers: add `normalizeHash` and `withNormalizedHash` --- lib/fetchers.nix | 144 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index b2fe7872a12b9..20e2914db9194 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -1,6 +1,6 @@ # snippets that can be shared by multiple fetchers (pkgs/build-support) { lib }: -{ +rec { proxyImpureEnvVars = [ # We borrow these environment variables from the caller to allow @@ -14,4 +14,146 @@ "NIX_SSL_CERT_FILE" ]; + /** + Converts an attrset containing one of `hash`, `sha256` or `sha512`, + into one containing `outputHash{,Algo}` as accepted by `mkDerivation`. + + All other attributes in the set remain as-is. + + # Example + + ```nix + normalizeHash { } { hash = lib.fakeHash; foo = "bar"; } + => + { + outputHash = lib.fakeHash; + outputHashAlgo = null; + foo = "bar"; + } + ``` + + ```nix + normalizeHash { } { sha256 = lib.fakeSha256; } + => + { + outputHash = lib.fakeSha256; + outputHashAlgo = "sha256"; + } + ``` + + ```nix + normalizeHash { } { sha512 = lib.fakeSha512; } + => + { + outputHash = lib.fakeSha512; + outputHashAlgo = "sha512"; + } + ``` + + # Type + ``` + normalizeHash :: { hashTypes :: List String, required :: Bool } -> AttrSet -> AttrSet + ``` + + # Arguments + + hashTypes + : the set of attribute names accepted as hash inputs, in addition to `hash` + + required + : whether to throw if no hash was present in the input; otherwise returns the original input, unmodified + */ + normalizeHash = { + hashTypes ? [ "sha256" ], + required ? true, + }: args: + with builtins; with lib; + let + hNames = [ "hash" ] ++ hashTypes; + + # The argument hash, as a {name, value} pair + h = + let _h = attrsToList (intersectAttrs (genAttrs hNames (const {})) args); in + if _h == [] then + throw "fetcher called without `hash`" + else if tail _h != [] then + throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) _h}" + else + head _h + ; + in + if args ? "outputHash" then + args + else + removeAttrs args hNames // { + outputHash = h.value; + outputHashAlgo = if h.name == "hash" then null else h.name; + } + ; + + /** + Wraps a function which accepts `outputHash{,Algo}` into one which accepts `hash` or `sha{256,512}` + + # Example + ```nix + withNormalizedHash { hashTypes = [ "sha256" "sha512" ]; } ( + { outputHash, outputHashAlgo, ... }: + ... + ) + ``` + is a function which accepts one of `hash`, `sha256`, or `sha512` (or the original's `outputHash` and `outputHashAlgo`). + + Its `functionArgs` metadata only lists `hash` as a parameter, optional iff. `outputHash` was an optional parameter of + the original function. `sha256`, `sha512`, `outputHash`, or `outputHashAlgo` are not mentioned in the `functionArgs` + metadata. + + # Type + ``` + withNormalizedHash :: { hashTypes :: List String } -> (AttrSet -> T) -> (AttrSet -> T) + ``` + + # Arguments + + hashTypes + : the set of attribute names accepted as hash inputs, in addition to `hash` + : they must correspond to a valid value for `outputHashAlgo`, currently one of: `md5`, `sha1`, `sha256`, or `sha512`. + + f + : the function to be wrapped + + ::: {.note} + In nixpkgs, `mkDerivation` rejects MD5 `outputHash`es, and SHA-1 is being deprecated. + + As such, there is no reason to add `md5` to `hashTypes`, and + `sha1` should only ever be included for backwards compatibility. + ::: + + # Output + + `withNormalizedHash { inherit hashTypes; } f` is functionally equivalent to + ```nix + args: f (normalizeHash { + inherit hashTypes; + required = !(lib.functionArgs f).outputHash; + } args) + ``` + + However, `withNormalizedHash` preserves `functionArgs` metadata insofar as possible, + and is implemented somewhat more efficiently. + */ + withNormalizedHash = { + hashTypes ? [ "sha256" ] + }: fetcher: + with builtins; with lib; + let + hAttrs = genAttrs ([ "hash" ] ++ hashTypes) (const {}); + fArgs = functionArgs fetcher; + in + # The o.g. fetcher must *only* accept outputHash and outputHashAlgo + assert !fArgs.outputHash && !fArgs.outputHashAlgo; + assert intersectAttrs fArgs hAttrs == {}; + + setFunctionArgs + (args: fetcher (normalizeHash { inherit hashTypes; } args)) + (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = false; }); } From 80cafd06e612e5e06e6599c964d2dace4cdcb1ce Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 09:53:12 +0000 Subject: [PATCH 02/15] fetchRepoProject: support `hash` attribute, wrapping in `withnormalizedHash` --- .../fetchrepoproject/default.nix | 164 +++++++++--------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/pkgs/build-support/fetchrepoproject/default.nix b/pkgs/build-support/fetchrepoproject/default.nix index a5e79c8d4ef97..354913574b0f2 100644 --- a/pkgs/build-support/fetchrepoproject/default.nix +++ b/pkgs/build-support/fetchrepoproject/default.nix @@ -1,83 +1,83 @@ { lib, stdenvNoCC, gitRepo, cacert, copyPathsToStore }: - -{ name, manifest, rev ? "HEAD", sha256 -# Optional parameters: -, repoRepoURL ? "", repoRepoRev ? "", referenceDir ? "", manifestName ? "" -, localManifests ? [], createMirror ? false, useArchive ? false -}: - -assert repoRepoRev != "" -> repoRepoURL != ""; -assert createMirror -> !useArchive; - -let - inherit (lib) - concatMapStringsSep - concatStringsSep - fetchers - optionalString - ; - - extraRepoInitFlags = [ - (optionalString (repoRepoURL != "") "--repo-url=${repoRepoURL}") - (optionalString (repoRepoRev != "") "--repo-branch=${repoRepoRev}") - (optionalString (referenceDir != "") "--reference=${referenceDir}") - (optionalString (manifestName != "") "--manifest-name=${manifestName}") - ]; - - repoInitFlags = [ - "--manifest-url=${manifest}" - "--manifest-branch=${rev}" - "--depth=1" - (optionalString createMirror "--mirror") - (optionalString useArchive "--archive") - ] ++ extraRepoInitFlags; - - local_manifests = copyPathsToStore localManifests; - -in stdenvNoCC.mkDerivation { - inherit name; - - inherit cacert manifest rev repoRepoURL repoRepoRev referenceDir; # TODO - - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = sha256; - - preferLocalBuild = true; - enableParallelBuilding = true; - - impureEnvVars = fetchers.proxyImpureEnvVars ++ [ - "GIT_PROXY_COMMAND" "SOCKS_SERVER" - ]; - - nativeBuildInputs = [ gitRepo cacert ]; - - GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - - buildCommand = '' - # Path must be absolute (e.g. for GnuPG: ~/.repoconfig/gnupg/pubring.kbx) - export HOME="$(pwd)" - - mkdir $out - cd $out - - mkdir .repo - ${optionalString (local_manifests != []) '' - mkdir .repo/local_manifests - for local_manifest in ${concatMapStringsSep " " toString local_manifests}; do - cp $local_manifest .repo/local_manifests/$(stripHash $local_manifest) - done - ''} - - repo init ${concatStringsSep " " repoInitFlags} - repo sync --jobs=$NIX_BUILD_CORES --current-branch - - # TODO: The git-index files (and probably the files in .repo as well) have - # different contents each time and will therefore change the final hash - # (i.e. creating a mirror probably won't work). - ${optionalString (!createMirror) '' - rm -rf .repo - find -type d -name '.git' -prune -exec rm -rf {} + - ''} - ''; -} +lib.fetchers.withNormalizedHash { } ( + { name, manifest, rev ? "HEAD", outputHash, outputHashAlgo + # Optional parameters: + , repoRepoURL ? "", repoRepoRev ? "", referenceDir ? "", manifestName ? "" + , localManifests ? [], createMirror ? false, useArchive ? false + }: + + assert repoRepoRev != "" -> repoRepoURL != ""; + assert createMirror -> !useArchive; + + let + inherit (lib) + concatMapStringsSep + concatStringsSep + fetchers + optionalString + ; + + extraRepoInitFlags = [ + (optionalString (repoRepoURL != "") "--repo-url=${repoRepoURL}") + (optionalString (repoRepoRev != "") "--repo-branch=${repoRepoRev}") + (optionalString (referenceDir != "") "--reference=${referenceDir}") + (optionalString (manifestName != "") "--manifest-name=${manifestName}") + ]; + + repoInitFlags = [ + "--manifest-url=${manifest}" + "--manifest-branch=${rev}" + "--depth=1" + (optionalString createMirror "--mirror") + (optionalString useArchive "--archive") + ] ++ extraRepoInitFlags; + + local_manifests = copyPathsToStore localManifests; + + in stdenvNoCC.mkDerivation { + inherit name; + + inherit cacert manifest rev repoRepoURL repoRepoRev referenceDir; # TODO + + inherit outputHash outputHashAlgo; + outputHashMode = "recursive"; + + preferLocalBuild = true; + enableParallelBuilding = true; + + impureEnvVars = fetchers.proxyImpureEnvVars ++ [ + "GIT_PROXY_COMMAND" "SOCKS_SERVER" + ]; + + nativeBuildInputs = [ gitRepo cacert ]; + + GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + buildCommand = '' + # Path must be absolute (e.g. for GnuPG: ~/.repoconfig/gnupg/pubring.kbx) + export HOME="$(pwd)" + + mkdir $out + cd $out + + mkdir .repo + ${optionalString (local_manifests != []) '' + mkdir .repo/local_manifests + for local_manifest in ${concatMapStringsSep " " toString local_manifests}; do + cp $local_manifest .repo/local_manifests/$(stripHash $local_manifest) + done + ''} + + repo init ${concatStringsSep " " repoInitFlags} + repo sync --jobs=$NIX_BUILD_CORES --current-branch + + # TODO: The git-index files (and probably the files in .repo as well) have + # different contents each time and will therefore change the final hash + # (i.e. creating a mirror probably won't work). + ${optionalString (!createMirror) '' + rm -rf .repo + find -type d -name '.git' -prune -exec rm -rf {} + + ''} + ''; + } +) From 50ca073e52de000affae314f75612bed0227f4cd Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 10:07:07 +0000 Subject: [PATCH 03/15] =?UTF-8?q?amdvlk:=20`sha256`=20=E2=86=92=20`hash`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/am/amdvlk/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/am/amdvlk/package.nix b/pkgs/by-name/am/amdvlk/package.nix index 8adf2b030d521..a50e51685f91f 100644 --- a/pkgs/by-name/am/amdvlk/package.nix +++ b/pkgs/by-name/am/amdvlk/package.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation (finalAttrs: { name = "amdvlk-src"; manifest = "https://github.com/GPUOpen-Drivers/AMDVLK.git"; rev = "refs/tags/v-${finalAttrs.version}"; - sha256 = "1Svdr93ShjhaWJUTLn5y1kBM4hHey1dUVDiHqFIKgrU="; + hash = "sha256-1Svdr93ShjhaWJUTLn5y1kBM4hHey1dUVDiHqFIKgrU="; }; buildInputs = From c361b3f7c8137cb64b16c82cf3707d2ef355cb30 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 14:36:12 +0000 Subject: [PATCH 04/15] buildBazelPackage: support `fetchAttrs.hash` --- pkgs/build-support/build-bazel-package/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 3ffff74f70e23..e752ccdb7dfca 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -64,7 +64,7 @@ let ; }; fBuildAttrs = fArgs // buildAttrs; - fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ]; + fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "hash" "sha256" ]; bazelCmd = { cmd, additionalFlags, targets, targetRunFlags ? [ ] }: lib.optionalString (targets != [ ]) '' # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables] @@ -197,8 +197,10 @@ stdenv.mkDerivation (fBuildAttrs // { dontFixup = true; allowedRequisites = []; - outputHashAlgo = "sha256"; - outputHash = fetchAttrs.sha256; + inherit (lib.fetchers.normalizeHash { hashTypes = [ "sha256" ]; } fetchAttrs) + outputHash + outputHashAlgo + ; }); nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ]; From f9d60a4751ab0a773d75dd7d676d4a4e3b359d9b Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 10:25:13 +0000 Subject: [PATCH 05/15] =?UTF-8?q?perf=5Fdata=5Fconverter:=20`sha256`=20?= =?UTF-8?q?=E2=86=92=20`hash`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/pe/perf_data_converter/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/pe/perf_data_converter/package.nix b/pkgs/by-name/pe/perf_data_converter/package.nix index 217640f2fbff1..827be3d7d242c 100644 --- a/pkgs/by-name/pe/perf_data_converter/package.nix +++ b/pkgs/by-name/pe/perf_data_converter/package.nix @@ -26,7 +26,7 @@ buildBazelPackage rec { ]; fetchAttrs = { - sha256 = "sha256-Qm6Ng9cXvKx043P7qyNHyyMvdGK9aNarX1ZKeCp3mgY="; + hash = "sha256-Qm6Ng9cXvKx043P7qyNHyyMvdGK9aNarX1ZKeCp3mgY="; }; nativeBuildInputs = [ jdk ]; From b19ba3fa515fd8befb21bb411c3907e2c3819e5d Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 10:25:13 +0000 Subject: [PATCH 06/15] =?UTF-8?q?protoc-gen-js:=20`sha256`=20=E2=86=92=20`?= =?UTF-8?q?hash`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/pr/protoc-gen-js/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/pr/protoc-gen-js/package.nix b/pkgs/by-name/pr/protoc-gen-js/package.nix index 78d03e9b2b7be..5f3a592e0faa0 100644 --- a/pkgs/by-name/pr/protoc-gen-js/package.nix +++ b/pkgs/by-name/pr/protoc-gen-js/package.nix @@ -19,7 +19,7 @@ buildBazelPackage rec { LIBTOOL = lib.optionalString stdenv.isDarwin "${cctools}/bin/libtool"; - fetchAttrs.sha256 = "sha256-WOBlZ0XNrl5UxIaSDxZeOfzS2a8ZkrKdTLKHBDC9UNQ="; + fetchAttrs.hash = "sha256-WOBlZ0XNrl5UxIaSDxZeOfzS2a8ZkrKdTLKHBDC9UNQ="; buildAttrs.installPhase = '' mkdir -p $out/bin From 0bddcc1df93a200b9b84049e9a1a8da8414c8d47 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 10:25:13 +0000 Subject: [PATCH 07/15] =?UTF-8?q?verible:=20`sha256`=20=E2=86=92=20`hash`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/ve/verible/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/ve/verible/package.nix b/pkgs/by-name/ve/verible/package.nix index 11ace1650d774..ac2bd40c1c793 100644 --- a/pkgs/by-name/ve/verible/package.nix +++ b/pkgs/by-name/ve/verible/package.nix @@ -37,7 +37,7 @@ buildBazelPackage rec { ]; fetchAttrs = { - sha256 = "sha256-bKASgc5KftCWtMvJkGA4nweBAtgdnyC9uXIJxPjKYS0="; + hash = "sha256-bKASgc5KftCWtMvJkGA4nweBAtgdnyC9uXIJxPjKYS0="; }; nativeBuildInputs = [ From 39446161fc351c6c51aa15cfb9e30f9c74766cbd Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 21:07:57 +0000 Subject: [PATCH 08/15] lib.fetchers.withNormalizedHash: handle fetchers whose hash parameter is optional --- lib/fetchers.nix | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index 20e2914db9194..3f4eec5225ce5 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -75,7 +75,7 @@ rec { h = let _h = attrsToList (intersectAttrs (genAttrs hNames (const {})) args); in if _h == [] then - throw "fetcher called without `hash`" + throwIf required "fetcher called without `hash`" null else if tail _h != [] then throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) _h}" else @@ -85,10 +85,10 @@ rec { if args ? "outputHash" then args else - removeAttrs args hNames // { + removeAttrs args hNames // (optionalAttrs (h != null) { outputHash = h.value; outputHashAlgo = if h.name == "hash" then null else h.name; - } + }) ; /** @@ -148,12 +148,13 @@ rec { let hAttrs = genAttrs ([ "hash" ] ++ hashTypes) (const {}); fArgs = functionArgs fetcher; + required = !fArgs.outputHash; in # The o.g. fetcher must *only* accept outputHash and outputHashAlgo - assert !fArgs.outputHash && !fArgs.outputHashAlgo; + assert fArgs ? outputHash && fArgs ? outputHashAlgo; assert intersectAttrs fArgs hAttrs == {}; setFunctionArgs - (args: fetcher (normalizeHash { inherit hashTypes; } args)) - (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = false; }); + (args: fetcher (normalizeHash { inherit hashTypes required; } args)) + (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = !required; }); } From 0518f4d245207a4d9948c0f74e0f8d5ce752b362 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 15 Sep 2024 21:13:26 +0000 Subject: [PATCH 09/15] fetchgit: factor-out the hash logic to `lib.fetchers.withNormalizedHash` --- pkgs/build-support/fetchgit/default.nix | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 92c7468753e87..1b000fb49a99e 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -10,8 +10,9 @@ appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}"; in "${if matched == null then base else builtins.head matched}${appendShort}"; in -lib.makeOverridable ( -{ url, rev ? "HEAD", sha256 ? "", hash ? "", leaveDotGit ? deepClone +lib.makeOverridable (lib.fetchers.withNormalizedHash { } ( +{ url, rev ? "HEAD", leaveDotGit ? deepClone +, outputHash ? lib.fakeHash, outputHashAlgo ? null , fetchSubmodules ? true, deepClone ? false , branchName ? null , sparseCheckout ? [] @@ -56,9 +57,7 @@ lib.makeOverridable ( assert deepClone -> leaveDotGit; assert nonConeMode -> (sparseCheckout != []); -if hash != "" && sha256 != "" then - throw "Only one of sha256 or hash can be set" -else if builtins.isString sparseCheckout then +if builtins.isString sparseCheckout then # Changed to throw on 2023-06-04 throw "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." else @@ -70,14 +69,8 @@ stdenvNoCC.mkDerivation { nativeBuildInputs = [ git cacert ] ++ lib.optionals fetchLFS [ git-lfs ]; - outputHashAlgo = if hash != "" then null else "sha256"; + inherit outputHash outputHashAlgo; outputHashMode = "recursive"; - outputHash = if hash != "" then - hash - else if sha256 != "" then - sha256 - else - lib.fakeSha256; # git-sparse-checkout(1) says: # > When the --stdin option is provided, the directories or patterns are read @@ -105,4 +98,4 @@ stdenvNoCC.mkDerivation { gitRepoUrl = url; }; } -) +)) From f470dc703ca3cbc16d8c13d64aed74564332e722 Mon Sep 17 00:00:00 2001 From: nicoo Date: Mon, 16 Sep 2024 09:05:29 +0000 Subject: [PATCH 10/15] lib.fetchers: optimize `normalizeHash` and `withNormalizedHash` via min-scoping --- lib/fetchers.nix | 53 +++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index 3f4eec5225ce5..0736dd598d905 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -66,29 +66,32 @@ rec { normalizeHash = { hashTypes ? [ "sha256" ], required ? true, - }: args: - with builtins; with lib; + }: let hNames = [ "hash" ] ++ hashTypes; - - # The argument hash, as a {name, value} pair - h = - let _h = attrsToList (intersectAttrs (genAttrs hNames (const {})) args); in - if _h == [] then - throwIf required "fetcher called without `hash`" null - else if tail _h != [] then - throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) _h}" - else - head _h - ; + hAttrs = genAttrs hNames (const {}); in - if args ? "outputHash" then - args - else - removeAttrs args hNames // (optionalAttrs (h != null) { - outputHash = h.value; - outputHashAlgo = if h.name == "hash" then null else h.name; - }) + with builtins; with lib; + args: + if args ? "outputHash" then + args + else + let + # The argument hash, as a {name, value} pair + h = + let _h = attrsToList (intersectAttrs hAttrs args); in + if _h == [] then + throwIf required "fetcher called without `hash`" null + else if tail _h != [] then + throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) _h}" + else + head _h + ; + in + removeAttrs args hNames // (optionalAttrs (h != null) { + outputHash = h.value; + outputHashAlgo = if h.name == "hash" then null else h.name; + }) ; /** @@ -148,13 +151,17 @@ rec { let hAttrs = genAttrs ([ "hash" ] ++ hashTypes) (const {}); fArgs = functionArgs fetcher; - required = !fArgs.outputHash; + + normalize = normalizeHash { + inherit hashTypes; + required = !fArgs.outputHash; + }; in # The o.g. fetcher must *only* accept outputHash and outputHashAlgo assert fArgs ? outputHash && fArgs ? outputHashAlgo; assert intersectAttrs fArgs hAttrs == {}; setFunctionArgs - (args: fetcher (normalizeHash { inherit hashTypes required; } args)) - (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = !required; }); + (args: fetcher (normalize args)) + (removeAttrs fArgs [ "outputHash" "outputHashAlgo" ] // { hash = fArgs.outputHash; }); } From 8376e18291905bc0b5b67d1188e4f3b26ae94fa4 Mon Sep 17 00:00:00 2001 From: nicoo Date: Mon, 16 Sep 2024 09:12:12 +0000 Subject: [PATCH 11/15] lib.fetchers: replace `with`-bindings with explicit `let inherit` --- lib/fetchers.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index 0736dd598d905..ed2340b80322d 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -68,10 +68,12 @@ rec { required ? true, }: let + inherit (lib) concatMapStringsSep const head tail throwIf; + inherit (lib.attrsets) attrsToList genAttrs intersectAttrs removeAttrs optionalAttrs; + hNames = [ "hash" ] ++ hashTypes; hAttrs = genAttrs hNames (const {}); in - with builtins; with lib; args: if args ? "outputHash" then args @@ -147,8 +149,10 @@ rec { withNormalizedHash = { hashTypes ? [ "sha256" ] }: fetcher: - with builtins; with lib; let + inherit (lib.attrsets) genAttrs intersectAttrs removeAttrs; + inherit (lib.trivial) const functionArgs setFunctionArgs; + hAttrs = genAttrs ([ "hash" ] ++ hashTypes) (const {}); fArgs = functionArgs fetcher; From 4e59b77c7099f6459619d11131855010c555e5de Mon Sep 17 00:00:00 2001 From: nicoo Date: Mon, 16 Sep 2024 21:42:39 +0000 Subject: [PATCH 12/15] lib.fetchers: factor-out definitions common to `normalizeHash` and `withNormalizedHash` --- lib/fetchers.nix | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index ed2340b80322d..07ed09669c7cd 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -1,6 +1,11 @@ # snippets that can be shared by multiple fetchers (pkgs/build-support) { lib }: -rec { +let + commonH = hashTypes: rec { + hNames = [ "hash" ] ++ hashTypes; + hAttrs = lib.genAttrs hNames (lib.const {}); + }; +in rec { proxyImpureEnvVars = [ # We borrow these environment variables from the caller to allow @@ -68,11 +73,10 @@ rec { required ? true, }: let - inherit (lib) concatMapStringsSep const head tail throwIf; - inherit (lib.attrsets) attrsToList genAttrs intersectAttrs removeAttrs optionalAttrs; + inherit (lib) concatMapStringsSep head tail throwIf; + inherit (lib.attrsets) attrsToList intersectAttrs removeAttrs optionalAttrs; - hNames = [ "hash" ] ++ hashTypes; - hAttrs = genAttrs hNames (const {}); + inherit (commonH hashTypes) hAttrs hNames; in args: if args ? "outputHash" then @@ -153,7 +157,7 @@ rec { inherit (lib.attrsets) genAttrs intersectAttrs removeAttrs; inherit (lib.trivial) const functionArgs setFunctionArgs; - hAttrs = genAttrs ([ "hash" ] ++ hashTypes) (const {}); + inherit (commonH hashTypes) hAttrs; fArgs = functionArgs fetcher; normalize = normalizeHash { From 4c991b74d39a470111d8c73f82c5b60476eabed5 Mon Sep 17 00:00:00 2001 From: nicoo Date: Tue, 17 Sep 2024 07:18:39 +0000 Subject: [PATCH 13/15] lib.fetchers.normalizeHash: replace `""` with `lib.fake*` --- lib/fetchers.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index 07ed09669c7cd..0cb96c5b310da 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -5,6 +5,12 @@ let hNames = [ "hash" ] ++ hashTypes; hAttrs = lib.genAttrs hNames (lib.const {}); }; + + fakeH = { + hash = lib.fakeHash; + sha256 = lib.fakeSha256; + sha512 = lib.fakeSha512; + }; in rec { proxyImpureEnvVars = [ @@ -23,12 +29,15 @@ in rec { Converts an attrset containing one of `hash`, `sha256` or `sha512`, into one containing `outputHash{,Algo}` as accepted by `mkDerivation`. + An appropriate “fake hash” is substituted when the hash value is `""`, + as is the [convention for fetchers](#sec-pkgs-fetchers-updating-source-hashes-fakehash-method). + All other attributes in the set remain as-is. # Example ```nix - normalizeHash { } { hash = lib.fakeHash; foo = "bar"; } + normalizeHash { } { hash = ""; foo = "bar"; } => { outputHash = lib.fakeHash; @@ -95,8 +104,12 @@ in rec { ; in removeAttrs args hNames // (optionalAttrs (h != null) { - outputHash = h.value; outputHashAlgo = if h.name == "hash" then null else h.name; + outputHash = + if h.value == "" then + fakeH.${h.name} or (throw "no “fake hash” defined for ${h.name}") + else + h.value; }) ; From 0aa52428290ee1c8fa4b366373edfb604925abc4 Mon Sep 17 00:00:00 2001 From: nicoo Date: Tue, 17 Sep 2024 08:18:37 +0000 Subject: [PATCH 14/15] lib.fetchers: add tests --- lib/tests/fetchers.nix | 165 ++++++++++++++++++++++++++++++++++++ lib/tests/test-with-nix.nix | 1 + 2 files changed, 166 insertions(+) create mode 100644 lib/tests/fetchers.nix diff --git a/lib/tests/fetchers.nix b/lib/tests/fetchers.nix new file mode 100644 index 0000000000000..c36da35673a71 --- /dev/null +++ b/lib/tests/fetchers.nix @@ -0,0 +1,165 @@ +let + lib = import ./..; + + inherit (lib) + fakeHash + fakeSha256 + fakeSha512 + flip + functionArgs + runTests + ; + inherit (lib.fetchers) normalizeHash withNormalizedHash; + + testingThrow = expr: { + expr = with builtins; tryEval (seq expr "didn't throw"); + expected = { + success = false; + value = false; + }; + }; + + # hashes of empty + sri256 = "sha256-d6xi4mKdjkX2JFicDIv5niSzpyI0m/Hnm8GGAIU04kY="; + sri512 = "sha512-AXFyVo7jiZ5we10fxZ5E9qfPjSfqkizY2apCzORKFVYZaNhCIVbooY+J4cYST00ztLf0EjivIBPPdtIYFUMfzQ=="; + + unionOfDisjoints = lib.foldl lib.attrsets.unionOfDisjoint { }; + + genTests = n: f: { + "test${n}AlreadyNormalized" = { + expr = f { } { + outputHash = ""; + outputHashAlgo = "md42"; + }; + expected = { + outputHash = ""; + outputHashAlgo = "md42"; + }; + }; + + "test${n}EmptySha256" = { + expr = f { } { sha256 = ""; }; + expected = { + outputHash = fakeSha256; + outputHashAlgo = "sha256"; + }; + }; + + "test${n}EmptySha512" = { + expr = f { hashTypes = [ "sha512" ]; } { sha512 = ""; }; + expected = { + outputHash = fakeSha512; + outputHashAlgo = "sha512"; + }; + }; + + "test${n}EmptyHash" = { + expr = f { } { hash = ""; }; + expected = { + outputHash = fakeHash; + outputHashAlgo = null; + }; + }; + + "test${n}Sri256" = { + expr = f { } { hash = sri256; }; + expected = { + outputHash = sri256; + outputHashAlgo = null; + }; + }; + + "test${n}Sri512" = { + expr = f { } { hash = sri512; }; + expected = { + outputHash = sri512; + outputHashAlgo = null; + }; + }; + + "test${n}PreservesAttrs" = { + expr = f { } { + hash = "aaaa"; + destination = "Earth"; + }; + expected = { + outputHash = "aaaa"; + outputHashAlgo = null; + destination = "Earth"; + }; + }; + + "test${n}RejectsSha1ByDefault" = testingThrow (f { } { sha1 = ""; }); + "test${n}RejectsSha512ByDefault" = testingThrow (f { } { sha512 = ""; }); + + "test${n}ThrowsOnMissing" = testingThrow (f { } { gibi = false; }); + }; +in +runTests (unionOfDisjoints [ + (genTests "NormalizeHash" normalizeHash) + (genTests "WithNormalized" ( + flip withNormalizedHash ({ outputHash, outputHashAlgo, ... }@args: args) + )) + { + testNormalizeNotRequiredEquivalent = { + expr = normalizeHash { required = false; } { + hash = ""; + prof = "shadoko"; + }; + expected = normalizeHash { } { + hash = ""; + prof = "shadoko"; + }; + }; + + testNormalizeNotRequiredPassthru = { + expr = normalizeHash { required = false; } { "ga bu" = "zo meu"; }; + expected."ga bu" = "zo meu"; + }; + + testOptionalArg = { + expr = withNormalizedHash { } ( + { + outputHash ? "", + outputHashAlgo ? null, + ... + }@args: + args + ) { author = "Jacques Rouxel"; }; + expected.author = "Jacques Rouxel"; + }; + + testOptionalArgMetadata = { + expr = functionArgs ( + withNormalizedHash { } ( + { + outputHash ? "", + outputHashAlgo ? null, + }: + { } + ) + ); + expected.hash = true; + }; + + testPreservesArgsMetadata = { + expr = functionArgs ( + withNormalizedHash { } ( + { + outputHash, + outputHashAlgo, + pumping ? true, + }: + { } + ) + ); + expected = { + hash = false; + pumping = true; + }; + }; + + testRejectsMissingHashArg = testingThrow (withNormalizedHash { } ({ outputHashAlgo }: { })); + testRejectsMissingAlgoArg = testingThrow (withNormalizedHash { } ({ outputHash }: { })); + } +]) diff --git a/lib/tests/test-with-nix.nix b/lib/tests/test-with-nix.nix index 63b4b10bae8c4..1316aaa49b88c 100644 --- a/lib/tests/test-with-nix.nix +++ b/lib/tests/test-with-nix.nix @@ -17,6 +17,7 @@ pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { buildInputs = [ (import ./check-eval.nix) + (import ./fetchers.nix) (import ./maintainers.nix { inherit pkgs; lib = import ../.; From 09eb3c64e8025e86d838e7624ac3063e9c698d5c Mon Sep 17 00:00:00 2001 From: nicoo Date: Tue, 17 Sep 2024 14:48:44 +0000 Subject: [PATCH 15/15] lib.fetchers.normalizeHash: more implementation comment and clearer variable names --- lib/fetchers.nix | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index 0cb96c5b310da..cd6d5d7916d3e 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -2,8 +2,8 @@ { lib }: let commonH = hashTypes: rec { - hNames = [ "hash" ] ++ hashTypes; - hAttrs = lib.genAttrs hNames (lib.const {}); + hashNames = [ "hash" ] ++ hashTypes; + hashSet = lib.genAttrs hashNames (lib.const {}); }; fakeH = { @@ -85,7 +85,7 @@ in rec { inherit (lib) concatMapStringsSep head tail throwIf; inherit (lib.attrsets) attrsToList intersectAttrs removeAttrs optionalAttrs; - inherit (commonH hashTypes) hAttrs hNames; + inherit (commonH hashTypes) hashNames hashSet; in args: if args ? "outputHash" then @@ -94,16 +94,17 @@ in rec { let # The argument hash, as a {name, value} pair h = - let _h = attrsToList (intersectAttrs hAttrs args); in - if _h == [] then + # All hashes passed in arguments (possibly 0 or >1) as a list of {name, value} pairs + let hashesAsNVPairs = attrsToList (intersectAttrs hashSet args); in + if hashesAsNVPairs == [] then throwIf required "fetcher called without `hash`" null - else if tail _h != [] then - throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) _h}" + else if tail hashesAsNVPairs != [] then + throw "fetcher called with mutually-incompatible arguments: ${concatMapStringsSep ", " (a: a.name) hashesAsNVPairs}" else - head _h + head hashesAsNVPairs ; in - removeAttrs args hNames // (optionalAttrs (h != null) { + removeAttrs args hashNames // (optionalAttrs (h != null) { outputHashAlgo = if h.name == "hash" then null else h.name; outputHash = if h.value == "" then @@ -170,7 +171,7 @@ in rec { inherit (lib.attrsets) genAttrs intersectAttrs removeAttrs; inherit (lib.trivial) const functionArgs setFunctionArgs; - inherit (commonH hashTypes) hAttrs; + inherit (commonH hashTypes) hashSet; fArgs = functionArgs fetcher; normalize = normalizeHash { @@ -180,7 +181,7 @@ in rec { in # The o.g. fetcher must *only* accept outputHash and outputHashAlgo assert fArgs ? outputHash && fArgs ? outputHashAlgo; - assert intersectAttrs fArgs hAttrs == {}; + assert intersectAttrs fArgs hashSet == {}; setFunctionArgs (args: fetcher (normalize args))