From 43bbe83915a86aec522ca734e52fd17fb06a5003 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 9 Dec 2024 23:22:12 +0100 Subject: [PATCH] ci/eval: add rebuildsByPlatform to the comparison result --- ci/eval/compare/default.nix | 8 +++-- ci/eval/compare/utils.nix | 61 ++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 8b5e9059cd5b8d..28f9b4fd924c2d 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -11,6 +11,8 @@ let inherit (import ./utils.nix { inherit lib; }) diff groupByKernel + processSystemPath + groupByPlatform extractPackageNames getLabels uniqueStrings @@ -25,8 +27,10 @@ let changed-paths = let rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed); + rebuildsAttrs = builtins.map processSystemPath rebuilds; - rebuildsByKernel = groupByKernel rebuilds; + rebuildsByPlatform = groupByPlatform rebuildsAttrs; + rebuildsByKernel = groupByKernel rebuildsAttrs; rebuildCountByKernel = lib.mapAttrs ( kernel: kernelRebuilds: lib.length kernelRebuilds ) rebuildsByKernel; @@ -34,7 +38,7 @@ let writeText "changed-paths.json" ( builtins.toJSON { attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs; - inherit rebuildsByKernel rebuildCountByKernel; + inherit rebuildsByPlatform rebuildsByKernel rebuildCountByKernel; labels = getLabels rebuildCountByKernel; } ); diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix index 22353499d69933..2fbb7e1db43ca1 100644 --- a/ci/eval/compare/utils.nix +++ b/ci/eval/compare/utils.nix @@ -3,7 +3,14 @@ rec { # Borrowed from https://github.com/NixOS/nixpkgs/pull/355616 uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list); - _processSystemPath = + # Turns + # "hello.aarch64-linux" + # into + # { + # name = "hello"; + # system = "aarch64-linux"; + # } + processSystemPath = packageSystemPath: let # python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"] @@ -41,7 +48,7 @@ rec { packageSystemPaths: builtins.attrNames ( builtins.removeAttrs (builtins.groupBy ( - packageSystemPath: (_processSystemPath packageSystemPath).name + packageSystemPath: (processSystemPath packageSystemPath).name ) packageSystemPaths) [ "" ] ); @@ -71,35 +78,53 @@ rec { # Turns # [ - # "hello.aarch64-linux" - # "hello.x86_64-linux" - # "hello.aarch64-darwin" - # "hello.x86_64-darwin" - # "bye.x86_64-darwin" - # "bye.aarch64-darwin" + # { name = "hello"; system = "aarch64-linux"; } + # { name = "hello"; system = "x86_64-linux"; } + # { name = "hello"; system = "aarch64-darwin"; } + # { name = "hello"; system = "x86_64-darwin"; } + # { name = "bye"; system = "aarch64-darwin"; } + # { name = "bye"; system = "x86_64-darwin"; } # ] # # into # # { - # linux = [ - # "hello" - # ]; - # darwin = [ - # "hello" - # "bye" - # ]; + # aarch64-linux = [ "hello" ]; + # x86_64-linux = [ "hello" ]; + # aarch64-darwin = [ "hello" "bye" ]; + # x86_64-darwin = [ "hello" "bye" ]; # } - groupByKernel = + groupByPlatform = systemPaths: let - systemPaths' = builtins.map _processSystemPath systemPaths; + systemPathsByPlatform = builtins.groupBy (systemPath: systemPath.system) systemPaths; + in + map (systemPath: systemPath.name) systemPathsByPlatform; + # Turns + # [ + # { name = "hello"; system = "aarch64-linux"; } + # { name = "hello"; system = "x86_64-linux"; } + # { name = "hello"; system = "aarch64-darwin"; } + # { name = "hello"; system = "x86_64-darwin"; } + # { name = "bye"; system = "aarch64-darwin"; } + # { name = "bye"; system = "x86_64-darwin"; } + # ] + # + # into + # + # { + # linux = [ "hello" ]; + # darwin = [ "hello" "bye" ]; + # } + groupByKernel = + systemPaths: + let filterKernel = kernel: builtins.attrNames ( builtins.groupBy (systemPath: systemPath.name) ( - builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths' + builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths ) ); in