libtensorflow: 2.16.2 -> 2.17.0

NixOS · Nov 17, 2024 · 1280551 · 1280551
1 parent b30b925
commit 1280551
Show file tree

Hide file tree

Showing 6 changed files with 117 additions and 37 deletions.
diff --git a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch
@@ -10,7 +10,7 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.
      "pretty_function",
  ]]
 
-@@ -22,7 +24,13 @@
+@@ -22,7 +24,14 @@
 
  cc_library(
      name = "raw_logging_internal",
@@ -21,6 +21,7 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.
 +        "-labsl_log_internal_message",
 +        "-labsl_log_internal_nullguard",
 +        "-labsl_log_internal_check_op",
++        "-labsl_vlog_config_internal",
 +    ],
      visibility = [
          "//absl:__subpackages__",
@@ -36,6 +37,17 @@ diff -Naurd x/third_party/absl/system.absl.crc.BUILD y/third_party/absl/system.a
 +    linkopts = ["-labsl_crc32c"],
 +    visibility = ["//visibility:public"],
 +)
+diff -Naurd x/third_party/absl/system.absl.flags.BUILD y/third_party/absl/system.absl.flags.BUILD
+--- x/third_party/absl/system.absl.flags.BUILD	2024-10-14 22:24:37.247740325 +0000
++++ y/third_party/absl/system.absl.flags.BUILD	2024-10-15 22:38:39.508870283 +0000
+@@ -97,7 +97,6 @@
+
+ cc_library(
+     name = "flag",
+-    linkopts = ["-labsl_flags"],
+     deps = [
+         ":config",
+         ":flag_internal",
 diff -Naurd x/third_party/absl/system.absl.functional.BUILD y/third_party/absl/system.absl.functional.BUILD
 --- x/third_party/absl/system.absl.functional.BUILD	2024-10-10 10:15:40.640185040 +0000
 +++ y/third_party/absl/system.absl.functional.BUILD	2024-10-10 10:17:59.221375986 +0000
@@ -189,6 +201,20 @@ diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.a
 +        "//absl/strings",
 +    ],
 +)
+diff -Naurd x/third_party/absl/system.absl.random.BUILD y/third_party/absl/system.absl.random.BUILD
+--- x/third_party/absl/system.absl.random.BUILD	2024-10-14 22:24:37.267740358 +0000
++++ y/third_party/absl/system.absl.random.BUILD	2024-10-14 22:26:16.211902901 +0000
+@@ -54,10 +54,6 @@
+
+ cc_library(
+     name = "bit_gen_ref",
+-)
+-
+-cc_library(
+-    name = "bit_gen_ref",
+     deps = [
+         ":random",
+         "//absl/base:core_headers",
 diff -Naurd x/third_party/absl/system.absl.strings.BUILD y/third_party/absl/system.absl.strings.BUILD
 --- x/third_party/absl/system.absl.strings.BUILD	2024-10-10 10:15:40.644185074 +0000
 +++ y/third_party/absl/system.absl.strings.BUILD	2024-10-10 18:52:37.648334488 +0000
@@ -203,6 +229,17 @@ diff -Naurd x/third_party/absl/system.absl.strings.BUILD y/third_party/absl/syst
      name = "strings",
      linkopts = ["-labsl_strings"],
      deps = [
+diff -Naurd x/third_party/absl/system.absl.synchronization.BUILD y/third_party/absl/system.absl.synchronization.BUILD
+--- x/third_party/absl/system.absl.synchronization.BUILD	2024-10-14 22:24:37.259740345 +0000
++++ y/third_party/absl/system.absl.synchronization.BUILD	2024-10-15 22:42:21.497207586 +0000
+@@ -20,6 +20,7 @@
+     name = "synchronization",
+     linkopts = [
+         "-labsl_synchronization",
++        "-labsl_kernel_timeout_internal",
+         "-pthread",
+     ],
+     deps = [
 diff -Naurd x/third_party/absl/workspace.bzl y/third_party/absl/workspace.bzl
 --- x/third_party/absl/workspace.bzl	2024-10-10 10:16:56.508840929 +0000
 +++ y/third_party/absl/workspace.bzl	2024-10-10 20:09:16.685514935 +0000

diff --git a/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch b/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch
@@ -4,8 +4,8 @@ diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl
 @@ -862,6 +862,7 @@
      tf_http_archive(
          name = "riegeli",
-         sha256 = "870ca080cdfc5eba696a72ccc3a54cbf0f2271befc0d459eafa8f065edfaadb2",
+         sha256 = "1d216d5c97fa60632143d209a1bb48c2a83788efdb876902e7bbc06396d5ee1f",
 +        patch_file = ["//third_party:riegeli-proto.patch"],
-         strip_prefix = "riegeli-264ef7b4a1314d97265b37544b27cd3923ea72d2",
-         urls = tf_mirror_urls("https://github.com/google/riegeli/archive/264ef7b4a1314d97265b37544b27cd3923ea72d2.zip"),
+         strip_prefix = "riegeli-5d75119232cd4f6db8dfa69a1503289f050e9643",
+         urls = tf_mirror_urls("https://github.com/google/riegeli/archive/5d75119232cd4f6db8dfa69a1503289f050e9643.zip"),
      )
diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix
@@ -44,7 +44,7 @@
   charset-normalizer,
   markupsafe,
   psutil,
-  dm-tree,
+  optree,
   # Common deps
   git,
   pybind11,
@@ -59,6 +59,8 @@
   lndir,
   writeShellScriptBin,
   writeText,
+  patchelf,
+  unzip,
   # Common libraries
   jemalloc,
   mpi,
@@ -195,7 +197,7 @@ let
 
   tfFeature = x: if x then "1" else "0";
 
-  version = "2.16.2";
+  version = "2.17.0";
   format = "setuptools";
   variant = lib.optionalString cudaSupport "-gpu";
   pname = "tensorflow${variant}";
@@ -520,18 +522,18 @@ let
     '';
   };
 
-  dm-tree-bazel = stdenv.mkDerivation {
-    name = "${dm-tree.pname}-bazel-${dm-tree.version}";
+  optree-bazel = stdenv.mkDerivation {
+    name = "${optree.pname}-bazel-${optree.version}";
     src = _bazel-build.deps;
     dontConfigure = true;
     dontBuild = true;
     installPhase = ''
       runHook preInstall
       mkdir $out
-      cp pypi_dm_tree/* $out
+      cp pypi_optree/* $out
       mkdir $out/site-packages
       ${lndir}/bin/lndir \
-        ${dm-tree}/lib/python${python_dotted}/site-packages \
+        ${optree}/lib/python${python_dotted}/site-packages \
         $out/site-packages
       runHook postInstall
     '';
@@ -664,7 +666,8 @@ let
             "--override_repository=pypi_h5py=${h5py-bazel}"
             "--override_repository=pypi_wrapt=${wrapt-bazel}"
             "--override_repository=pypi_markupsafe=${markupsafe-bazel}"
-            "--override_repository=pypi_dm_tree=${dm-tree-bazel}"
+            "--override_repository=pypi_optree=${optree-bazel}"
+            "--override_repository=ml_dtypes=${ml_dtypes-fix}"
             "--override_repository=pypi=${pypi-bazel}"
             "--override_repository=python=${python-bazel}"
             "--override_repository=python_version_repo=${python_version_repo-bazel}"
@@ -675,7 +678,6 @@ let
           ++ lib.optionals cudaSupport [
             "--override_repository=nccl_archive=${nccl_archive-fix}"
             "--override_repository=eigen_archive=${eigen_archive-fix}"
-            "--override_repository=ml_dtypes=${ml_dtypes-fix}"
           ];
       })
     else
@@ -699,7 +701,7 @@ let
       owner = "tensorflow";
       repo = "tensorflow";
       rev = "refs/tags/v${version}";
-      hash = "sha256-yBDS9DFkVLIT+bu/vqh/0U60UwoIzqlTKQsflM7GWCA=";
+      hash = "sha256-dJGmoYdOTumFdS8AWWGYp5HRxk/WMKQNRi2ubyD/TAU=";
     };
 
     # On update, it can be useful to steal the changes from gentoo
@@ -717,6 +719,7 @@ let
         protobuf-extra
         wrapped_clang
         lndir
+        patchelf
       ]
       ++ lib.optional cudaSupport addDriverRunpath
       ++ lib.optional stdenv.isDarwin xcbuild
@@ -849,6 +852,16 @@ let
           url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch";
           hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w=";
         })
+        # this commit causes a libprotobuf error in:
+        # src/google/protobuf/descriptor_database.cc:642
+        # File already exists in database:
+        # tensorflow/core/tpu/kernels/sparse_core_layout.proto
+        (fetchpatch {
+          name = "fix-sparse-core-layout-proto-duplicate-loading.patch";
+          url = "https://github.com/tensorflow/tensorflow/commit/1d36138fd0204daa48d97d66e415f46ba91f0697.diff";
+          hash = "sha256-1rzz9zNTQVdKMI3ByeNT/CdLhOiCaYSrDE+/C31ekig=";
+          revert = true;
+        })
         ./fix-syslib-references.patch
         ./protobuf_lite.patch
         ./protobuf_cc_toolchain.patch
@@ -875,6 +888,8 @@ let
         rm -f .bazelversion
         patchShebangs .
         ln -s ${./riegeli-proto.patch} third_party/riegeli-proto.patch
+        substituteInPlace tensorflow/tools/pip_package/build_pip_package.py \
+          --replace-fail '"patchelf"' '"${patchelf}/bin/patchelf"'
       ''
       + lib.optionalString (!stdenv.isDarwin) ''
         sed \
@@ -952,7 +967,7 @@ let
     ] ++ lib.optionals (mklSupport) [ "--config=mkl" ];
 
     bazelTargets = [
-      "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow"
+      "//tensorflow/tools/pip_package:wheel //tensorflow/tools/lib_package:libtensorflow"
     ];
 
     removeRulesCC = false;
@@ -977,7 +992,8 @@ let
           rm -rf external/pypi_h5py/site-packages
           rm -rf external/pypi_wrapt/site-packages
           rm -rf external/pypi_markupsafe/site-packages
-          rm -rf external/pypi_dm_tree/site-packages
+          rm -rf external/pypi_optree/site-packages
+          rm -rf external/pypi_ml_dtypes/site-packages
         ''
         + lib.optionalString (stdenv.isAarch64) ''
           rm -rf external/pypi_psutil/site-packages
@@ -1015,7 +1031,7 @@ let
             if cudaSupport then
               "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
             else
-              "sha256-SlIIHoD4mLcUbcjkXch/NFA4khbHsj19v05Z6xMX0/Q=";
+              "sha256-bg3ii3T/Ic01xV73lSA2SYVmDIkNCasQo6fEhk9lVVk=";
           aarch64-linux =
             if cudaSupport then
               "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
@@ -1038,7 +1054,7 @@ let
 
       # need to rebuild schemas since we use a different flatbuffers version
       preBuild = ''
-        (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema.fbs)
+        (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema_v3c.fbs)
         (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c conversion_metadata.fbs)
         (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc -o configuration.fbs --proto configuration.proto)
         sed -i s,tflite.proto,tflite,g tensorflow/lite/acceleration/configuration/configuration.fbs/configuration.fbs
@@ -1066,10 +1082,9 @@ let
         Cflags: -I$out/include/tensorflow
         EOF
 
-        # build the source code, then copy it to $python (build_pip_package
-        # actually builds a symlink farm so we must dereference them).
-        bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist"
-        cp -Lr "$PWD/dist" "$python"
+        mkdir "$python"
+        cp bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-${version}-*.whl \
+          "$python/${pname}-${version}.whl"
       '';
 
       postFixup = lib.optionalString cudaSupport ''
@@ -1108,7 +1123,29 @@ buildPythonPackage {
   inherit version pname;
   disabled = pythonOlder "3.9";
 
-  src = bazel-build.python;
+  format = "other";
+
+  src = "${bazel-build.python}/${pname}-${version}.whl";
+
+  sourceRoot = ".";
+
+  unpackPhase = ''
+    runHook preUnpack
+    ${unzip}/bin/unzip $src
+    runHook postUnpack
+  '';
+
+  dontConfigure = true;
+  dontBuild = true;
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out/${python.sitePackages}
+    cp -r $sourceRoot/tensorflow* $out/${python.sitePackages}
+
+    runHook postInstall
+  '';
 
   # Adjust dependency requirements:
   # - Drop tensorflow-io dependency until we get it to build
@@ -1127,14 +1164,6 @@ buildPythonPackage {
     "keras"
   ];
 
-  # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow
-  # and the propagated input tensorboard, which causes environment collisions.
-  # Another possibility would be to have tensorboard only in the buildInputs
-  # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79
-  postInstall = ''
-    rm $out/bin/tensorboard
-  '';
-
   setupPyGlobalFlags = [ "--project_name ${pname}" ];
 
   # tensorflow/tools/pip_package/setup.py
@@ -1189,8 +1218,13 @@ buildPythonPackage {
   # TODO better test (files in tensorflow/tools/ci_build/builds/*test)
   # TEST_PACKAGES in tensorflow/tools/pip_package/setup.py
   nativeCheckInputs = [
+    curl
     dill
+    double-conversion
+    grpc
+    jsoncpp
     portpicker
+    snappy
     tblib
   ];
   checkPhase = ''

diff --git a/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch b/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch
@@ -1,7 +1,7 @@
 diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/BUILD
 --- x/tensorflow/tools/pip_package/BUILD	2023-10-14 20:09:43.001857258 +0000
 +++ y/tensorflow/tools/pip_package/BUILD	2023-10-14 20:15:17.894102008 +0000
-@@ -209,7 +209,7 @@
+@@ -89,13 +89,12 @@
          "@com_google_protobuf//:LICENSE",
          "@curl//:COPYING",
          "@ducc//:LICENSE",
@@ -10,3 +10,9 @@ diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/
          "@gemmlowp//:LICENSE",
          "@libjpeg_turbo//:LICENSE.md",
          "@llvm-project//llvm:LICENSE.TXT",
+         "@llvm-project//mlir:LICENSE.TXT",
+         "@local_config_tensorrt//:LICENSE",
+-        "@ml_dtypes//:LICENSE",
+         "@org_brotli//:LICENSE",
+         "@pasta//:LICENSE",
+         "@png//:LICENSE",
diff --git a/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch b/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch
@@ -1,17 +1,20 @@
 diff -Naurd x/third_party/systemlibs/protobuf.BUILD y/third_party/systemlibs/protobuf.BUILD
 --- x/third_party/systemlibs/protobuf.BUILD	2023-12-03 15:39:27.742087246 +0000
 +++ y/third_party/systemlibs/protobuf.BUILD	2023-12-04 13:25:41.777638732 +0000
-@@ -1,4 +1,8 @@
+@@ -4,7 +4,11 @@
+     "proto_gen",
+     "py_proto_library",
+ )
 -load("@rules_proto//proto:defs.bzl", "proto_library")
 +load(
 +    "@rules_proto//proto:defs.bzl",
 +    "proto_lang_toolchain",
 +    "proto_library",
 +)
- load(
-     "@com_google_protobuf//:protobuf.bzl",
-     "cc_proto_library",
-@@ -118,3 +122,13 @@
+
+ licenses(["notice"])
+
+@@ -124,3 +128,13 @@
      srcs = [proto[1][0] for proto in WELL_KNOWN_PROTO_MAP.items()],
      visibility = ["//visibility:public"],
  )

diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
@@ -15326,7 +15326,7 @@ self: super: with self; {
 
   tensorflow-build = let
     compat = rec {
-      abseil-cppTF = pkgs.abseil-cpp_202301;
+      abseil-cppTF = pkgs.abseil-cpp_202401;
       protobufTF = pkgs.protobuf_21.override {
         abseil-cpp = abseil-cppTF;
       };