From 12805518eb66edb458585d0babe3ba5a5515f762 Mon Sep 17 00:00:00 2001 From: Isidor Zeuner Date: Sun, 17 Nov 2024 23:40:22 +0100 Subject: [PATCH] libtensorflow: 2.16.2 -> 2.17.0 --- .../tensorflow/com_google_absl_add_log.patch | 39 ++++++++- .../tensorflow/core-riegeli-proto.patch | 6 +- .../python-modules/tensorflow/default.nix | 86 +++++++++++++------ .../tensorflow/fix-syslib-references.patch | 8 +- .../tensorflow/protobuf_cc_toolchain.patch | 13 +-- pkgs/top-level/python-packages.nix | 2 +- 6 files changed, 117 insertions(+), 37 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch index b9f0901c9b6c84..506740e30cc87d 100644 --- a/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch +++ b/pkgs/development/python-modules/tensorflow/com_google_absl_add_log.patch @@ -10,7 +10,7 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system. "pretty_function", ]] -@@ -22,7 +24,13 @@ +@@ -22,7 +24,14 @@ cc_library( name = "raw_logging_internal", @@ -21,6 +21,7 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system. + "-labsl_log_internal_message", + "-labsl_log_internal_nullguard", + "-labsl_log_internal_check_op", ++ "-labsl_vlog_config_internal", + ], visibility = [ "//absl:__subpackages__", @@ -36,6 +37,17 @@ diff -Naurd x/third_party/absl/system.absl.crc.BUILD y/third_party/absl/system.a + linkopts = ["-labsl_crc32c"], + visibility = ["//visibility:public"], +) +diff -Naurd x/third_party/absl/system.absl.flags.BUILD y/third_party/absl/system.absl.flags.BUILD +--- x/third_party/absl/system.absl.flags.BUILD 2024-10-14 22:24:37.247740325 +0000 ++++ y/third_party/absl/system.absl.flags.BUILD 2024-10-15 22:38:39.508870283 +0000 +@@ -97,7 +97,6 @@ + + cc_library( + name = "flag", +- linkopts = ["-labsl_flags"], + deps = [ + ":config", + ":flag_internal", diff -Naurd x/third_party/absl/system.absl.functional.BUILD y/third_party/absl/system.absl.functional.BUILD --- x/third_party/absl/system.absl.functional.BUILD 2024-10-10 10:15:40.640185040 +0000 +++ y/third_party/absl/system.absl.functional.BUILD 2024-10-10 10:17:59.221375986 +0000 @@ -189,6 +201,20 @@ diff -Naurd x/third_party/absl/system.absl.log.BUILD y/third_party/absl/system.a + "//absl/strings", + ], +) +diff -Naurd x/third_party/absl/system.absl.random.BUILD y/third_party/absl/system.absl.random.BUILD +--- x/third_party/absl/system.absl.random.BUILD 2024-10-14 22:24:37.267740358 +0000 ++++ y/third_party/absl/system.absl.random.BUILD 2024-10-14 22:26:16.211902901 +0000 +@@ -54,10 +54,6 @@ + + cc_library( + name = "bit_gen_ref", +-) +- +-cc_library( +- name = "bit_gen_ref", + deps = [ + ":random", + "//absl/base:core_headers", diff -Naurd x/third_party/absl/system.absl.strings.BUILD y/third_party/absl/system.absl.strings.BUILD --- x/third_party/absl/system.absl.strings.BUILD 2024-10-10 10:15:40.644185074 +0000 +++ y/third_party/absl/system.absl.strings.BUILD 2024-10-10 18:52:37.648334488 +0000 @@ -203,6 +229,17 @@ diff -Naurd x/third_party/absl/system.absl.strings.BUILD y/third_party/absl/syst name = "strings", linkopts = ["-labsl_strings"], deps = [ +diff -Naurd x/third_party/absl/system.absl.synchronization.BUILD y/third_party/absl/system.absl.synchronization.BUILD +--- x/third_party/absl/system.absl.synchronization.BUILD 2024-10-14 22:24:37.259740345 +0000 ++++ y/third_party/absl/system.absl.synchronization.BUILD 2024-10-15 22:42:21.497207586 +0000 +@@ -20,6 +20,7 @@ + name = "synchronization", + linkopts = [ + "-labsl_synchronization", ++ "-labsl_kernel_timeout_internal", + "-pthread", + ], + deps = [ diff -Naurd x/third_party/absl/workspace.bzl y/third_party/absl/workspace.bzl --- x/third_party/absl/workspace.bzl 2024-10-10 10:16:56.508840929 +0000 +++ y/third_party/absl/workspace.bzl 2024-10-10 20:09:16.685514935 +0000 diff --git a/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch b/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch index 8129b0119bd71f..77d6702b2acf04 100644 --- a/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch +++ b/pkgs/development/python-modules/tensorflow/core-riegeli-proto.patch @@ -4,8 +4,8 @@ diff -Naurd x/tensorflow/workspace2.bzl y/tensorflow/workspace2.bzl @@ -862,6 +862,7 @@ tf_http_archive( name = "riegeli", - sha256 = "870ca080cdfc5eba696a72ccc3a54cbf0f2271befc0d459eafa8f065edfaadb2", + sha256 = "1d216d5c97fa60632143d209a1bb48c2a83788efdb876902e7bbc06396d5ee1f", + patch_file = ["//third_party:riegeli-proto.patch"], - strip_prefix = "riegeli-264ef7b4a1314d97265b37544b27cd3923ea72d2", - urls = tf_mirror_urls("https://github.com/google/riegeli/archive/264ef7b4a1314d97265b37544b27cd3923ea72d2.zip"), + strip_prefix = "riegeli-5d75119232cd4f6db8dfa69a1503289f050e9643", + urls = tf_mirror_urls("https://github.com/google/riegeli/archive/5d75119232cd4f6db8dfa69a1503289f050e9643.zip"), ) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 2c92ade47b0a12..d358fb027df304 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -44,7 +44,7 @@ charset-normalizer, markupsafe, psutil, - dm-tree, + optree, # Common deps git, pybind11, @@ -59,6 +59,8 @@ lndir, writeShellScriptBin, writeText, + patchelf, + unzip, # Common libraries jemalloc, mpi, @@ -195,7 +197,7 @@ let tfFeature = x: if x then "1" else "0"; - version = "2.16.2"; + version = "2.17.0"; format = "setuptools"; variant = lib.optionalString cudaSupport "-gpu"; pname = "tensorflow${variant}"; @@ -520,18 +522,18 @@ let ''; }; - dm-tree-bazel = stdenv.mkDerivation { - name = "${dm-tree.pname}-bazel-${dm-tree.version}"; + optree-bazel = stdenv.mkDerivation { + name = "${optree.pname}-bazel-${optree.version}"; src = _bazel-build.deps; dontConfigure = true; dontBuild = true; installPhase = '' runHook preInstall mkdir $out - cp pypi_dm_tree/* $out + cp pypi_optree/* $out mkdir $out/site-packages ${lndir}/bin/lndir \ - ${dm-tree}/lib/python${python_dotted}/site-packages \ + ${optree}/lib/python${python_dotted}/site-packages \ $out/site-packages runHook postInstall ''; @@ -664,7 +666,8 @@ let "--override_repository=pypi_h5py=${h5py-bazel}" "--override_repository=pypi_wrapt=${wrapt-bazel}" "--override_repository=pypi_markupsafe=${markupsafe-bazel}" - "--override_repository=pypi_dm_tree=${dm-tree-bazel}" + "--override_repository=pypi_optree=${optree-bazel}" + "--override_repository=ml_dtypes=${ml_dtypes-fix}" "--override_repository=pypi=${pypi-bazel}" "--override_repository=python=${python-bazel}" "--override_repository=python_version_repo=${python_version_repo-bazel}" @@ -675,7 +678,6 @@ let ++ lib.optionals cudaSupport [ "--override_repository=nccl_archive=${nccl_archive-fix}" "--override_repository=eigen_archive=${eigen_archive-fix}" - "--override_repository=ml_dtypes=${ml_dtypes-fix}" ]; }) else @@ -699,7 +701,7 @@ let owner = "tensorflow"; repo = "tensorflow"; rev = "refs/tags/v${version}"; - hash = "sha256-yBDS9DFkVLIT+bu/vqh/0U60UwoIzqlTKQsflM7GWCA="; + hash = "sha256-dJGmoYdOTumFdS8AWWGYp5HRxk/WMKQNRi2ubyD/TAU="; }; # On update, it can be useful to steal the changes from gentoo @@ -717,6 +719,7 @@ let protobuf-extra wrapped_clang lndir + patchelf ] ++ lib.optional cudaSupport addDriverRunpath ++ lib.optional stdenv.isDarwin xcbuild @@ -849,6 +852,16 @@ let url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch"; hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w="; }) + # this commit causes a libprotobuf error in: + # src/google/protobuf/descriptor_database.cc:642 + # File already exists in database: + # tensorflow/core/tpu/kernels/sparse_core_layout.proto + (fetchpatch { + name = "fix-sparse-core-layout-proto-duplicate-loading.patch"; + url = "https://github.com/tensorflow/tensorflow/commit/1d36138fd0204daa48d97d66e415f46ba91f0697.diff"; + hash = "sha256-1rzz9zNTQVdKMI3ByeNT/CdLhOiCaYSrDE+/C31ekig="; + revert = true; + }) ./fix-syslib-references.patch ./protobuf_lite.patch ./protobuf_cc_toolchain.patch @@ -875,6 +888,8 @@ let rm -f .bazelversion patchShebangs . ln -s ${./riegeli-proto.patch} third_party/riegeli-proto.patch + substituteInPlace tensorflow/tools/pip_package/build_pip_package.py \ + --replace-fail '"patchelf"' '"${patchelf}/bin/patchelf"' '' + lib.optionalString (!stdenv.isDarwin) '' sed \ @@ -952,7 +967,7 @@ let ] ++ lib.optionals (mklSupport) [ "--config=mkl" ]; bazelTargets = [ - "//tensorflow/tools/pip_package:build_pip_package //tensorflow/tools/lib_package:libtensorflow" + "//tensorflow/tools/pip_package:wheel //tensorflow/tools/lib_package:libtensorflow" ]; removeRulesCC = false; @@ -977,7 +992,8 @@ let rm -rf external/pypi_h5py/site-packages rm -rf external/pypi_wrapt/site-packages rm -rf external/pypi_markupsafe/site-packages - rm -rf external/pypi_dm_tree/site-packages + rm -rf external/pypi_optree/site-packages + rm -rf external/pypi_ml_dtypes/site-packages '' + lib.optionalString (stdenv.isAarch64) '' rm -rf external/pypi_psutil/site-packages @@ -1015,7 +1031,7 @@ let if cudaSupport then "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" else - "sha256-SlIIHoD4mLcUbcjkXch/NFA4khbHsj19v05Z6xMX0/Q="; + "sha256-bg3ii3T/Ic01xV73lSA2SYVmDIkNCasQo6fEhk9lVVk="; aarch64-linux = if cudaSupport then "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" @@ -1038,7 +1054,7 @@ let # need to rebuild schemas since we use a different flatbuffers version preBuild = '' - (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema.fbs) + (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c schema_v3c.fbs) (cd tensorflow/lite/schema;${flatbuffers-core}/bin/flatc --gen-object-api -c conversion_metadata.fbs) (cd tensorflow/lite/acceleration/configuration;${flatbuffers-core}/bin/flatc -o configuration.fbs --proto configuration.proto) sed -i s,tflite.proto,tflite,g tensorflow/lite/acceleration/configuration/configuration.fbs/configuration.fbs @@ -1066,10 +1082,9 @@ let Cflags: -I$out/include/tensorflow EOF - # build the source code, then copy it to $python (build_pip_package - # actually builds a symlink farm so we must dereference them). - bazel-bin/tensorflow/tools/pip_package/build_pip_package --src "$PWD/dist" - cp -Lr "$PWD/dist" "$python" + mkdir "$python" + cp bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-${version}-*.whl \ + "$python/${pname}-${version}.whl" ''; postFixup = lib.optionalString cudaSupport '' @@ -1108,7 +1123,29 @@ buildPythonPackage { inherit version pname; disabled = pythonOlder "3.9"; - src = bazel-build.python; + format = "other"; + + src = "${bazel-build.python}/${pname}-${version}.whl"; + + sourceRoot = "."; + + unpackPhase = '' + runHook preUnpack + ${unzip}/bin/unzip $src + runHook postUnpack + ''; + + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + mkdir -p $out/${python.sitePackages} + cp -r $sourceRoot/tensorflow* $out/${python.sitePackages} + + runHook postInstall + ''; # Adjust dependency requirements: # - Drop tensorflow-io dependency until we get it to build @@ -1127,14 +1164,6 @@ buildPythonPackage { "keras" ]; - # Upstream has a pip hack that results in bin/tensorboard being in both tensorflow - # and the propagated input tensorboard, which causes environment collisions. - # Another possibility would be to have tensorboard only in the buildInputs - # https://github.com/tensorflow/tensorflow/blob/v1.7.1/tensorflow/tools/pip_package/setup.py#L79 - postInstall = '' - rm $out/bin/tensorboard - ''; - setupPyGlobalFlags = [ "--project_name ${pname}" ]; # tensorflow/tools/pip_package/setup.py @@ -1189,8 +1218,13 @@ buildPythonPackage { # TODO better test (files in tensorflow/tools/ci_build/builds/*test) # TEST_PACKAGES in tensorflow/tools/pip_package/setup.py nativeCheckInputs = [ + curl dill + double-conversion + grpc + jsoncpp portpicker + snappy tblib ]; checkPhase = '' diff --git a/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch b/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch index 95e3b85b0a08be..b472a7f88f0f9d 100644 --- a/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch +++ b/pkgs/development/python-modules/tensorflow/fix-syslib-references.patch @@ -1,7 +1,7 @@ diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/BUILD --- x/tensorflow/tools/pip_package/BUILD 2023-10-14 20:09:43.001857258 +0000 +++ y/tensorflow/tools/pip_package/BUILD 2023-10-14 20:15:17.894102008 +0000 -@@ -209,7 +209,7 @@ +@@ -89,13 +89,12 @@ "@com_google_protobuf//:LICENSE", "@curl//:COPYING", "@ducc//:LICENSE", @@ -10,3 +10,9 @@ diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/ "@gemmlowp//:LICENSE", "@libjpeg_turbo//:LICENSE.md", "@llvm-project//llvm:LICENSE.TXT", + "@llvm-project//mlir:LICENSE.TXT", + "@local_config_tensorrt//:LICENSE", +- "@ml_dtypes//:LICENSE", + "@org_brotli//:LICENSE", + "@pasta//:LICENSE", + "@png//:LICENSE", diff --git a/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch b/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch index 1b63851f095a11..fd4b282e0dc316 100644 --- a/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch +++ b/pkgs/development/python-modules/tensorflow/protobuf_cc_toolchain.patch @@ -1,17 +1,20 @@ diff -Naurd x/third_party/systemlibs/protobuf.BUILD y/third_party/systemlibs/protobuf.BUILD --- x/third_party/systemlibs/protobuf.BUILD 2023-12-03 15:39:27.742087246 +0000 +++ y/third_party/systemlibs/protobuf.BUILD 2023-12-04 13:25:41.777638732 +0000 -@@ -1,4 +1,8 @@ +@@ -4,7 +4,11 @@ + "proto_gen", + "py_proto_library", + ) -load("@rules_proto//proto:defs.bzl", "proto_library") +load( + "@rules_proto//proto:defs.bzl", + "proto_lang_toolchain", + "proto_library", +) - load( - "@com_google_protobuf//:protobuf.bzl", - "cc_proto_library", -@@ -118,3 +122,13 @@ + + licenses(["notice"]) + +@@ -124,3 +128,13 @@ srcs = [proto[1][0] for proto in WELL_KNOWN_PROTO_MAP.items()], visibility = ["//visibility:public"], ) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index adb8330c7a7dae..cf5c1abd2c24aa 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -15326,7 +15326,7 @@ self: super: with self; { tensorflow-build = let compat = rec { - abseil-cppTF = pkgs.abseil-cpp_202301; + abseil-cppTF = pkgs.abseil-cpp_202401; protobufTF = pkgs.protobuf_21.override { abseil-cpp = abseil-cppTF; };