Skip to content

Commit

Permalink
libtensorflow: 2.13.0 -> 2.14.0
Browse files Browse the repository at this point in the history
  • Loading branch information
Isidor Zeuner committed Jan 1, 2024
1 parent 87e5690 commit 6a99e95
Show file tree
Hide file tree
Showing 12 changed files with 211 additions and 427 deletions.

This file was deleted.

389 changes: 0 additions & 389 deletions pkgs/development/python-modules/tensorflow/absl_to_std.patch

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.absl.base.BUILD
--- x/third_party/absl/system.absl.base.BUILD 2023-09-17 09:12:05.499753364 +0000
+++ y/third_party/absl/system.absl.base.BUILD 2023-09-17 09:16:01.200082822 +0000
@@ -22,7 +22,12 @@
@@ -22,7 +22,13 @@

cc_library(
name = "raw_logging_internal",
Expand All @@ -11,6 +11,7 @@ diff -Naurd x/third_party/absl/system.absl.base.BUILD y/third_party/absl/system.
+ "-labsl_log_internal_conditions",
+ "-labsl_log_internal_message",
+ "-labsl_log_internal_nullguard",
+ "-labsl_log_internal_check_op",
+ ],
visibility = [
"//absl:__subpackages__",
Expand Down
14 changes: 14 additions & 0 deletions pkgs/development/python-modules/tensorflow/compute-links.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
diff -Naurd x/tensorflow/tensorflow.bzl y/tensorflow/tensorflow.bzl
--- x/tensorflow/tensorflow.bzl 2023-12-13 21:33:03.430643611 +0000
+++ y/tensorflow/tensorflow.bzl 2023-12-15 17:24:41.445609613 +0000
@@ -960,8 +960,9 @@
outs = [src],
srcs = [dest],
output_to_bindir = 1,
- cmd = "ln -sf $$(realpath --relative-to=$(RULEDIR) $<) $@",
+ cmd = "ln -sf $$($(location :compute-relative.sh) $(RULEDIR) $<) $@",
visibility = visibility,
+ tools = [":compute-relative.sh"],
)

def _get_shared_library_name_os_version_matrix(name, per_os_targets = False, version = None):
12 changes: 12 additions & 0 deletions pkgs/development/python-modules/tensorflow/compute-relative.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!@bash@
base="$1/"
target="$2"
while true
do echo "$base" | grep -q / || break
base_component="`echo "$base" | sed s,/.*,,`"
target_component="`echo "$target" | sed s,/.*,,`"
test "X$base_component" != "X$target_component" && break
base="`echo "$base" | sed "s<^$base_component/<<"`"
target="`echo "$target" | sed "s<^$target_component/<<"`"
done
echo "$target"
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff -Naurd x/WORKSPACE y/WORKSPACE
--- x/WORKSPACE 2023-11-16 08:55:37.370373002 +0000
+++ y/WORKSPACE 2023-11-20 01:55:57.812931627 +0000
@@ -17,6 +17,9 @@
sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841",
strip_prefix = "rules_python-0.23.1",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz",
+ patches = [
+ "//tensorflow:rules-python-use-nix.patch"
+ ],
)

load("@rules_python//python:repositories.bzl", "python_register_toolchains")
64 changes: 38 additions & 26 deletions pkgs/development/python-modules/tensorflow/default.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{ stdenv, bazel_5, buildBazelPackage, lib, fetchFromGitHub, symlinkJoin
{ stdenv, bazel_6, buildBazelPackage, lib, fetchFromGitHub, symlinkJoin
, addOpenGLRunpath, fetchpatch, fetchzip, linkFarm
# Python deps
, buildPythonPackage, pythonOlder, python
# Python libraries
, numpy, tensorboard, abseil-cpp, absl-py
, packaging, setuptools, wheel, keras-preprocessing, google-pasta
, opt-einsum, astunparse, h5py
, opt-einsum, astunparse, h5py, ml-dtypes
, termcolor, grpcio, six, wrapt, protobuf-python, tensorflow-estimator-bin
, dill, flatbuffers-python, portpicker, tblib, typing-extensions
# Common deps
, git, pybind11, which, binutils, glibcLocales, cython, perl
, git, pybind11, which, binutils, glibcLocales, cython, perl, clang, lndir
# Common libraries
, jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp, nsync
, curl, snappy, flatbuffers-core, icu, double-conversion, libpng, libjpeg_turbo, giflib, protobuf-core
Expand All @@ -29,7 +29,7 @@
, avx2Support ? stdenv.hostPlatform.avx2Support
, fmaSupport ? stdenv.hostPlatform.fmaSupport
# Darwin deps
, Foundation, Security, cctools, llvmPackages_11
, Foundation, Security, cctools, llvmPackages_16
}:

let
Expand All @@ -51,13 +51,13 @@ let
# translation units, so the build fails at link time
stdenv =
if cudaSupport then cudaPackagesGoogle.backendStdenv
else if originalStdenv.isDarwin then llvmPackages_11.stdenv
else if originalStdenv.isDarwin then llvmPackages_16.stdenv
else originalStdenv;
inherit (cudaPackagesGoogle) cudatoolkit nccl;
# use compatible cuDNN (https://www.tensorflow.org/install/source#gpu)
# cudaPackages.cudnn led to this:
# https://github.com/tensorflow/tensorflow/issues/60398
cudnnAttribute = "cudnn_8_6";
cudnnAttribute = "cudnn_8_7";
cudnn = cudaPackagesGoogle.${cudnnAttribute};
gentoo-patches = fetchzip {
url = "https://dev.gentoo.org/~perfinion/patches/tensorflow-patches-2.12.0.tar.bz2";
Expand Down Expand Up @@ -103,7 +103,7 @@ let

tfFeature = x: if x then "1" else "0";

version = "2.13.0";
version = "2.14.0";
format = "setuptools";
variant = lib.optionalString cudaSupport "-gpu";
pname = "tensorflow${variant}";
Expand Down Expand Up @@ -207,20 +207,20 @@ let

_bazel-build = buildBazelPackage.override { inherit stdenv; } {
name = "${pname}-${version}";
bazel = bazel_5;
bazel = bazel_6;

src = fetchFromGitHub {
owner = "tensorflow";
repo = "tensorflow";
rev = "refs/tags/v${version}";
hash = "sha256-Rq5pAVmxlWBVnph20fkAwbfy+iuBNlfFy14poDPd5h0=";
hash = "sha256-OvYb1YkYT9xeUGz3yBRdgNd/0s4YNhXPlw7yOR7pxB0=";
};

# On update, it can be useful to steal the changes from gentoo
# https://gitweb.gentoo.org/repo/gentoo.git/tree/sci-libs/tensorflow

nativeBuildInputs = [
which pythonEnv cython perl protobuf-core protobuf-extra
which pythonEnv cython perl protobuf-core protobuf-extra clang lndir
] ++ lib.optional cudaSupport addOpenGLRunpath;

buildInputs = [
Expand Down Expand Up @@ -343,19 +343,29 @@ let
url = "https://raw.githubusercontent.com/conda-forge/tensorflow-feedstock/0a63c5a962451b4da99a9948323d8b3ed462f461/recipe/patches/0001-Omit-linking-to-layout_proto_cc-if-protobuf-linkage-.patch";
hash = "sha256-/7buV6DinKnrgfqbe7KKSh9rCebeQdXv2Uj+Xg/083w=";
})
./fix-syslib-references.patch
./pybind11-osx.patch
./com_google_absl_add_log.patch
./absl_py_argparse_flags.patch
./core-rules-python-use-nix.patch
./protobuf_python.patch
./pybind11_protobuf_python_runtime_dep.patch
./pybind11_protobuf_newer_version.patch
] ++ lib.optionals (stdenv.hostPlatform.system == "aarch64-darwin") [
./absl_to_std.patch
# the implementation from upstream using realpath breaks in the sandbox
./compute-links.patch
];

postPatch = ''
# bazel 3.3 should work just as well as bazel 3.1
rm -f .bazelversion
patchShebangs .
sed 's,@bash@,${stdenv.shell},g' < ${./compute-relative.sh} \
> tensorflow/compute-relative.sh
chmod +x tensorflow/compute-relative.sh
sed \
-e 's,@python@,${python},g' \
-e 's,@python_version@,${python.version},g' \
< ${./rules-python-use-nix.patch} \
> tensorflow/rules-python-use-nix.patch
'' + lib.optionalString (stdenv.hostPlatform.system == "x86_64-darwin") ''
cat ${./com_google_absl_fix_macos.patch} >> third_party/absl/com_google_absl_fix_mac_and_nvcc_build.patch
'' + lib.optionalString (!withTensorboard) ''
Expand All @@ -382,12 +392,10 @@ let
chmod +x dummy-ldconfig/ldconfig
export PATH="$PWD/dummy-ldconfig:$PATH"
export PYTHONPATH="${pythonEnv}/lib/${pythonEnv.libPrefix}/site-packages"
export PYTHON_LIB_PATH="$NIX_BUILD_TOP/site-packages"
export CC_OPT_FLAGS="${lib.concatStringsSep " " opt_flags}"
mkdir -p "$PYTHON_LIB_PATH"
# To avoid mixing Python 2 and Python 3
unset PYTHONPATH
'';

configurePhase = ''
Expand All @@ -400,8 +408,6 @@ let

bazelBuildFlags = [
"--config=opt" # optimize using the flags set in the configure phase
]
++ lib.optionals stdenv.cc.isClang [
"--cxxopt=-x" "--cxxopt=c++"
"--host_cxxopt=-x" "--host_cxxopt=c++"

Expand All @@ -419,13 +425,13 @@ let
fetchAttrs = {
sha256 = {
x86_64-linux = if cudaSupport
then "sha256-5VFMNHeLrUxW5RTr6EhT3pay9nWJ5JkZTGirDds5QkU="
else "sha256-KzgWV69Btr84FdwQ5JI2nQEsqiPg1/+TWdbw5bmxXOE=";
then "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
else "sha256-MW0cdMCBTSRWzAM1+3mYf8m4TyT1iYR9GPpI5s/JHcU=";
aarch64-linux = if cudaSupport
then "sha256-ty5+51BwHWE1xR4/0WcWTp608NzSAS/iiyN+9zx7/wI="
else "sha256-9btXrNHqd720oXTPDhSmFidv5iaZRLjCVX8opmrMjXk=";
x86_64-darwin = "sha256-gqb03kB0z2pZQ6m1fyRp1/Nbt8AVVHWpOJSeZNCLc4w=";
aarch64-darwin = "sha256-WdgAaFZU+ePwWkVBhLzjlNT7ELfGHOTaMdafcAMD5yo=";
then "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
else "sha256-m+4GrkyFT/e0UViSwfPZa4kkf3AgAsNJPx30Qazqi6E=";
x86_64-darwin = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
aarch64-darwin = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
}.${stdenv.hostPlatform.system} or (throw "unsupported system ${stdenv.hostPlatform.system}");
};

Expand Down Expand Up @@ -487,7 +493,8 @@ let
maintainers = with maintainers; [ abbradar ];
platforms = with platforms; linux ++ darwin;
broken =
stdenv.isDarwin
#stdenv.isDarwin
false
|| !(xlaSupport -> cudaSupport)
|| !(cudaSupport -> builtins.hasAttr cudnnAttribute cudaPackagesGoogle)
|| !(cudaSupport -> cudaPackagesGoogle ? cudatoolkit);
Expand All @@ -499,10 +506,14 @@ let

in buildPythonPackage {
inherit version pname;
disabled = pythonOlder "3.8";
disabled = pythonOlder "3.9";

src = bazel-build.python;

patches = [
./remove-keras-dependency.patch
];

# Adjust dependency requirements:
# - Drop tensorflow-io dependency until we get it to build
# - Relax flatbuffers and gast version requirements
Expand Down Expand Up @@ -540,6 +551,7 @@ in buildPythonPackage {
grpcio
h5py
keras-preprocessing
ml-dtypes
numpy
opt-einsum
packaging
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff -Naurd x/tensorflow/tools/pip_package/BUILD y/tensorflow/tools/pip_package/BUILD
--- x/tensorflow/tools/pip_package/BUILD 2023-10-14 20:09:43.001857258 +0000
+++ y/tensorflow/tools/pip_package/BUILD 2023-10-14 20:15:17.894102008 +0000
@@ -208,7 +208,7 @@
"@com_google_protobuf//:LICENSE",
"@curl//:COPYING",
"@dill_archive//:LICENSE",
- "@flatbuffers//:LICENSE",
+ "@flatbuffers//:LICENSE.txt",
"@gast_archive//:PKG-INFO",
"@gemmlowp//:LICENSE",
"@libjpeg_turbo//:LICENSE.md",
12 changes: 12 additions & 0 deletions pkgs/development/python-modules/tensorflow/pybind11-osx.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
diff -Naurd x/third_party/systemlibs/pybind11.BUILD y/third_party/systemlibs/pybind11.BUILD
--- x/third_party/systemlibs/pybind11.BUILD 2023-10-17 19:01:11.988196127 +0000
+++ y/third_party/systemlibs/pybind11.BUILD 2023-10-17 19:04:09.916356016 +0000
@@ -6,3 +6,8 @@
"@org_tensorflow//third_party/python_runtime:headers",
],
)
+
+config_setting(
+ name = "osx",
+ constraint_values = ["@platforms//os:osx"],
+)
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff -Naurd x/setup.py y/setup.py
--- x/setup.py 2023-12-01 20:07:55.694564265 +0000
+++ y/setup.py 2023-12-01 20:09:08.526682512 +0000
@@ -121,9 +121,6 @@
'tensorflow_estimator >= 2.14.0, < 2.15',
'tf-estimator-nightly ~= 2.15.0.dev',
),
- standard_or_nightly(
- 'keras >= 2.14.0, < 2.15', 'keras-nightly ~= 2.15.0.dev'
- ),
]
REQUIRED_PACKAGES = [p for p in REQUIRED_PACKAGES if p is not None]

Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
diff -Naurd python/repositories.bzl python-use-nix/repositories.bzl
--- python/repositories.bzl 2023-11-29 23:07:55.169106050 +0000
+++ python-use-nix/repositories.bzl 2023-12-01 12:00:06.746231531 +0000
@@ -103,60 +103,22 @@
fail("Exactly one of (url, urls) must be set.")

platform = rctx.attr.platform
- python_version = rctx.attr.python_version
+ python_version = "@python_version@"
python_short_version = python_version.rpartition(".")[0]
release_filename = rctx.attr.release_filename
urls = rctx.attr.urls or [rctx.attr.url]

- if release_filename.endswith(".zst"):
- rctx.download(
- url = urls,
- sha256 = rctx.attr.sha256,
- output = release_filename,
- )
- unzstd = rctx.which("unzstd")
- if not unzstd:
- url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version)
- rctx.download_and_extract(
- url = url,
- sha256 = rctx.attr.zstd_sha256,
- )
- working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version)
- make_result = rctx.execute(
- ["make", "--jobs=4"],
- timeout = 600,
- quiet = True,
- working_directory = working_directory,
- )
- if make_result.return_code:
- fail_msg = (
- "Failed to compile 'zstd' from source for use in Python interpreter extraction. " +
- "'make' error message: {}".format(make_result.stderr)
- )
- fail(fail_msg)
- zstd = "{working_directory}/zstd".format(working_directory = working_directory)
- unzstd = "./unzstd"
- rctx.symlink(zstd, unzstd)
-
- exec_result = rctx.execute([
- "tar",
- "--extract",
- "--strip-components=2",
- "--use-compress-program={unzstd}".format(unzstd = unzstd),
- "--file={}".format(release_filename),
- ])
- if exec_result.return_code:
- fail_msg = (
- "Failed to extract Python interpreter from '{}'. ".format(release_filename) +
- "'tar' error message: {}".format(exec_result.stderr)
- )
- fail(fail_msg)
- else:
- rctx.download_and_extract(
- url = urls,
- sha256 = rctx.attr.sha256,
- stripPrefix = rctx.attr.strip_prefix,
+ lndir_result = rctx.execute([
+ "lndir",
+ "@python@",
+ ".",
+ ])
+ if lndir_result.return_code:
+ fail_msg = (
+ "Failed to link Python interpreter from '{}'. ".format("@python@") +
+ "'lndir' error message: {}".format(lndir_result.stderr)
)
+ fail(fail_msg)

patches = rctx.attr.patches
if patches:
Loading

0 comments on commit 6a99e95

Please sign in to comment.