[2/4] Support for macOS - Fix python code build

google-deepmind · Nov 21, 2020 · b6063a3 · b6063a3
1 parent 5786912
commit b6063a3
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 35 deletions.
diff --git a/configure.py b/configure.py
@@ -70,7 +70,7 @@ def main():
   reset_configure_bazelrc()
   setup_python(environ_cp)
 
-  write_to_bazelrc('\n')
+  write_to_bazelrc('')
   if platform.system() == 'Darwin':
     write_to_bazelrc('# https://github.com/googleapis/google-cloud-cpp-spanner/issues/1003')
     write_to_bazelrc('build --copt=-DGRPC_BAZEL_BUILD')

diff --git a/oss_build.sh b/oss_build.sh
@@ -87,17 +87,39 @@ for python_version in $PYTHON_VERSIONS; do
     bazel clean
   fi
 
-  if [ "$python_version" = "3.6" ]; then
-    export PYTHON_BIN_PATH=/usr/bin/python3.6 && export PYTHON_LIB_PATH=/usr/local/lib/python3.6/dist-packages
-  elif [ "$python_version" = "3.7" ]; then
-    export PYTHON_BIN_PATH=/usr/local/bin/python3.7 && export PYTHON_LIB_PATH=/usr/local/lib/python3.7/dist-packages
-    ABI=cp37
-  elif [ "$python_version" = "3.8" ]; then
-    export PYTHON_BIN_PATH=/usr/bin/python3.8 && export PYTHON_LIB_PATH=/usr/local/lib/python3.8/dist-packages
-    ABI=cp38
+  if [ "$(uname)" = "Darwin" ]; then
+    if [ "$python_version" = "3.6" ]; then
+      export PYTHON_BIN_PATH=python3.6
+    elif [ "$python_version" = "3.7" ]; then
+      export PYTHON_BIN_PATH=python3.7
+      ABI=cp37
+    elif [ "$python_version" = "3.8" ]; then
+      export PYTHON_BIN_PATH=python3.8
+      ABI=cp38
+    else
+      echo "Error unknown --python. Only [3.6|3.7|3.8]"
+      exit
+    fi
+
+    bazel_config=""
+    version=`sw_vers | grep ProductVersion | awk '{print $2}' | sed 's/\./_/g' | cut -d"_" -f1,2`
+    PLATFORM="macosx_${version}_x86_64"
   else
-    echo "Error unknown --python. Only [3.6|3.7|3.8]"
-    exit
+    if [ "$python_version" = "3.6" ]; then
+      export PYTHON_BIN_PATH=/usr/bin/python3.6 && export PYTHON_LIB_PATH=/usr/local/lib/python3.6/dist-packages
+    elif [ "$python_version" = "3.7" ]; then
+      export PYTHON_BIN_PATH=/usr/local/bin/python3.7 && export PYTHON_LIB_PATH=/usr/local/lib/python3.7/dist-packages
+      ABI=cp37
+    elif [ "$python_version" = "3.8" ]; then
+      export PYTHON_BIN_PATH=/usr/bin/python3.8 && export PYTHON_LIB_PATH=/usr/local/lib/python3.8/dist-packages
+      ABI=cp38
+    else
+      echo "Error unknown --python. Only [3.6|3.7|3.8]"
+      exit
+    fi
+
+    bazel_config="--config=manylinux2010"
+    PLATFORM="manylinux2010_x86_64"
   fi
 
   # Configures Bazel environment for selected Python version.
@@ -110,11 +132,11 @@ for python_version in $PYTHON_VERSIONS; do
   # someone's system unexpectedly. We are executing the python tests after
   # installing the final package making this approach satisfactory.
   # TODO(b/157223742): Execute Python tests as well.
-  bazel test -c opt --copt=-mavx --config=manylinux2010 --test_output=errors //reverb/cc/...
+  bazel test -c opt --copt=-mavx $bazel_config --test_output=errors //reverb/cc/...
 
-  # Builds Reverb and creates the wheel package.
-  bazel build -c opt --copt=-mavx --config=manylinux2010 reverb/pip_package:build_pip_package
-  ./bazel-bin/reverb/pip_package/build_pip_package --dst $OUTPUT_DIR $PIP_PKG_EXTRA_ARGS
+  # # Builds Reverb and creates the wheel package.
+  bazel build -c opt --copt=-mavx $bazel_config reverb/pip_package:build_pip_package
+  ./bazel-bin/reverb/pip_package/build_pip_package --dst $OUTPUT_DIR $PIP_PKG_EXTRA_ARGS --plat "$PLATFORM"
 
   # Installs pip package.
   $PYTHON_BIN_PATH -mpip install ${OUTPUT_DIR}*${ABI}*.whl
@@ -123,7 +145,7 @@ for python_version in $PYTHON_VERSIONS; do
     echo "Run Python tests..."
     set +e
 
-    bash run_python_tests.sh |& tee ./unittest_log.txt
+    bash run_python_tests.sh 2>&1| tee ./unittest_log.txt
     UNIT_TEST_ERROR_CODE=$?
     set -e
     if [[ $UNIT_TEST_ERROR_CODE != 0 ]]; then

diff --git a/reverb/cc/platform/default/build_rules.bzl b/reverb/cc/platform/default/build_rules.bzl
@@ -309,9 +309,16 @@ def reverb_gen_op_wrapper_py(name, out, kernel_lib, linkopts = [], **kwargs):
             "-fvisibility=hidden",  # avoid symbol clashes between DSOs.
         ],
         linkshared = 1,
-        linkopts = linkopts + _rpath_linkopts(module_name) + [
-            "$(location %s)" % version_script_file,
-        ],
+        linkopts = linkopts + _rpath_linkopts(module_name) + select({
+            "//reverb:macos": [
+                "-Wl",
+                "$(location %s)" % version_script_file,
+            ],
+            "//conditions:default": [
+                "-Wl,--version-script",
+                "$(location %s)" % version_script_file,
+            ],
+        }),
         **kwargs
     )
     native.genrule(
@@ -358,14 +365,7 @@ def _rpath_linkopts(name):
     # ops) are picked up as long as they are in either the same or a parent
     # directory in the tensorflow/ tree.
     levels_to_root = native.package_name().count("/") + name.count("/")
-    return select({
-        "//reverb:macos": [
-            "-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
-        ],
-        "//conditions:default": [
-            "-Wl,--version-script,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
-        ],
-    })
+    return ["-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),)]
 
 def reverb_pybind_extension(
         name,
@@ -447,9 +447,15 @@ def reverb_pybind_extension(
             "-fexceptions",  # pybind relies on exceptions, required to compile.
             "-fvisibility=hidden",  # avoid pybind symbol clashes between DSOs.
         ],
-        linkopts = linkopts + _rpath_linkopts(module_name) + [
-            "$(location %s)" % version_script_file,
-        ],
+        linkopts = linkopts + _rpath_linkopts(module_name) +
+            select({"//reverb:macos": [
+                        "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file,
+                    ],
+                    "//conditions:default": [
+                        "-Wl,--version-script",
+                        "$(location %s)" % version_script_file,
+                    ],
+        }),
         deps = depset(deps + [
             exported_symbols_file,
             version_script_file,

diff --git a/reverb/cc/platform/default/repo.bzl b/reverb/cc/platform/default/repo.bzl
@@ -90,9 +90,6 @@ def _find_python_solib_path(repo_ctx):
             .format(exec_result.stderr))
     version = exec_result.stdout.splitlines()[-1]
     basename = "lib{}.so".format(version)
-    if repo_ctx.os.name.lower().find("mac") != -1:
-        basename = "lib{}m.a".format(version)
-
     exec_result = repo_ctx.execute(
         ["{}-config".format(version), "--configdir"],
         quiet = True,
@@ -101,6 +98,10 @@ def _find_python_solib_path(repo_ctx):
         fail("Could not locate python shared library path:\n{}"
             .format(exec_result.stderr))
     solib_dir = exec_result.stdout.splitlines()[-1]
+    if repo_ctx.os.name.lower().find("mac") != -1:
+        basename = "lib{}m.dylib".format(version)
+        solib_dir = "/".join(solib_dir.split("/")[:-2])
+
     full_path = repo_ctx.path("{}/{}".format(solib_dir, basename))
     if not full_path.exists:
         fail("Unable to find python shared library file:\n{}/{}"
@@ -222,7 +223,7 @@ filegroup(
 def _tensorflow_solib_repo_impl(repo_ctx):
     tf_lib_path = _find_tf_lib_path(repo_ctx)
     repo_ctx.symlink(tf_lib_path, "tensorflow_solib")
-    suffix = "2.so"
+    suffix = "so.2"
     if repo_ctx.os.name.lower().find("mac") != -1:
         suffix = "2.dylib"
 

diff --git a/reverb/pip_package/build_pip_package.sh b/reverb/pip_package/build_pip_package.sh
@@ -32,7 +32,7 @@ function build_wheel() {
   pushd ${TMPDIR} > /dev/null
 
   echo $(date) : "=== Building wheel"
-  "${PYTHON_BIN_PATH}" setup.py bdist_wheel ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} --plat manylinux2010_x86_64 > /dev/null
+  "${PYTHON_BIN_PATH}" setup.py bdist_wheel ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} --plat $PLATFORM > /dev/null
   DEST=${TMPDIR}/dist/
   if [[ ! "$TMPDIR" -ef "$DESTDIR" ]]; then
     mkdir -p ${DESTDIR}
@@ -80,6 +80,7 @@ function usage() {
   echo "    --release         build a release version"
   echo "    --dst             path to copy the .whl into."
   echo "    --tf-version      tensorflow version dependency passed to setup.py."
+  echo "    --plat            platform."
   echo ""
   exit 1
 }
@@ -91,6 +92,8 @@ function main() {
   # This is where the source code is copied and where the whl will be built.
   DST_DIR=""
 
+  PLATFORM="manylinux2010_x86_64"
+
   while true; do
     if [[ "$1" == "--help" ]]; then
       usage
@@ -103,6 +106,9 @@ function main() {
     elif [[ "$1" == "--tf-version" ]]; then
       shift
       TF_VERSION_FLAG="--tf-version $1"
+    elif [[ "$1" == "--plat" ]]; then
+      shift
+      PLATFORM=$1
     fi
 
     if [[ -z "$1" ]]; then