Switch to using setuptools to build tfdv.

* Update README.md

* Update setup.py

* Update RELEASE.md

* Update install.md

* Update BUILD

* Update and rename build_pip_package.sh to move_generated_files.sh

* Update build_manylinux.sh

Author: dhruvesh09

README.md

@@ -111,18 +111,13 @@ branch), pass `-b <branchname>` to the `git clone` command.
 
 ### 3. Build the pip package
 
-TFDV uses Bazel to build the pip package from source. Before invoking the
-following commands, make sure the `python` in your `$PATH` is the one of the
-target version and has NumPy installed.
+`TFDV` wheel is Python version dependent -- to build the pip package that
+works for a specific Python version, use that Python binary to run:
 
 ```shell
-bazel run -c opt --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0 tensorflow_data_validation:build_pip_package
+python setup.py bdist_wheel
 ```
 
-Note that we are assuming here that dependent packages (e.g. PyArrow) are built
-with a GCC older than 5.1 and use the flag `D_GLIBCXX_USE_CXX11_ABI=0` to be
-[compatible with the old std::string ABI](https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html).
-
 You can find the generated `.whl` file in the `dist` subdirectory.
 
 ### 4. Install the pip package

g3doc/install.md

@@ -64,15 +64,14 @@ branch), pass `-b <branchname>` to the `git clone` command.
 
 ### 3. Build the pip package
 
-Then, run the following at the project root:
+`TFDV` wheel is Python version dependent -- to build the pip package that
+works for a specific Python version, use that Python binary to run:
 
-```bash
-sudo docker-compose build manylinux2010
-sudo docker-compose run -e PYTHON_VERSION=${PYTHON_VERSION} manylinux2010
+```shell
+python setup.py bdist_wheel
 ```
-where `PYTHON_VERSION` is one of `{35, 36, 37}`.
 
-A wheel will be produced under `dist/`.
+You can find the generated `.whl` file in the `dist` subdirectory.
 
 ### 4. Install the pip package
 

setup.py

@@ -13,16 +13,84 @@
 # limitations under the License.
 """Package Setup script for TensorFlow Data Validation."""
 
+# pylint:disable=g-bad-import-order
+# setuptools must be imported prior to distutils.
+import setuptools
+from distutils import spawn
+from distutils.command import build
+# pylint:enable=g-bad-import-order
+import os
+import platform
+import subprocess
+
 from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.install import install
 from setuptools.dist import Distribution
 
 
+class _BuildCommand(build.build):
+  """Build everything that is needed to install.
+
+  This overrides the original distutils "build" command to to run bazel_build
+  command before any sub_commands.
+
+  build command is also invoked from bdist_wheel and install command, therefore
+  this implementation covers the following commands:
+    - pip install . (which invokes bdist_wheel)
+    - python setup.py install (which invokes install command)
+    - python setup.py bdist_wheel (which invokes bdist_wheel command)
+  """
+
+  def _build_cc_extensions(self):
+    return True
+
+  # Add "bazel_build" command as the first sub_command of "build". Each
+  # sub_command of "build" (e.g. "build_py", "build_ext", etc.) is executed
+  # sequentially when running a "build" command, if the second item in the tuple
+  # (predicate method) is evaluated to true.
+  sub_commands = [
+      ('bazel_build', _build_cc_extensions),
+  ] + build.build.sub_commands
+
+
+class _BazelBuildCommand(setuptools.Command):
+  """Build TFDV C++ extensions and public protos with Bazel.
+
+  Running this command will populate foo_pb2.py file next to your foo.proto
+  file.
+  """
+
+  def initialize_options(self):
+    pass
+
+  def finalize_options(self):
+    self._bazel_cmd = spawn.find_executable('bazel')
+    if not self._bazel_cmd:
+      raise RuntimeError(
+          'Could not find "bazel" binary. Please visit '
+          'https://docs.bazel.build/versions/master/install.html for '
+          'installation instruction.')
+    self._additional_build_options = []
+    if platform.system() == 'Darwin':
+      self._additional_build_options = ['--macos_minimum_os=10.9']
+    elif platform.system() == 'Windows':
+      self._additional_build_options = ['--copt=-DWIN32_LEAN_AND_MEAN']
+
+  def run(self):
+    subprocess.check_call(
+        [self._bazel_cmd, 'run', '-c', 'opt'] + self._additional_build_options +
+        ['//tensorflow_data_validation:move_generated_files'],
+        # Bazel should be invoked in a directory containing bazel WORKSPACE
+        # file, which is the root directory.
+        cwd=os.path.dirname(os.path.realpath(__file__)),
+    )
+
+
 # TFDV is not a purelib. However because of the extension module is not built
 # by setuptools, it will be incorrectly treated as a purelib. The following
 # works around that bug.
-class _InstallPlatlib(install):
+class _InstallPlatlibCommand(install):
 
   def finalize_options(self):
     install.finalize_options(self)
@@ -57,7 +125,7 @@ def _make_all_extra_requirements():
 # Get version from version module.
 with open('tensorflow_data_validation/version.py') as fp:
   globals_dict = {}
-  exec (fp.read(), globals_dict)  # pylint: disable=exec-used
+  exec(fp.read(), globals_dict)  # pylint: disable=exec-used
 __version__ = globals_dict['__version__']
 
 # Get the long description from the README file.
@@ -81,7 +149,6 @@ def _make_all_extra_requirements():
         'Operating System :: Microsoft :: Windows',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3 :: Only',
@@ -129,5 +196,9 @@ def _make_all_extra_requirements():
     url='https://www.tensorflow.org/tfx/data_validation',
     download_url='https://github.com/tensorflow/data-validation/tags',
     requires=[],
-    cmdclass={'install': _InstallPlatlib})
+    cmdclass={
+        'install': _InstallPlatlibCommand,
+        'build': _BuildCommand,
+        'bazel_build': _BazelBuildCommand,
+    })
 

tensorflow_data_validation/BUILD

@@ -8,8 +8,8 @@ config_setting(
 )
 
 sh_binary(
-    name = "build_pip_package",
-    srcs = ["build_pip_package.sh"],
+    name = "move_generated_files",
+    srcs = ["move_generated_files.sh"],
     data = select({
         ":windows": [
             "//tensorflow_data_validation/anomalies/proto:validation_config_pb2.py",

tensorflow_data_validation/build_pip_package.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Moves the bazel generated files needed for packaging the wheel to the source
+# tree.
+
+function _is_windows() {
+  [[ "$(uname -s | tr 'A-Z' 'a-z')" =~ (cygwin|mingw32|mingw64|msys)_nt* ]]
+}
+
+function tfdv::move_generated_files() {
+  if _is_windows; then
+    # Newer bazel does not create bazel-genfiles any more (
+    # https://github.com/bazelbuild/bazel/issues/6761). It's merged with bazel-bin
+    GENFILES=bazel-genfiles
+    if [[ ! -d ${BUILD_WORKSPACE_DIRECTORY}/${GENFILES} ]]; then
+      GENFILES=bazel-bin
+    fi
+    PYWRAP_TFDV="tensorflow_data_validation/pywrap/tensorflow_data_validation_extension.pyd"
+    cp -f "${BUILD_WORKSPACE_DIRECTORY}/bazel-out/x64_windows-opt/bin/${PYWRAP_TFDV}" \
+      "${BUILD_WORKSPACE_DIRECTORY}/${PYWRAP_TFDV}"
+
+    cp -f ${BUILD_WORKSPACE_DIRECTORY}/${GENFILES}/tensorflow_data_validation/anomalies/proto/validation_config_pb2.py \
+      ${BUILD_WORKSPACE_DIRECTORY}/tensorflow_data_validation/anomalies/proto
+    cp -f ${BUILD_WORKSPACE_DIRECTORY}/${GENFILES}/tensorflow_data_validation/anomalies/proto/validation_metadata_pb2.py \
+      ${BUILD_WORKSPACE_DIRECTORY}/tensorflow_data_validation/anomalies/proto
+  else
+    PYWRAP_TFDV="tensorflow_data_validation/pywrap/tensorflow_data_validation_extension.so"
+    cp -f "${BUILD_WORKSPACE_DIRECTORY}/bazel-bin/${PYWRAP_TFDV}" \
+      "${BUILD_WORKSPACE_DIRECTORY}/${PYWRAP_TFDV}"
+
+    # If run by "bazel run", $(pwd) is the .runfiles dir that contains all the
+    # data dependencies.
+    RUNFILES_DIR=$(pwd)
+    cp -f ${RUNFILES_DIR}/tensorflow_data_validation/anomalies/proto/validation_config_pb2.py \
+      ${BUILD_WORKSPACE_DIRECTORY}/tensorflow_data_validation/anomalies/proto
+    cp -f ${RUNFILES_DIR}/tensorflow_data_validation/anomalies/proto/validation_metadata_pb2.py \
+      ${BUILD_WORKSPACE_DIRECTORY}/tensorflow_data_validation/anomalies/proto
+  fi
+  chmod +w "${BUILD_WORKSPACE_DIRECTORY}/${PYWRAP_TFDV}"
+}
+
+tfdv::move_generated_files

tensorflow_data_validation/move_generated_files.sh

@@ -55,13 +55,9 @@ function install_numpy() {
   ${PIP_BIN} install "numpy>=1.16,<2"
 }
 
-function bazel_build() {
+function build_wheel() {
   rm -rf dist
-  bazel run -c opt \
-    --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0 \
-    tensorflow_data_validation:build_pip_package \
-    --\
-    --python_bin_path "${PYTHON_BIN_PATH}"
+  "${PYTHON_BIN_PATH}" setup.py bdist_wheel
 }
 
 function stamp_wheel() {
@@ -72,11 +68,8 @@ function stamp_wheel() {
   rm "${WHEEL_PATH}"
 }
 
-setup_environment
-set -e
 set -x
-install_numpy
-bazel_build
+setup_environment && \
+install_numpy && \
+build_wheel && \
 stamp_wheel
-set +e
-set +x