Split TensorFlow Privacy pypi package into two parts-- one for DP tra…

…ining, and one for privacy tests.

PiperOrigin-RevId: 603100586

pip_tools/build_empirical_pip_package.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright 2020, The TensorFlow Privacy Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tool to build the TensorFlow Privacy/Privacy Tests pip package.
+set -e
+
+main() {
+  # Create a working directory.
+  local temp_dir="$(mktemp -d)"
+  trap "rm -rf ${temp_dir}" EXIT
+
+  # Create a virtual environment
+  python3.11 -m venv "${temp_dir}/venv"
+  source "${temp_dir}/venv/bin/activate"
+  python --version
+  pip install --upgrade pip
+  pip --version
+
+  # Build the pip package
+  pip install --upgrade setuptools wheel
+  python "setup_empirical.py" sdist bdist_wheel --dist-dir=privacy_tests/dist
+  cd privacy_tests
+
+  # Cleanup.
+  deactivate
+}
+
+main "$@"

pip_tools/publish_pip_package.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2020, The TensorFlow Privacy Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tool to publish the TensorFlow Privacy pip package.
+set -e
+
+main() {
+  # Create a working directory.
+  local temp_dir="$(mktemp -d)"
+  trap "rm -rf ${temp_dir}" EXIT
+
+  # Create a virtual environment
+  python3.11 -m venv "${temp_dir}/venv"
+  source "${temp_dir}/venv/bin/activate"
+  python --version
+  pip install --upgrade pip
+  pip --version
+
+  # Publish the pip package.
+  package="$(ls "dist/"*".whl" | head -n1)"
+  pip install --upgrade twine
+  twine check "${package}"
+  twine upload "${package}"
+
+  # Cleanup.
+  deactivate
+}
+
+main "$@"

pip_tools/test_empirical_pip_package.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2020, The TensorFlow Privacy Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tool to build the TensorFlow Privacy pip package.
+set -e
+
+main() {
+  # Create a working directory.
+  local temp_dir="$(mktemp -d)"
+  trap "rm -rf ${temp_dir}" EXIT
+
+  # Create a virtual environment
+  python3.11 -m venv "${temp_dir}/venv"
+  source "${temp_dir}/venv/bin/activate"
+  python --version
+  pip install --upgrade pip
+  pip --version
+
+  # Test the pip package.
+  package="$(ls "dist/"*".whl" | head -n1)"
+  pip install --upgrade "${package}"
+  pip freeze
+  python -c "import privacy_tests as pt; print(pt.__version__)"
+
+  # Cleanup.
+  deactivate
+}
+
+main "$@"

requirements.txt

@@ -26,7 +26,6 @@
 # `~=x.0`.
 
 absl-py>=1.0,==1.*
-attrs>=21.4
 dm-tree==0.1.8
 dp-accounting==0.4.3
 immutabledict~=2.2

setup.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TensorFlow Privacy library setup file for pip."""
+"""TensorFlow Privacy/DP Training library setup file for pip."""
 
 import setuptools
 
@@ -33,24 +33,18 @@
     long_description_content_type='text/plain',
     url='https://github.com/tensorflow/privacy',
     license='Apache-2.0',
-    packages=setuptools.find_packages(),
+    packages=setuptools.find_packages(exclude=['*privacy.privacy_tests*']),
     install_requires=[
         'absl-py>=1.0,==1.*',
-        'attrs>=21.4',
         'dm-tree==0.1.8',
         'dp-accounting==0.4.3',
-        'immutabledict~=2.2',
-        'matplotlib~=3.3',
         'numpy~=1.21',
         'packaging~=22.0',
-        'pandas~=1.4',
         'scikit-learn>=1.0,==1.*',
         'scipy~=1.9',
-        'statsmodels==0.14.0',
         'tensorflow-estimator~=2.4',
         'tensorflow-probability~=0.22.0',
         'tensorflow~=2.4',
-        'tf-models-official~=2.13',
     ],
     python_requires='>=3.9.0,<3.12',
 )

tensorflow_privacy/privacy/membership_inference_attack/__init__.py

@@ -19,5 +19,5 @@
     "\nMembership inference attack sources were moved. Please replace"
     "\nimport tensorflow_privacy.privacy.membership_inference_attack\n"
     "\nwith"
-    "\nimport tensorflow_privacy.privacy.privacy_tests.membership_inference_attack"
+    "\nimport privacy_tests.membership_inference_attack"
 )

tensorflow_privacy/privacy/membership_inference_attack/data_structures.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.data_structures import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/dataset_slicing.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.dataset_slicing import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.dataset_slicing import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/keras_evaluation.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.keras_evaluation import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.keras_evaluation import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/membership_inference_attack.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.membership_inference_attack import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.membership_inference_attack import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/models.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.models import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.models import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/plotting.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.plotting import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.plotting import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/privacy_report.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.plotting import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.plotting import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/seq2seq_mia.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.seq2seq_mia import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.seq2seq_mia import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/membership_inference_attack/tf_estimator_evaluation.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 """Moved to privacy_attack/membership_inference_attack."""
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.tf_estimator_evaluation import *  # pylint: disable=wildcard-import
+from privacy_tests.membership_inference_attack.tf_estimator_evaluation import *  # pylint: disable=wildcard-import

tensorflow_privacy/privacy/privacy_tests/BUILD

@@ -5,6 +5,9 @@ licenses(["notice"])
 py_library(
     name = "privacy_tests",
     srcs = ["__init__.py"],
+    deps = [
+        ":version",
+    ],
 )
 
 py_test(
@@ -32,3 +35,9 @@ py_library(
     name = "epsilon_lower_bound",
     srcs = ["epsilon_lower_bound.py"],
 )
+
+py_library(
+    name = "version",
+    srcs = ["version.py"],
+    srcs_version = "PY3",
+)

tensorflow_privacy/privacy/privacy_tests/__init__.py

@@ -11,3 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""TensorFlow Empirical Privacy."""
+
+from privacy_tests.version import __version__  # pylint: disable=g-bad-import-order

tensorflow_privacy/privacy/privacy_tests/epsilon_lower_bound_test.py

@@ -15,7 +15,7 @@
 from absl.testing import absltest
 from absl.testing import parameterized
 import numpy as np
-from tensorflow_privacy.privacy.privacy_tests import epsilon_lower_bound as elb
+from privacy_tests import epsilon_lower_bound as elb
 
 
 class TPFPTest(parameterized.TestCase):

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/advanced_mia.py

@@ -17,7 +17,7 @@
 from typing import Optional, Sequence, Union
 import numpy as np
 import scipy.stats
-from tensorflow_privacy.privacy.privacy_tests.utils import log_loss
+from privacy_tests.utils import log_loss
 
 
 def replace_nan_with_column_mean(a: np.ndarray):

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/advanced_mia_example.py

@@ -23,11 +23,11 @@
 import matplotlib.pyplot as plt
 import numpy as np
 import tensorflow as tf
-from tensorflow_privacy.privacy.privacy_tests import utils
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import advanced_mia as amia
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import membership_inference_attack as mia
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import plotting as mia_plotting
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackInputData
+from privacy_tests import utils
+from privacy_tests.membership_inference_attack import advanced_mia as amia
+from privacy_tests.membership_inference_attack import membership_inference_attack as mia
+from privacy_tests.membership_inference_attack import plotting as mia_plotting
+from privacy_tests.membership_inference_attack.data_structures import AttackInputData
 
 FLAGS = flags.FLAGS
 _LR = flags.DEFINE_float('learning_rate', 0.02, 'Learning rate for training')

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/advanced_mia_test.py

@@ -16,7 +16,7 @@
 from absl.testing import absltest
 from absl.testing import parameterized
 import numpy as np
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import advanced_mia as amia
+from privacy_tests.membership_inference_attack import advanced_mia as amia
 
 
 class TestScoreOffset(parameterized.TestCase):

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/codelabs/example.py

@@ -27,10 +27,10 @@
 import scipy.stats
 from sklearn import metrics
 import tensorflow as tf
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import data_structures
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import membership_inference_attack as mia
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import plotting
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import privacy_report
+from privacy_tests.membership_inference_attack import data_structures
+from privacy_tests.membership_inference_attack import membership_inference_attack as mia
+from privacy_tests.membership_inference_attack import plotting
+from privacy_tests.membership_inference_attack import privacy_report
 
 _CUSTOM_SLICES = flags.DEFINE_boolean(
     "custom_slices",

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/data_structures.py

@@ -26,8 +26,8 @@
 import pandas as pd
 from scipy import special
 from sklearn import metrics
-from tensorflow_privacy.privacy.privacy_tests import epsilon_lower_bound as elb
-from tensorflow_privacy.privacy.privacy_tests import utils
+from privacy_tests import epsilon_lower_bound as elb
+from privacy_tests import utils
 
 # The minimum TPR or FPR below which they are considered equal.
 _ABSOLUTE_TOLERANCE = 1e-3

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/data_structures_test.py

@@ -20,22 +20,22 @@
 from absl.testing import parameterized
 import numpy as np
 import pandas as pd
-from tensorflow_privacy.privacy.privacy_tests import epsilon_lower_bound as elb
-from tensorflow_privacy.privacy.privacy_tests import utils
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack import data_structures
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import _log_value
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackInputData
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackResults
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackResultsCollection
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackType
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import DataSize
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import EpsilonLowerBoundValue
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import PrivacyReportMetadata
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import RocCurve
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SingleAttackResult
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SingleMembershipProbabilityResult
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SingleSliceSpec
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SlicingFeature
+from privacy_tests import epsilon_lower_bound as elb
+from privacy_tests import utils
+from privacy_tests.membership_inference_attack import data_structures
+from privacy_tests.membership_inference_attack.data_structures import _log_value
+from privacy_tests.membership_inference_attack.data_structures import AttackInputData
+from privacy_tests.membership_inference_attack.data_structures import AttackResults
+from privacy_tests.membership_inference_attack.data_structures import AttackResultsCollection
+from privacy_tests.membership_inference_attack.data_structures import AttackType
+from privacy_tests.membership_inference_attack.data_structures import DataSize
+from privacy_tests.membership_inference_attack.data_structures import EpsilonLowerBoundValue
+from privacy_tests.membership_inference_attack.data_structures import PrivacyReportMetadata
+from privacy_tests.membership_inference_attack.data_structures import RocCurve
+from privacy_tests.membership_inference_attack.data_structures import SingleAttackResult
+from privacy_tests.membership_inference_attack.data_structures import SingleMembershipProbabilityResult
+from privacy_tests.membership_inference_attack.data_structures import SingleSliceSpec
+from privacy_tests.membership_inference_attack.data_structures import SlicingFeature
 
 
 class SingleSliceSpecTest(parameterized.TestCase):
@@ -557,7 +557,7 @@ def test_ppv_perfect_classifier(self):
       ('test_train_ratio_large', 1000.0, 0.0),
   )
   @mock.patch(
-      'tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures._ABSOLUTE_TOLERANCE',
+      'privacy_tests.membership_inference_attack.data_structures._ABSOLUTE_TOLERANCE',
       1e-4)
   def test_ppv_perfect_classifier_when_tpr_fpr_small(self, test_train_ratio,
                                                      expected_ppv):
@@ -570,7 +570,7 @@ def test_ppv_perfect_classifier_when_tpr_fpr_small(self, test_train_ratio,
     np.testing.assert_allclose(roc.get_ppv(), expected_ppv, atol=1e-3)
 
   @mock.patch(
-      'tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures._ABSOLUTE_TOLERANCE',
+      'privacy_tests.membership_inference_attack.data_structures._ABSOLUTE_TOLERANCE',
       1e-4)
   def test_ppv_random_classifier_when_tpr_fpr_small_and_test_train_is_1(self):
     roc = RocCurve(

tensorflow_privacy/privacy/privacy_tests/membership_inference_attack/dataset_slicing.py

@@ -20,10 +20,10 @@
 
 import numpy as np
 
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import AttackInputData
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SingleSliceSpec
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SlicingFeature
-from tensorflow_privacy.privacy.privacy_tests.membership_inference_attack.data_structures import SlicingSpec
+from privacy_tests.membership_inference_attack.data_structures import AttackInputData
+from privacy_tests.membership_inference_attack.data_structures import SingleSliceSpec
+from privacy_tests.membership_inference_attack.data_structures import SlicingFeature
+from privacy_tests.membership_inference_attack.data_structures import SlicingSpec
 
 
 _MAX_NUM_OF_SLICES = 1000