diff --git a/.codecov.yml b/.codecov.yml index b09d15da..8a9841a4 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -3,4 +3,5 @@ ignore: - "setup.py" - "aisdc/safemodel/classifiers/new_model_template.py" + - "aisdc/preprocessing" ... diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 327f447e..a3c05604 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,35 +31,17 @@ repos: .*\.ipynb )$ - # Autoremoves unused imports - - repo: https://github.com/hadialqattan/pycln - rev: "v2.4.0" + # Ruff, the Python auto-correcting linter/formatter written in Rust + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.8 hooks: - - id: pycln - stages: [manual] - - # Sort includes - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 - hooks: - - id: pyupgrade - - # Upgrade old Python syntax - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - args: ["--profile", "black"] - - # Black format Python and notebooks - - repo: https://github.com/psf/black - rev: 24.4.2 - hooks: - - id: black-jupyter + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format # Format docstrings - repo: https://github.com/DanielNoord/pydocstringformatter - rev: "v0.7.3" + rev: v0.7.3 hooks: - id: pydocstringformatter args: ["--style=numpydoc"] diff --git a/aisdc/attacks/attack_report_formatter.py b/aisdc/attacks/attack_report_formatter.py index 2a44d688..b694d01d 100644 --- a/aisdc/attacks/attack_report_formatter.py +++ b/aisdc/attacks/attack_report_formatter.py @@ -117,9 +117,7 @@ def __str__(self): raise NotImplementedError() -class FinalRecommendationModule( - AnalysisModule -): # pylint: disable=too-many-instance-attributes +class FinalRecommendationModule(AnalysisModule): # pylint: disable=too-many-instance-attributes """Module that generates the first layer of a recommendation report.""" def __init__(self, report: dict): diff --git a/aisdc/attacks/likelihood_attack.py b/aisdc/attacks/likelihood_attack.py index 3cb067bb..a283eb10 100644 --- a/aisdc/attacks/likelihood_attack.py +++ b/aisdc/attacks/likelihood_attack.py @@ -488,9 +488,9 @@ def _construct_metadata(self) -> None: if auc_p <= self.p_thresh else f"Not significant at p={self.p_thresh}" ) - self.metadata["global_metrics"][ - "null_auc_3sd_range" - ] = f"{0.5 - 3 * auc_std} -> {0.5 + 3 * auc_std}" + self.metadata["global_metrics"]["null_auc_3sd_range"] = ( + f"{0.5 - 3 * auc_std} -> {0.5 + 3 * auc_std}" + ) self.metadata["attack"] = str(self) @@ -540,6 +540,9 @@ def _get_attack_metrics_instances(self) -> dict: attack_metrics_instances = {} for rep, _ in enumerate(self.attack_metrics): + self.attack_metrics[rep]["n_shadow_models_trained"] = ( + self.attack_failfast_shadow_models_trained + ) attack_metrics_instances["instance_" + str(rep)] = self.attack_metrics[rep] attack_metrics_experiment["attack_instance_logger"] = attack_metrics_instances diff --git a/aisdc/attacks/report.py b/aisdc/attacks/report.py index fd55157c..178e3a73 100644 --- a/aisdc/attacks/report.py +++ b/aisdc/attacks/report.py @@ -111,18 +111,14 @@ def title(pdf, text, border=BORDER, font_size=24, font_style="B"): pdf.ln(h=5) -def subtitle( - pdf, text, indent=10, border=BORDER, font_size=12, font_style="B" -): # pylint: disable = too-many-arguments +def subtitle(pdf, text, indent=10, border=BORDER, font_size=12, font_style="B"): # pylint: disable = too-many-arguments """Write a subtitle block.""" pdf.cell(indent, border=border) pdf.set_font("arial", font_style, font_size) pdf.cell(75, 10, text, border, 1) -def line( - pdf, text, indent=0, border=BORDER, font_size=11, font_style="", font="arial" -): # pylint: disable = too-many-arguments +def line(pdf, text, indent=0, border=BORDER, font_size=11, font_style="", font="arial"): # pylint: disable = too-many-arguments """Write a standard block.""" if indent > 0: pdf.cell(indent, border=border) diff --git a/aisdc/safemodel/classifiers/new_model_template.py b/aisdc/safemodel/classifiers/new_model_template.py index 258a00fe..4f97dce4 100644 --- a/aisdc/safemodel/classifiers/new_model_template.py +++ b/aisdc/safemodel/classifiers/new_model_template.py @@ -1,7 +1,7 @@ """This is a template for implementing supplementary models - Obviously we have invented an sklearn ensemble called ModelToMakeSafer - Replace this with details of the model you wish to create a wrapper for - and then remove the comment which disables the pylint warning. +Obviously we have invented an sklearn ensemble called ModelToMakeSafer +Replace this with details of the model you wish to create a wrapper for +and then remove the comment which disables the pylint warning. """ # pylint: disable=duplicate-code diff --git a/aisdc/safemodel/classifiers/safedecisiontreeclassifier.py b/aisdc/safemodel/classifiers/safedecisiontreeclassifier.py index 868013d6..5095b2f9 100644 --- a/aisdc/safemodel/classifiers/safedecisiontreeclassifier.py +++ b/aisdc/safemodel/classifiers/safedecisiontreeclassifier.py @@ -127,9 +127,7 @@ def get_tree_k_anonymity(thetree: DecisionTreeClassifier, X: Any) -> int: return k_anonymity -class SafeDecisionTreeClassifier( - SafeModel, DecisionTreeClassifier -): # pylint: disable=too-many-ancestors +class SafeDecisionTreeClassifier(SafeModel, DecisionTreeClassifier): # pylint: disable=too-many-ancestors """Privacy protected Decision Tree classifier.""" def __init__(self, **kwargs: Any) -> None: diff --git a/aisdc/safemodel/classifiers/safekeras.py b/aisdc/safemodel/classifiers/safekeras.py index df45f517..7c7cdebc 100644 --- a/aisdc/safemodel/classifiers/safekeras.py +++ b/aisdc/safemodel/classifiers/safekeras.py @@ -1,11 +1,10 @@ """Safekeras.py: - Jim Smith, Andrew McCarty and Richard Preen - UWE 2022. +Jim Smith, Andrew McCarty and Richard Preen +UWE 2022. """ # general imports - import os import warnings @@ -219,10 +218,7 @@ def load_safe_keras_model(name: str = "undefined") -> Tuple[bool, Any]: class SafeKerasModel(KerasModel, SafeModel): - """Privacy Protected Wrapper around tf.keras.Model class from tensorflow 2.8 - disabling pylont warnings about number of instance attributes - as this is necessarily complex. - """ + """Privacy Protected Wrapper around tf.keras.Model class from tensorflow 2.8.""" # pylint: disable=too-many-instance-attributes def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -236,6 +232,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ##inputs = kwargs.get("inputs","notFound") ##if inputs=="notFound": ## inputs = args[0] if len(args) == 3 else None + inputs = None if "inputs" in kwargs.keys(): # pylint: disable=consider-iterating-dictionary inputs = the_kwargs["inputs"] elif len(args) == 3: # defaults is for Model(input,outputs,names) @@ -252,7 +249,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # call the keras super class first as this comes first in chain super().__init__( # pylint: disable=unexpected-keyword-arg - inputs=inputs, outputs=outputs # pylint: disable=used-before-assignment + inputs=inputs, + outputs=outputs, # pylint: disable=used-before-assignment ) # set values where the user has supplied them diff --git a/aisdc/safemodel/classifiers/safetf.py b/aisdc/safemodel/classifiers/safetf.py index 5ee5066c..3b685e51 100644 --- a/aisdc/safemodel/classifiers/safetf.py +++ b/aisdc/safemodel/classifiers/safetf.py @@ -1,6 +1,6 @@ """Work in progress to allow use of the DPModel classes - Jim smith 2022 - When ready, linting of the imports will be enabled. +Jim smith 2022 +When ready, linting of the imports will be enabled. """ # pylint: disable=unused-import @@ -28,7 +28,7 @@ def __init__( noise_multiplier: float, use_xla: bool, *args: any, - **kwargs: any + **kwargs: any, ) -> None: """Creates model and applies constraints to parameters.""" # safemodel.__init__(self) diff --git a/pyproject.toml b/pyproject.toml index 291b1373..1075166c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,3 +30,47 @@ min-public-methods = 2 # Minimum number of public methods for a class (see R090 [tool.pylint.format] max-line-length = 100 # Maximum number of characters on a single line. max-module-lines = 1000 # Maximum number of lines in a module. + +[tool.ruff] +indent-width = 4 +line-length = 88 +target-version = "py39" + +lint.select = [ +# "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments +# "B", # flake8-bugbear +# "C4", # flake8-comprehensions +# "C90", # mccabe +# "D", # pydocstyle +# "DTZ", # flake8-datetimez +# "E", # pycodestyle +# "EM", # flake8-errmsg +# "ERA", # eradicate +# "F", # Pyflakes + "I", # isort + "ICN", # flake8-import-conventions +# "N", # pep8-naming +# "PD", # pandas-vet + "PGH", # pygrep-hooks +# "PIE", # flake8-pie +# "PL", # Pylint + "PLC", # Pylint + "PLE", # Pylint +# "PLR", # Pylint +# "PLW", # Pylint + "PT", # flake8-pytest-style + "Q", # flake8-quotes +# "RET", # flake8-return + "RUF100", # Ruff-specific +# "S", # flake8-bandit +# "SIM", # flake8-simplify +# "T20", # flake8-print +# "TID", # flake8-tidy-imports +# "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] + +[tool.ruff.per-file-ignores] +"tests/**/*" = ["S101"] diff --git a/tests/lrconfig.json b/tests/attacks/lrconfig.json similarity index 100% rename from tests/lrconfig.json rename to tests/attacks/lrconfig.json diff --git a/tests/lrconfig_cmd.json b/tests/attacks/lrconfig_cmd.json similarity index 100% rename from tests/lrconfig_cmd.json rename to tests/attacks/lrconfig_cmd.json diff --git a/tests/attacks/test_attack_report_formatter.py b/tests/attacks/test_attack_report_formatter.py index 864814e2..1773cca9 100644 --- a/tests/attacks/test_attack_report_formatter.py +++ b/tests/attacks/test_attack_report_formatter.py @@ -1,12 +1,9 @@ -"""Test_generate_report.py -Copyright (C) Jim Smith 2022 . -""" +"""Test report generation.""" from __future__ import annotations import json import os -import shutil import unittest import pytest @@ -69,14 +66,6 @@ def get_target_report(): return target_formatted -def clean_up(name): - """Removes unwanted files or directory.""" - if os.path.exists(name) and os.path.isfile(name): # h5 - os.remove(name) - elif os.path.exists(name) and os.path.isdir(name): # tf - shutil.rmtree(name) - - class TestGenerateReport(unittest.TestCase): """Class which tests the attack_report_formatter.py file.""" @@ -95,9 +84,6 @@ def process_json_from_file(self, json_formatted): with open(output_filename, encoding="utf-8") as file: data = file.read() - clean_up(filename) - clean_up(output_filename) - return data def test_not_implemented(self): @@ -113,13 +99,11 @@ def test_json_formatter(self): """Test which tests the GenerateJSONModule.""" g = GenerateJSONModule() filename = g.get_output_filename() - self.assertIsNotNone(filename) - clean_up(filename) + assert filename is not None test_filename = "example_filename.json" g = GenerateJSONModule(test_filename) - self.assertEqual(test_filename, g.get_output_filename()) - clean_up(test_filename) + assert test_filename == g.get_output_filename() # check file is overwritten when the same file is passed test_filename = "filename_to_rewrite.json" @@ -135,10 +119,8 @@ def test_json_formatter(self): with open(test_filename, encoding="utf-8") as f: file_contents = json.loads(f.read()) - self.assertIn(msg_1, file_contents["FirstTestAttack"]["test_output"]) - self.assertIn(msg_2, file_contents["SecondTestAttack"]["test_output"]) - - clean_up(test_filename) + assert msg_1 in file_contents["FirstTestAttack"]["test_output"] + assert msg_2 in file_contents["SecondTestAttack"]["test_output"] def test_pretty_print(self): """Test which tests the pretty_print function with nested dictionaries.""" @@ -192,12 +174,8 @@ def test_process_attack_target_json(self): g.process_attack_target_json(attack_json, target_json) g.export_to_file(output_filename) - clean_up(target_json) - clean_up(attack_json) - clean_up(output_filename) - def test_whitespace_in_filenames(self): - """Test to make sure whitespace is removed from the output file when creating the report.""" + """Test whitespace is removed from the output file when creating a report.""" json_formatted = get_test_report() filename = "test.json" @@ -210,12 +188,8 @@ def test_whitespace_in_filenames(self): g.process_attack_target_json(filename) g.export_to_file(output_filename) - assert os.path.exists("filename should be changed.txt") is False - assert os.path.exists("filename_should_be_changed.txt") is True - - clean_up(filename) - clean_up(output_filename) - clean_up("filename_should_be_changed.txt") + assert not os.path.exists("filename should be changed.txt") + assert os.path.exists("filename_should_be_changed.txt") def test_move_files(self): """Test the move_files parameter inside export_to_file.""" @@ -236,7 +210,7 @@ def test_move_files(self): g.export_to_file(output_filename, move_files=True, release_dir="release_dir") # Check when no model name has been provided - assert os.path.exists(os.path.join("release_dir", output_filename)) is True + assert os.path.exists(os.path.join("release_dir", output_filename)) g.export_to_file( output_filename, @@ -246,18 +220,14 @@ def test_move_files(self): ) # Check model file has been copied (NOT moved) - assert os.path.exists(os.path.join("release_dir", dummy_model)) is True - assert os.path.exists(dummy_model) is True - - clean_up(dummy_model) - - clean_up(output_filename) + assert os.path.exists(os.path.join("release_dir", dummy_model)) + assert os.path.exists(dummy_model) png_file = "log_roc.png" with open(png_file, "w", encoding="utf-8") as f: pass - assert os.path.exists(png_file) is True + assert os.path.exists(png_file) g = GenerateTextReport() g.process_attack_target_json(filename) @@ -268,35 +238,27 @@ def test_move_files(self): artefacts_dir="training_artefacts", ) - assert os.path.exists(png_file) is False - assert os.path.exists(os.path.join("training_artefacts", png_file)) is True - - clean_up(filename) - clean_up(output_filename) - clean_up("release_dir") - clean_up("training_artefacts") + assert not os.path.exists(png_file) + assert os.path.exists(os.path.join("training_artefacts", png_file)) def test_complete_runthrough(self): - """Test the full process_json file end-to-end when valid parameters are passed.""" + """Test process_json file end-to-end when valid parameters are passed.""" json_formatted = get_test_report() _ = self.process_json_from_file(json_formatted) class TestFinalRecommendationModule(unittest.TestCase): - """Class which tests the FinalRecommendatiionModule inside attack_report_formatter.py.""" + """Tests the FinalRecommendatiionModule inside attack_report_formatter.py.""" def test_instance_based(self): - """Test the process_json function when the target model is an instance based model.""" + """Test process_json function when target model is an instance based model.""" json_formatted = get_test_report() f = FinalRecommendationModule(json_formatted) f.process_dict() returned = f.get_recommendation() immediate_rejection = returned[0] - # support_rejection = returned[1] - # support_release = returned[2] - - self.assertEqual(len(immediate_rejection), 0) + assert len(immediate_rejection) == 0 json_formatted["model_name"] = "SVC" f = FinalRecommendationModule(json_formatted) @@ -304,10 +266,7 @@ def test_instance_based(self): returned = f.get_recommendation() immediate_rejection = returned[0] - # support_rejection = returned[1] - # support_release = returned[2] - - self.assertIn("Model is SVM", immediate_rejection) + assert "Model is SVM" in immediate_rejection json_formatted["model_name"] = "KNeighborsClassifier" f = FinalRecommendationModule(json_formatted) @@ -315,13 +274,10 @@ def test_instance_based(self): returned = f.get_recommendation() immediate_rejection = returned[0] - # support_rejection = returned[1] - # support_release = returned[2] - - self.assertIn("Model is kNN", immediate_rejection) + assert "Model is kNN" in immediate_rejection def test_min_samples_leaf(self): - """Test the process_json function when the target model includes decision trees.""" + """Test process_json when the target model includes decision trees.""" # test when min_samples_leaf > 5 json_formatted = get_test_report() @@ -331,10 +287,7 @@ def test_min_samples_leaf(self): returned = f.get_recommendation() immediate_rejection = returned[0] - # support_rejection = returned[1] - # support_release = returned[2] - - self.assertEqual(len(immediate_rejection), 0) + assert len(immediate_rejection) == 0 # test when min_samples_leaf < 5 json_formatted["model_params"]["min_samples_leaf"] = 2 @@ -343,15 +296,12 @@ def test_min_samples_leaf(self): f.process_dict() returned = f.get_recommendation() - # immediate_rejection = returned[0] support_rejection = returned[1] - # support_release = returned[2] - support_rejection = ", ".join(support_rejection) - self.assertIn("Min samples per leaf", support_rejection) + assert "Min samples per leaf" in support_rejection def test_statistically_significant(self): - """Test the statistically significant AUC p-values check in FinalRecommendationModule.""" + """Test statistically significant AUC p-values in FinalRecommendationModule.""" json_formatted = get_test_report() json_formatted["WorstCaseAttack"]["attack_experiment_logger"][ "attack_instance_logger" @@ -376,14 +326,11 @@ def test_statistically_significant(self): f.process_dict() returned = f.get_recommendation() - # immediate_rejection = returned[0] support_rejection = returned[1] - # support_release = returned[2] - support_rejection = ", ".join(support_rejection) - self.assertIn(">10% AUC are statistically significant", support_rejection) - self.assertIn("Attack AUC > threshold", support_rejection) + assert ">10% AUC are statistically significant" in support_rejection + assert "Attack AUC > threshold" in support_rejection metrics_dict["AUC"] = 0.5 @@ -396,23 +343,19 @@ def test_statistically_significant(self): f.process_dict() returned = f.get_recommendation() - # immediate_rejection = returned[0] - # support_rejection = returned[1] support_release = returned[2] - support_release = ", ".join(support_release) - - self.assertIn("Attack AUC <= threshold", support_release) + assert "Attack AUC <= threshold" in support_release def test_print(self): """Test the FinalRecommendationModule printing.""" json_formatted = get_test_report() f = FinalRecommendationModule(json_formatted) - self.assertEqual("Final Recommendation", str(f)) + assert str(f) == "Final Recommendation" class TestSummariseUnivariateMetricsModule(unittest.TestCase): - """Class which tests the SummariseUnivariateMetricsModule inside attack_report_formatter.py.""" + """Tests the SummariseUnivariateMetricsModule inside attack_report_formatter.py.""" def test_univariate_metrics_module(self): """Test the SummariseUnivariateMetricsModule.""" @@ -442,27 +385,27 @@ def test_univariate_metrics_module(self): wca_auc = returned["WorstCaseAttack"]["AUC"] for k in wca_auc.keys(): - self.assertAlmostEqual(auc_value, wca_auc[k]) + assert auc_value == pytest.approx(wca_auc[k]) wca_acc = returned["WorstCaseAttack"]["ACC"] for k in wca_acc.keys(): - self.assertAlmostEqual(acc_value, wca_acc[k]) + assert acc_value == pytest.approx(wca_acc[k]) wca_fdif = returned["WorstCaseAttack"]["FDIF01"] for k in wca_fdif.keys(): - self.assertAlmostEqual(fdif01_value, wca_fdif[k]) + assert fdif01_value == pytest.approx(wca_fdif[k]) - self.assertEqual("Summary of Univarite Metrics", str(f)) + assert str(f) == "Summary of Univarite Metrics" def test_print(self): """Test the SummariseUnivariateMetricsModule printing.""" json_formatted = get_test_report() f = SummariseUnivariateMetricsModule(json_formatted) - self.assertEqual("Summary of Univarite Metrics", str(f)) + assert str(f) == "Summary of Univarite Metrics" class TestSummariseAUCPvalsModule(unittest.TestCase): - """Class which tests the SummariseAUCPvalsModule inside attack_report_formatter.py.""" + """Tests the SummariseAUCPvalsModule inside attack_report_formatter.py.""" def test_auc_pvals_module(self): """Test the SummariseAUCPvalsModule.""" @@ -472,8 +415,8 @@ def test_auc_pvals_module(self): returned = f.process_dict() # test the default correction - self.assertEqual("bh", returned["correction"]) - self.assertEqual(10, returned["n_total"]) + assert returned["correction"] == "bh" + assert returned["n_total"] == 10 metrics_dict = {"P_HIGHER_AUC": 0.001} @@ -485,15 +428,14 @@ def test_auc_pvals_module(self): f = SummariseAUCPvalsModule(json_formatted) _ = str(f) returned = f.process_dict() - self.assertEqual(11, returned["n_total"]) + assert returned["n_total"] == 11 f = SummariseAUCPvalsModule(json_formatted, correction="bo") returned = f.process_dict() + assert returned["correction"] == "bo" - self.assertEqual("bo", returned["correction"]) - + f = SummariseAUCPvalsModule(json_formatted, correction="xyzabcd") with pytest.raises(NotImplementedError): - f = SummariseAUCPvalsModule(json_formatted, correction="xyzabcd") _ = f.process_dict() _ = json_formatted["WorstCaseAttack"].pop("attack_experiment_logger") @@ -503,11 +445,11 @@ def test_print(self): """Test the SummariseAUCPvalsModule printing.""" json_formatted = get_test_report() f = SummariseAUCPvalsModule(json_formatted) - self.assertIn("Summary of AUC p-values", str(f)) + assert "Summary of AUC p-values" in str(f) class TestSummariseFDIFPvalsModule(unittest.TestCase): - """Class which tests the SummariseFDIFPvalsModule inside attack_report_formatter.py.""" + """Test the SummariseFDIFPvalsModule inside attack_report_formatter.py.""" def test_fdif_pvals_module(self): """Test the SummariseFDIFPvalsModule.""" @@ -515,24 +457,24 @@ def test_fdif_pvals_module(self): f = SummariseFDIFPvalsModule(json_formatted) returned = f.process_dict() - self.assertEqual("bh", returned["correction"]) - self.assertEqual(10, returned["n_total"]) + assert returned["correction"] == "bh" + assert returned["n_total"] == 10 returned = f.get_metric_list( json_formatted["WorstCaseAttack"]["attack_experiment_logger"] ) - self.assertEqual(10, len(returned)) + assert len(returned) == 10 def test_print(self): """Test the SummariseFDIFPvalsModule printing.""" json_formatted = get_test_report() f = SummariseFDIFPvalsModule(json_formatted) - self.assertIn("Summary of FDIF p-values", str(f)) + assert "Summary of FDIF p-values" in str(f) class TestLogLogROCModule(unittest.TestCase): - """Class which tests the LogLogROCModule inside attack_report_formatter.py.""" + """Test the LogLogROCModule inside attack_report_formatter.py.""" def test_loglog_roc_module(self): """Test the LogLogROCModule.""" @@ -543,10 +485,8 @@ def test_loglog_roc_module(self): output_file = ( f"{json_formatted['log_id']}-{json_formatted['metadata']['attack']}.png" ) - self.assertIn(output_file, returned) - assert os.path.exists(output_file) is True - - clean_up(output_file) + assert output_file in returned + assert os.path.exists(output_file) f = LogLogROCModule(json_formatted, output_folder=".") returned = f.process_dict() @@ -554,10 +494,8 @@ def test_loglog_roc_module(self): output_file = ( f"{json_formatted['log_id']}-{json_formatted['metadata']['attack']}.png" ) - self.assertIn(output_file, returned) - assert os.path.exists(output_file) is True - - clean_up(output_file) + assert output_file in returned + assert os.path.exists(output_file) def test_loglog_multiple_files(self): """Test the LogLogROCModule with multiple tests.""" @@ -575,14 +513,11 @@ def test_loglog_multiple_files(self): f"{out_json_copy['log_id']}-{out_json_copy['metadata']['attack']}.png" ) - self.assertIn(output_file_1, returned) - self.assertIn(output_file_2, returned) - - clean_up(output_file_1) - clean_up(output_file_2) + assert output_file_1 in returned + assert output_file_2 in returned def test_print(self): """Test the LogLogROCModule printing.""" json_formatted = get_test_report() f = LogLogROCModule(json_formatted) - self.assertEqual("ROC Log Plot", str(f)) + assert str(f) == "ROC Log Plot" diff --git a/tests/attacks/test_attacks_target.py b/tests/attacks/test_attacks_target.py index 2361bff2..7fed9d7e 100644 --- a/tests/attacks/test_attacks_target.py +++ b/tests/attacks/test_attacks_target.py @@ -1,74 +1,20 @@ -"""Code to test the file attacks/target.py.""" +"""Test Target class.""" from __future__ import annotations -import builtins -import io -import os - import numpy as np import pytest from sklearn.ensemble import RandomForestClassifier from aisdc.attacks.target import Target -from ..common import clean, get_target - -# pylint: disable=redefined-outer-name - RES_DIR = "save_test" -def patch_open(open_func, files): - """Helper function for cleaning up created files.""" - - def open_patched( # pylint: disable=too-many-arguments - path, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, - ): - if "w" in mode and not os.path.isfile(path): - files.append(path) - return open_func( - path, - mode=mode, - buffering=buffering, - encoding=encoding, - errors=errors, - newline=newline, - closefd=closefd, - opener=opener, - ) - - return open_patched - - -@pytest.fixture -def cleanup_files(monkeypatch): - """Automatically remove created files.""" - files = [] - monkeypatch.setattr(builtins, "open", patch_open(builtins.open, files)) - monkeypatch.setattr(io, "open", patch_open(io.open, files)) - yield - for file in files: - try: - os.remove(file) - except FileNotFoundError: # pragma: no cover - pass - - -def test_target(cleanup_files): # pylint:disable=unused-argument - """ - Returns a randomly sampled 10+10% of - the nursery data set as a Target object - if needed fetches it from openml and saves. it. - """ - target = get_target(model=RandomForestClassifier(n_estimators=5, max_depth=5)) +@pytest.mark.parametrize("get_target", [RandomForestClassifier()], indirect=True) +def test_target(get_target): + """Test a randomly sampled 10% of the nursery dataset as a Target object.""" + target = get_target # [Researcher] Saves the target model and data target.save(RES_DIR) @@ -93,5 +39,3 @@ def test_target(cleanup_files): # pylint:disable=unused-argument assert np.array_equal(tre_target.y_train_orig, target.y_train_orig) assert np.array_equal(tre_target.x_test_orig, target.x_test_orig) assert np.array_equal(tre_target.y_test_orig, target.y_test_orig) - - clean(RES_DIR) diff --git a/tests/attacks/test_attribute_inference_attack.py b/tests/attacks/test_attribute_inference_attack.py index e8b12e5c..bcb120f0 100644 --- a/tests/attacks/test_attribute_inference_attack.py +++ b/tests/attacks/test_attribute_inference_attack.py @@ -1,20 +1,12 @@ -""" -Example demonstrating the attribute inference attacks. - -Running -------- - -Invoke this code from the root AI-SDC folder with -python -m examples.attribute_inference_example -""" +"""Test attribute inference attacks.""" from __future__ import annotations import json import os import sys -import unittest +import pytest from sklearn.ensemble import RandomForestClassifier from aisdc.attacks import attribute_attack @@ -24,140 +16,107 @@ _unique_max, ) -from ..common import clean, get_target - -# pylint: disable = duplicate-code - - -class TestAttributeInferenceAttack(unittest.TestCase): - """Class which tests the AttributeInferenceAttack module.""" - - def _common_setup(self): - """Basic commands to get ready to test some code.""" - model = RandomForestClassifier(bootstrap=False) - target = get_target(model) - model.fit(target.x_train, target.y_train) - attack_obj = attribute_attack.AttributeAttack(n_cpu=7, report_name="aia_report") - return target, attack_obj - - def test_attack_args(self): - """Tests methods in the attack_args class.""" - _, attack_obj = self._common_setup() - attack_obj.__dict__["newkey"] = True - thedict = attack_obj.__dict__ - - self.assertTrue(thedict["newkey"]) - - def test_unique_max(self): - """Tests the _unique_max helper function.""" - has_unique = (0.3, 0.5, 0.2) - no_unique = (0.5, 0.5) - self.assertTrue(_unique_max(has_unique, 0.0)) - self.assertFalse(_unique_max(has_unique, 0.6)) - self.assertFalse(_unique_max(no_unique, 0.0)) - - def test_categorical_via_modified_attack_brute_force(self): - """Test lots of functionality for categoricals - using code from brute_force but without multiprocessing. - """ - target, _ = self._common_setup() - - threshold = 0 - feature = 0 - # make predictions - t_low = _infer_categorical(target, feature, threshold) - t_low_correct = t_low["train"][0] - t_low_total = t_low["train"][1] - t_low_train_samples = t_low["train"][4] - - # Check the number of samples in the dataset - self.assertEqual(len(target.x_train), t_low_train_samples) - - # Check that all samples are correct for this threshold - self.assertEqual(t_low_correct, t_low_total) - - # or don't because threshold is too high - threshold = 999 - t_high = _infer_categorical(target, feature, threshold) - t_high_correct = t_high["train"][0] - t_high_train_samples = t_high["train"][4] - - self.assertEqual(len(target.x_train), t_high_train_samples) - self.assertEqual(0, t_high_correct) - - def test_continuous_via_modified_bounds_risk(self): - """Tests a lot of the code for continuous variables - via a copy of the _get_bounds_risk() - modified not to use multiprocessing. - """ - target, _ = self._common_setup() - returned = _get_bounds_risk( - target.model, "dummy", 8, target.x_train, target.x_test - ) - # Check the number of parameters returned - self.assertEqual(3, len(returned.keys())) - - # Check the value of the returned parameters - self.assertEqual(0.0, returned["train"]) - self.assertEqual(0.0, returned["test"]) - clean("output_attribute") - - # test below covers a lot of the plotting etc. - def test_AIA_on_nursery(self): - """Tests running AIA on the nursery data - with an added continuous feature. - """ - target, attack_obj = self._common_setup() - attack_obj.attack(target) - - output = attack_obj.make_report() - keys = output["attack_experiment_logger"]["attack_instance_logger"][ - "instance_0" - ].keys() - - self.assertIn("categorical", keys) - - def test_AIA_on_nursery_from_cmd(self): - """Tests running AIA on the nursery data - with an added continuous feature. - """ - target, _ = self._common_setup() - target.save(path="tests/test_aia_target") - - config = { - "n_cpu": 7, - "report_name": "commandline_aia_exampl1_report", - } - with open( - os.path.join("tests", "test_config_aia_cmd.json"), "w", encoding="utf-8" - ) as f: - f.write(json.dumps(config)) - - cmd_json = os.path.join("tests", "test_config_aia_cmd.json") - aia_target = os.path.join("tests", "test_aia_target") - os.system( - f"{sys.executable} -m aisdc.attacks.attribute_attack run-attack-from-configfile " - f"--attack-config-json-file-name {cmd_json} " - f"--attack-target-folder-path {aia_target} " +def pytest_generate_tests(metafunc): + """Generate target model for testing.""" + if "get_target" in metafunc.fixturenames: + metafunc.parametrize( + "get_target", [RandomForestClassifier(bootstrap=False)], indirect=True ) - def test_cleanup(self): - """Tidies up any files created.""" - files_made = ( - "delete-me.json", - "aia_example.json", - "aia_example.pdf", - "aia_report_cat_frac.png", - "aia_report_cat_risk.png", - "aia_report_quant_risk.png", - "aia_report.pdf", - "aia_report.json", - "aia_attack_from_configfile.json", - "test_attribute_attack.json", - os.path.join("tests", "test_config_aia_cmd.json"), - os.path.join("tests", "test_aia_target/"), - "output_attribute", - ) - for fname in files_made: - clean(fname) + +@pytest.fixture(name="common_setup") +def fixture_common_setup(get_target): + """Get ready to test some code.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + attack_obj = attribute_attack.AttributeAttack(n_cpu=7, report_name="aia_report") + return target, attack_obj + + +def test_attack_args(common_setup): + """Test methods in the attack_args class.""" + _, attack_obj = common_setup + attack_obj.__dict__["newkey"] = True + thedict = attack_obj.__dict__ + assert thedict["newkey"] + + +def test_unique_max(): + """Test the _unique_max helper function.""" + has_unique = (0.3, 0.5, 0.2) + no_unique = (0.5, 0.5) + assert _unique_max(has_unique, 0.0) + assert not _unique_max(has_unique, 0.6) + assert not _unique_max(no_unique, 0.0) + + +def test_categorical_via_modified_attack_brute_force(common_setup): + """Test lcategoricals using code from brute_force.""" + target, _ = common_setup + + threshold = 0 + feature = 0 + # make predictions + t_low = _infer_categorical(target, feature, threshold) + t_low_correct = t_low["train"][0] + t_low_total = t_low["train"][1] + t_low_train_samples = t_low["train"][4] + + # Check the number of samples in the dataset + assert len(target.x_train) == t_low_train_samples + # Check that all samples are correct for this threshold + assert t_low_correct == t_low_total + + # or don't because threshold is too high + threshold = 999 + t_high = _infer_categorical(target, feature, threshold) + t_high_correct = t_high["train"][0] + t_high_train_samples = t_high["train"][4] + assert len(target.x_train) == t_high_train_samples + assert t_high_correct == 0 + + +def test_continuous_via_modified_bounds_risk(common_setup): + """Test continuous variables get_bounds_risk().""" + target, _ = common_setup + returned = _get_bounds_risk(target.model, "dummy", 8, target.x_train, target.x_test) + # Check the number of parameters returned + assert len(returned.keys()) == 3 + # Check the value of the returned parameters + assert returned["train"] == 0 + assert returned["test"] == 0 + + +def test_AIA_on_nursery(common_setup): + """Test AIA on the nursery data with an added continuous feature.""" + target, attack_obj = common_setup + attack_obj.attack(target) + output = attack_obj.make_report() + keys = output["attack_experiment_logger"]["attack_instance_logger"][ + "instance_0" + ].keys() + assert "categorical" in keys + + +def test_AIA_on_nursery_from_cmd(common_setup): + """Test AIA on the nursery data with an added continuous feature.""" + target, _ = common_setup + target.save(path="tests/test_aia_target") + + config = { + "n_cpu": 7, + "report_name": "commandline_aia_exampl1_report", + } + with open( + os.path.join("tests", "test_config_aia_cmd.json"), "w", encoding="utf-8" + ) as f: + f.write(json.dumps(config)) + + cmd_json = os.path.join("tests", "test_config_aia_cmd.json") + aia_target = os.path.join("tests", "test_aia_target") + os.system( + f"{sys.executable} -m aisdc.attacks.attribute_attack run-attack-from-configfile " + f"--attack-config-json-file-name {cmd_json} " + f"--attack-target-folder-path {aia_target} " + ) diff --git a/tests/attacks/test_failfast.py b/tests/attacks/test_failfast.py index 56632a61..9cacc784 100644 --- a/tests/attacks/test_failfast.py +++ b/tests/attacks/test_failfast.py @@ -1,21 +1,19 @@ -"""Test_worst_case_attack.py -Copyright (C) Jim Smith 2022 . -""" +"""Test fail fast.""" from __future__ import annotations -import shutil import unittest from aisdc.attacks import failfast, worst_case_attack class TestFailFast(unittest.TestCase): - """Class which tests the fail fast functionality of the WortCaseAttack module.""" + """Tests the fail fast functionality of the WortCaseAttack module.""" def test_parse_boolean_argument(self): - """Test all comparison operators and both options for attack - being successful and not successful given a metric and + """Test all comparison operators and both options. + + Tests for attack being successful and not successful given a metric and comparison operator with a threshold value. """ metrics = {} @@ -30,7 +28,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="lte", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 2 attack_obj = worst_case_attack.WorstCaseAttack( @@ -39,7 +37,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="lte", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) + assert failfast_Obj.check_attack_success(metrics) # Option 3 attack_obj = worst_case_attack.WorstCaseAttack( @@ -48,7 +46,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="lt", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 4 attack_obj = worst_case_attack.WorstCaseAttack( @@ -57,7 +55,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="lt", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) + assert failfast_Obj.check_attack_success(metrics) # Option 5 attack_obj = worst_case_attack.WorstCaseAttack( @@ -66,7 +64,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="gte", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) + assert failfast_Obj.check_attack_success(metrics) # Option 6 attack_obj = worst_case_attack.WorstCaseAttack( @@ -75,7 +73,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="gte", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 7 attack_obj = worst_case_attack.WorstCaseAttack( @@ -84,7 +82,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="gt", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) + assert failfast_Obj.check_attack_success(metrics) # Option 8 attack_obj = worst_case_attack.WorstCaseAttack( @@ -93,7 +91,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="gt", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 9 attack_obj = worst_case_attack.WorstCaseAttack( @@ -102,7 +100,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="eq", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) + assert failfast_Obj.check_attack_success(metrics) # Option 10 attack_obj = worst_case_attack.WorstCaseAttack( @@ -111,7 +109,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="eq", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 11 attack_obj = worst_case_attack.WorstCaseAttack( @@ -120,7 +118,7 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="not_eq", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertFalse(failfast_Obj.check_attack_success(metrics)) + assert not failfast_Obj.check_attack_success(metrics) # Option 12 attack_obj = worst_case_attack.WorstCaseAttack( @@ -129,17 +127,15 @@ def test_parse_boolean_argument(self): attack_metric_success_comp_type="not_eq", ) failfast_Obj = failfast.FailFast(attack_obj) - self.assertTrue(failfast_Obj.check_attack_success(metrics)) - - self.assertEqual(0, failfast_Obj.get_fail_count()) - - shutil.rmtree("output_worstcase") + assert failfast_Obj.check_attack_success(metrics) + assert failfast_Obj.get_fail_count() == 0 def test_attack_success_fail_counts_and_overall_attack_success(self): - """Test success and fail counts of attacks for a given threshold - of a given metric based on a given comparison operation and - also test overall attack successes using - count threshold of attack being successful or not successful. + """Test success and fail counts of attacks. + + Tests for a given threshold of a given metric based on a given + comparison operation and also test overall attack successes using count + threshold of attack being successful or not successful. """ metrics = {} metrics["ACC"] = 0.9 @@ -157,14 +153,12 @@ def test_attack_success_fail_counts_and_overall_attack_success(self): _ = failfast_Obj.check_attack_success(metrics) metrics["P_HIGHER_AUC"] = 0.03 _ = failfast_Obj.check_attack_success(metrics) - self.assertFalse(failfast_Obj.check_overall_attack_success(attack_obj)) + assert not failfast_Obj.check_overall_attack_success(attack_obj) metrics["P_HIGHER_AUC"] = 0.02 _ = failfast_Obj.check_attack_success(metrics) metrics["P_HIGHER_AUC"] = 0.01 _ = failfast_Obj.check_attack_success(metrics) - self.assertEqual(3, failfast_Obj.get_success_count()) - self.assertEqual(2, failfast_Obj.get_fail_count()) - self.assertTrue(failfast_Obj.check_overall_attack_success(attack_obj)) - - shutil.rmtree("output_worstcase") + assert failfast_Obj.get_success_count() == 3 + assert failfast_Obj.get_fail_count() == 2 + assert failfast_Obj.check_overall_attack_success(attack_obj) diff --git a/tests/attacks/test_lira_attack.py b/tests/attacks/test_lira_attack.py index 51cfdbf1..4b372737 100644 --- a/tests/attacks/test_lira_attack.py +++ b/tests/attacks/test_lira_attack.py @@ -1,21 +1,13 @@ -"""Test_lira_attack.py -Copyright (C) Jim Smith2022 . -""" - -# pylint: disable = duplicate-code +"""Test LiRA attack.""" from __future__ import annotations -import logging import os -import shutil import sys -from unittest import TestCase - -# import json from unittest.mock import patch import numpy as np +import pytest from sklearn.datasets import load_breast_cancer from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split @@ -25,192 +17,156 @@ from aisdc.attacks.target import Target N_SHADOW_MODELS = 20 - -logger = logging.getLogger(__file__) - - -def clean_up(name): - """Removes unwanted files or directory.""" - if os.path.exists(name): - if os.path.isfile(name): - os.remove(name) - elif os.path.isdir(name): - shutil.rmtree(name) - logger.info("Removed %s", name) - - -class TestDummyClassifier(TestCase): - """Test the dummy classifier class.""" - - @classmethod - def setUpClass(cls): - """Create a dummy classifier object.""" - cls.dummy = DummyClassifier() - cls.X = np.array([[0.2, 0.8], [0.7, 0.3]]) - - def test_predict_proba(self): - """Test the predict_proba method.""" - pred_probs = self.dummy.predict_proba(self.X) - assert np.array_equal(pred_probs, self.X) - - def test_predict(self): - """Test the predict method.""" - expected_output = np.array([1, 0]) - pred = self.dummy.predict(self.X) - assert np.array_equal(pred, expected_output) - - -class TestLiraAttack(TestCase): - """Test the LIRA attack code.""" - - @classmethod - def setUpClass(cls): - """Setup the common things for the class.""" - logger.info("Setting up test class") - X, y = load_breast_cancer(return_X_y=True, as_frame=False) - cls.train_X, cls.test_X, cls.train_y, cls.test_y = train_test_split( - X, y, test_size=0.3 - ) - cls.target_model = RandomForestClassifier( - n_estimators=100, min_samples_split=2, min_samples_leaf=1 - ) - cls.target_model.fit(cls.train_X, cls.train_y) - cls.target = Target(cls.target_model) - cls.target.add_processed_data(cls.train_X, cls.train_y, cls.test_X, cls.test_y) - cls.target.save(path="test_lira_target") - - # Dump training and test data to csv - np.savetxt( - "train_data.csv", - np.hstack((cls.train_X, cls.train_y[:, None])), - delimiter=",", - ) - np.savetxt( - "test_data.csv", np.hstack((cls.test_X, cls.test_y[:, None])), delimiter="," - ) - # dump the training and test predictions into files - np.savetxt( - "train_preds.csv", - cls.target_model.predict_proba(cls.train_X), - delimiter=",", - ) - np.savetxt( - "test_preds.csv", cls.target_model.predict_proba(cls.test_X), delimiter="," - ) - - def test_lira_attack(self): - """Tests the lira code two ways.""" - attack_obj = LIRAAttack( - n_shadow_models=N_SHADOW_MODELS, - output_dir="test_output_lira", - attack_config_json_file_name=os.path.join("tests", "lrconfig.json"), - ) - attack_obj.setup_example_data() - attack_obj.attack_from_config() - attack_obj.example() - - attack_obj2 = LIRAAttack( - n_shadow_models=N_SHADOW_MODELS, - output_dir="test_output_lira", - report_name="lira_example1_report", - ) - attack_obj2.attack(self.target) - _ = attack_obj2.make_report() # also makes .pdf and .json files - - def test_check_and_update_dataset(self): - """Test the code that removes items from test set with classes - not present in training set. - """ - - attack_obj = LIRAAttack(n_shadow_models=N_SHADOW_MODELS) - - # now make test[0] have a class not present in training set# - local_test_y = np.copy(self.test_y) - local_test_y[0] = 5 - local_target = Target(self.target_model) - local_target.add_processed_data( - self.train_X, self.train_y, self.test_X, local_test_y - ) - unique_classes_pre = set(local_test_y) - n_test_examples_pre = len(local_test_y) - local_target = ( - attack_obj._check_and_update_dataset( # pylint:disable=protected-access - local_target - ) - ) - - unique_classes_post = set(local_target.y_test) - n_test_examples_post = len(local_target.y_test) - - self.assertNotEqual(local_target.y_test[0], 5) - self.assertEqual(n_test_examples_pre - n_test_examples_post, 1) - class_diff = unique_classes_pre - unique_classes_post # set diff - self.assertSetEqual(class_diff, {5}) - - def test_main_example(self): - """Test command line example.""" - testargs = [ - "prog", - "run-example", - "--output-dir", - "test_output_lira", - "--report-name", - "commandline_lira_example2_report", - ] - with patch.object(sys, "argv", testargs): - likelihood_attack.main() - - def test_main_config(self): - """Test command line with a config file.""" - testargs = [ - "prog", - "run-attack", - "-j", - os.path.join("tests", "lrconfig.json"), - "--output-dir", - "test_output_lira", - "--report-name", - "commandline_lira_example1_report", - ] - with patch.object(sys, "argv", testargs): - likelihood_attack.main() - - def test_main_from_configfile(self): - """Test command line with a config file.""" - testargs = [ - "prog", - "run-attack-from-configfile", - "-j", - os.path.join("tests", "lrconfig_cmd.json"), - "-t", - "test_lira_target", - ] - with patch.object(sys, "argv", testargs): - likelihood_attack.main() - - def test_main_example_data(self): - """Test command line example data creation.""" - testargs = [ - "prog", - "setup-example-data", - ] # , "--j", os.path.join("tests","lrconfig.json")] - with patch.object(sys, "argv", testargs): - likelihood_attack.main() - - @classmethod - def tearDownClass(cls): - """Cleans up various files made during the tests.""" - names = [ - "test_output_lira", - "output_lira", - "outputs_lira", - "test_lira_target", - "test_preds.csv", - "config.json", - "train_preds.csv", - "test_data.csv", - "train_data.csv", - "test_lira_target", - ] - for name in names: - clean_up(name) +LR_CONFIG = os.path.normpath("tests/attacks/lrconfig.json") +LR_CMD_CONFIG = os.path.normpath("tests/attacks/lrconfig_cmd.json") + + +@pytest.fixture(name="dummy_classifier_setup") +def fixture_dummy_classifier_setup(): + """Setup common things for DummyClassifier.""" + dummy = DummyClassifier() + X = np.array([[0.2, 0.8], [0.7, 0.3]]) + return dummy, X + + +def test_predict_proba(dummy_classifier_setup): + """Test the predict_proba method.""" + dummy, X = dummy_classifier_setup + pred_probs = dummy.predict_proba(X) + assert np.array_equal(pred_probs, X) + + +def test_predict(dummy_classifier_setup): + """Test the predict method.""" + dummy, X = dummy_classifier_setup + expected_output = np.array([1, 0]) + pred = dummy.predict(X) + assert np.array_equal(pred, expected_output) + + +@pytest.fixture(name="lira_classifier_setup") +def fixture_lira_classifier_setup(): + """Setup common things for LiRA.""" + X, y = load_breast_cancer(return_X_y=True, as_frame=False) + train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3) + target_model = RandomForestClassifier( + n_estimators=100, min_samples_split=2, min_samples_leaf=1 + ) + target_model.fit(train_X, train_y) + target = Target(target_model) + target.add_processed_data(train_X, train_y, test_X, test_y) + target.save(path="test_lira_target") + # Dump training and test data to csv + np.savetxt( + "train_data.csv", + np.hstack((train_X, train_y[:, None])), + delimiter=",", + ) + np.savetxt("test_data.csv", np.hstack((test_X, test_y[:, None])), delimiter=",") + # dump the training and test predictions into files + np.savetxt( + "train_preds.csv", + target_model.predict_proba(train_X), + delimiter=",", + ) + np.savetxt("test_preds.csv", target_model.predict_proba(test_X), delimiter=",") + return target + + +def test_lira_attack(lira_classifier_setup): + """Tests the lira code two ways.""" + target = lira_classifier_setup + attack_obj = LIRAAttack( + n_shadow_models=N_SHADOW_MODELS, + output_dir="test_output_lira", + attack_config_json_file_name=LR_CONFIG, + ) + attack_obj.setup_example_data() + attack_obj.attack_from_config() + attack_obj.example() + + attack_obj2 = LIRAAttack( + n_shadow_models=N_SHADOW_MODELS, + output_dir="test_output_lira", + report_name="lira_example1_report", + ) + attack_obj2.attack(target) + output2 = attack_obj2.make_report() + n_shadow_models_trained = output2["attack_experiment_logger"][ + "attack_instance_logger" + ]["instance_0"]["n_shadow_models_trained"] + n_shadow_models = output2["metadata"]["experiment_details"]["n_shadow_models"] + assert n_shadow_models_trained == n_shadow_models + + +def test_check_and_update_dataset(lira_classifier_setup): + """Test removal from test set with classes not present in training set.""" + target = lira_classifier_setup + attack_obj = LIRAAttack(n_shadow_models=N_SHADOW_MODELS) + + # now make test[0] have a class not present in training set# + local_test_y = np.copy(target.y_test) + local_test_y[0] = 5 + local_target = Target(target.model) + local_target.add_processed_data( + target.x_train, target.y_train, target.x_test, local_test_y + ) + unique_classes_pre = set(local_test_y) + n_test_examples_pre = len(local_test_y) + local_target = attack_obj._check_and_update_dataset( # pylint: disable=protected-access + local_target + ) + + unique_classes_post = set(local_target.y_test) + n_test_examples_post = len(local_target.y_test) + + assert local_target.y_test[0] != 5 + assert (n_test_examples_pre - n_test_examples_post) == 1 + class_diff = unique_classes_pre - unique_classes_post + assert class_diff == {5} + + # Test command line example. + testargs = [ + "prog", + "run-example", + "--output-dir", + "test_output_lira", + "--report-name", + "commandline_lira_example2_report", + ] + with patch.object(sys, "argv", testargs): + likelihood_attack.main() + + # Test command line with a config file. + testargs = [ + "prog", + "run-attack", + "-j", + LR_CONFIG, + "--output-dir", + "test_output_lira", + "--report-name", + "commandline_lira_example1_report", + ] + with patch.object(sys, "argv", testargs): + likelihood_attack.main() + + # Test command line with a config file. + testargs = [ + "prog", + "run-attack-from-configfile", + "-j", + LR_CMD_CONFIG, + "-t", + "test_lira_target", + ] + with patch.object(sys, "argv", testargs): + likelihood_attack.main() + + # Test command line example data creation. + testargs = [ + "prog", + "setup-example-data", + ] + with patch.object(sys, "argv", testargs): + likelihood_attack.main() \ No newline at end of file diff --git a/tests/attacks/test_metrics.py b/tests/attacks/test_metrics.py index 2c766bac..8f69bf3d 100644 --- a/tests/attacks/test_metrics.py +++ b/tests/attacks/test_metrics.py @@ -25,7 +25,7 @@ class DummyClassifier: - """Mocks the predict and predict_proba methods.""" + """Mock the predict and predict_proba methods.""" def predict(self, _): """Return dummy predictions.""" @@ -37,9 +37,7 @@ def predict_proba(self, _): class TestInputExceptions(unittest.TestCase): - """Test that the metrics.py errors with a helpful error message if an - invalid shape is supplied. - """ + """Test error message if an invalid shape is supplied.""" def _create_fake_test_data(self): y_test = np.zeros(4) @@ -49,47 +47,40 @@ def _create_fake_test_data(self): def test_wrong_shape(self): """Test the check which ensures y_pred_proba is of shape [:,:].""" y_test = self._create_fake_test_data() - with pytest.raises(ValueError): - y_pred_proba = np.zeros((4, 2, 2)) + y_pred_proba = np.zeros((4, 2, 2)) + with pytest.raises(ValueError, match="y_pred.*"): get_metrics(y_pred_proba, y_test) def test_wrong_size(self): """Test the check which ensures y_pred_proba is of size (:,2).""" y_test = self._create_fake_test_data() - with pytest.raises(ValueError): - y_pred_proba = np.zeros((4, 4)) + y_pred_proba = np.zeros((4, 4)) + with pytest.raises(ValueError, match=".*multiclass.*"): get_metrics(y_pred_proba, y_test) def test_valid_input(self): """Test to make sure a valid array does not throw an exception.""" y_test = self._create_fake_test_data() y_pred_proba = np.zeros((4, 2)) - returned = get_metrics(y_pred_proba, y_test) - acc = returned["ACC"] auc = returned["AUC"] p_auc = returned["P_HIGHER_AUC"] tpr = returned["TPR"] - - self.assertAlmostEqual(0.75, acc) - self.assertAlmostEqual(0.5, auc) - self.assertAlmostEqual(0.5, p_auc) - self.assertAlmostEqual(0.0, tpr) + assert pytest.approx(acc) == 0.75 + assert pytest.approx(auc) == 0.5 + assert pytest.approx(p_auc) == 0.5 + assert pytest.approx(tpr) == 0.0 class TestProbabilities(unittest.TestCase): """Test the checks on the input parameters of the get_probabilites function.""" def test_permute_rows_errors(self): - """ - Test to make sure an error is thrown when permute_rows is set to True, - but no y_test is supplied. - """ + """Test error when permute_rows is True, but no y_test is supplied.""" clf = DummyClassifier() testX = [] - - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="not enough values to unpack.*"): get_probabilities(clf, testX, permute_rows=True) def test_permute_rows_with_permute_rows(self): @@ -102,24 +93,21 @@ def test_permute_rows_with_permute_rows(self): returned = get_probabilities(clf, testX, testY, permute_rows=True) # Check the function returns two arguments - self.assertEqual(2, len(returned)) + assert len(returned) == 2 # Check that the second argument is the same shape as testY - self.assertEqual(testY.shape, returned[1].shape) + assert testY.shape == returned[1].shape # Check that the function is returning the right thing: predict_proba - self.assertEqual(clf.predict_proba(testX).shape, returned[0].shape) + assert clf.predict_proba(testX).shape == returned[0].shape def test_permute_rows_without_permute_rows(self): """Test permute_rows = False succeeds.""" - clf = DummyClassifier() testX = np.zeros((4, 2)) - y_pred_proba = get_probabilities(clf, testX, permute_rows=False) - # Check the function returns pnly y_pred_proba - self.assertEqual(clf.predict_proba(testX).shape, y_pred_proba.shape) + assert clf.predict_proba(testX).shape == y_pred_proba.shape class TestMetrics(unittest.TestCase): @@ -132,17 +120,17 @@ def test_metrics(self): testy = TRUE_CLASS y_pred_proba = get_probabilities(clf, testX, testy, permute_rows=False) metrics = get_metrics(y_pred_proba, testy) - self.assertAlmostEqual(metrics["TPR"], 2 / 3) - self.assertAlmostEqual(metrics["FPR"], 1 / 3) - self.assertAlmostEqual(metrics["FAR"], 1 / 3) - self.assertAlmostEqual(metrics["TNR"], 2 / 3) - self.assertAlmostEqual(metrics["PPV"], 2 / 3) - self.assertAlmostEqual(metrics["NPV"], 2 / 3) - self.assertAlmostEqual(metrics["FNR"], 1 / 3) - self.assertAlmostEqual(metrics["ACC"], 4 / 6) - self.assertAlmostEqual(metrics["F1score"], (8 / 9) / (2 / 3 + 2 / 3)) - self.assertAlmostEqual(metrics["Advantage"], 1 / 3) - self.assertAlmostEqual(metrics["AUC"], 8 / 9) + assert metrics["TPR"] == pytest.approx(2 / 3) + assert metrics["FPR"] == pytest.approx(1 / 3) + assert metrics["FAR"] == pytest.approx(1 / 3) + assert metrics["TNR"] == pytest.approx(2 / 3) + assert metrics["PPV"] == pytest.approx(2 / 3) + assert metrics["NPV"] == pytest.approx(2 / 3) + assert metrics["FNR"] == pytest.approx(1 / 3) + assert metrics["ACC"] == pytest.approx(4 / 6) + assert metrics["F1score"] == pytest.approx((8 / 9) / (2 / 3 + 2 / 3)) + assert metrics["Advantage"] == pytest.approx(1 / 3) + assert metrics["AUC"] == pytest.approx(8 / 9) def test_mia_extremecase(self): """Test the extreme case mia in metrics.py.""" @@ -157,11 +145,11 @@ def test_mia_extremecase(self): # right predictions - triggers override for very small logp _, _, _, pval = min_max_disc(y, right) - self.assertEqual(-115.13, pval) + assert pval == -115.13 # wrong predictions - probaility very close to 1 so logp=0 _, _, _, pval = min_max_disc(y, wrong) - self.assertAlmostEqual(0.0, pval) + assert pytest.approx(pval) == 0 class TestFPRatTPR(unittest.TestCase): @@ -171,30 +159,29 @@ def test_tpr(self): """Test tpr at fpr.""" y_true = TRUE_CLASS y_score = PREDICTED_PROBS[:, 1] - tpr = _tpr_at_fpr(y_true, y_score, fpr=0) - self.assertAlmostEqual(tpr, 2 / 3) + assert tpr == pytest.approx(2 / 3) tpr = _tpr_at_fpr(y_true, y_score, fpr=0.001) - self.assertAlmostEqual(tpr, 2 / 3) + assert tpr == pytest.approx(2 / 3) tpr = _tpr_at_fpr(y_true, y_score, fpr=0.1) - self.assertAlmostEqual(tpr, 2 / 3) + assert tpr == pytest.approx(2 / 3) tpr = _tpr_at_fpr(y_true, y_score, fpr=0.4) - self.assertAlmostEqual(tpr, 1) + assert tpr == pytest.approx(1) tpr = _tpr_at_fpr(y_true, y_score, fpr=1.0) - self.assertAlmostEqual(tpr, 1) + assert tpr == pytest.approx(1) tpr = _tpr_at_fpr(y_true, y_score, fpr_perc=True, fpr=100.0) - self.assertAlmostEqual(tpr, 1) + assert tpr == pytest.approx(1) class Test_Div(unittest.TestCase): - """Tests the _div functionality.""" + """Test the _div functionality.""" def test_div(self): """Test div for y=1 and 0.""" result = _div(8.0, 1.0, 99.0) - self.assertAlmostEqual(result, 8.0) + assert result == pytest.approx(8.0) result2 = _div(8.0, 0.0, 99.0) - self.assertAlmostEqual(result2, 99.0) + assert result2 == pytest.approx(99.0) class TestExtreme(unittest.TestCase): @@ -208,9 +195,9 @@ def test_extreme_default(self): # 10% of 6 is 1 so: # maxd should be 1 (the highest one is predicted as1) # mind should be 0 (the lowest one is not predicted as1) - self.assertAlmostEqual(maxd, 1.0) - self.assertAlmostEqual(mind, 0.0) - self.assertAlmostEqual(mmd, 1.0) + assert maxd == pytest.approx(1) + assert mind == pytest.approx(0) + assert mmd == pytest.approx(1) def test_extreme_higer_prop(self): """Tets with the dummy data but increase proportion to 0.5.""" @@ -220,6 +207,6 @@ def test_extreme_higer_prop(self): # 10% of 6 is 1 so: # maxd should be 1 (the highest one is predicted as1) # mind should be 0 (the lowest one is not predicted as1) - self.assertAlmostEqual(maxd, 2 / 3) - self.assertAlmostEqual(mind, 1 / 3) - self.assertAlmostEqual(mmd, 1 / 3) + assert maxd == pytest.approx(2 / 3) + assert mind == pytest.approx(1 / 3) + assert mmd == pytest.approx(1 / 3) diff --git a/tests/attacks/test_multiple_attacks.py b/tests/attacks/test_multiple_attacks.py index dc3cee0c..e10d0dc7 100644 --- a/tests/attacks/test_multiple_attacks.py +++ b/tests/attacks/test_multiple_attacks.py @@ -1,66 +1,39 @@ -""" -Test to run multiple attacks (MIA and AIA) using a single configuration file -having different configuration settings (i.e. attack type or configuration parameters). - -Running -------- - -Invoke this code from the root AI-SDC folder. -However to run this test file, it will be required to install pytest package -using 'pip install pytest' and then run following -python -m pytest .\tests\test_multiple_attacks.py -""" +"""Test multiple attacks (MIA and AIA) using a single configuration file.""" from __future__ import annotations import json import os -import shutil import sys -# ignore unused imports because it depends on whether data file is present -from sklearn.datasets import fetch_openml # pylint:disable=unused-import +import pytest from sklearn.ensemble import RandomForestClassifier -from sklearn.preprocessing import ( # pylint:disable=unused-import - LabelEncoder, - OneHotEncoder, -) from aisdc.attacks.multiple_attacks import ConfigFile, MultipleAttacks -from ..common import get_target - -# pylint: disable = duplicate-code +def pytest_generate_tests(metafunc): + """Generate target model for testing.""" + if "get_target" in metafunc.fixturenames: + metafunc.parametrize( + "get_target", [RandomForestClassifier(bootstrap=False)], indirect=True + ) -def cleanup_file(name: str): - """Removes unwanted files or directory.""" - if os.path.exists(name): - if os.path.isfile(name): - os.remove(name) - elif os.path.isdir(name): - shutil.rmtree(name) - -def common_setup(): - """Basic commands to get ready to test some code.""" - model = RandomForestClassifier(bootstrap=False) - target = get_target(model) - model.fit(target.x_train, target.y_train) - attack_obj = MultipleAttacks( - config_filename="test_single_config.json", - ) +@pytest.fixture(name="common_setup") +def fixture_common_setup(get_target): + """Get ready to test some code.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + attack_obj = MultipleAttacks(config_filename="test_single_config.json") return target, attack_obj def create_single_config_file(): - """Creates single configuration file using multiple attack configuration.""" - # instantiating a configfile object to add configurations - configfile_obj = ConfigFile( - filename="test_single_config.json", - ) + """Create single config file using multiple attack configuration.""" + configfile_obj = ConfigFile(filename="test_single_config.json") - # Example 1: Adding three different worst case configuration dictionaries to the JSON file + # Example 1: Add 3 different worst case configuration dictionaries to JSON config = { "n_reps": 10, "n_dummy_reps": 1, @@ -104,7 +77,7 @@ def create_single_config_file(): } configfile_obj.add_config(config, "worst_case") - # Adding two different lira attack configuration dictionaries to the JSON file + # Add 2 different lira attack configuration dictionaries to JSON config = { "n_shadow_models": 100, "output_dir": "outputs_multiple_attacks", @@ -132,10 +105,10 @@ def create_single_config_file(): "target_model_hyp": {"min_samples_split": 2, "min_samples_leaf": 1}, } configfile_obj.add_config(config, "lira") - # adding explicitly wrong attack name to cover codecov test + # add explicitly wrong attack name to cover codecov test configfile_obj.add_config(config, "lirrra") - # Example 3: Adding a lira JSON configuration file to a configuration file + # Example 3: Add a lira JSON configuration file to a configuration file # having multiple attack configurations config = { "n_shadow_models": 120, @@ -154,8 +127,8 @@ def create_single_config_file(): f.write(json.dumps(config)) configfile_obj.add_config("test_lira_config.json", "lira") - # Example 4: Adding a attribute configuration dictionary - # from an existing configuration file to the JSON configuration file + # Example 4: Add an attribute configuration dictionary + # from an existing configuration file to JSON config = { "n_cpu": 2, "output_dir": "outputs_multiple_attacks", @@ -167,27 +140,25 @@ def create_single_config_file(): def test_configfile_number(): - """Tests number of attack configurations in a configuration file.""" + """Test attack configurations in a configuration file.""" configfile_obj = create_single_config_file() configfile_data = configfile_obj.read_config_file() assert len(configfile_data) == 8 os.remove("test_single_config.json") -def test_multiple_attacks_programmatic(): - """Tests programmatically running attacks using a single configuration configuration file.""" - target, attack_obj = common_setup() +def test_multiple_attacks_programmatic(common_setup): + """Test programmatically running attacks using a single config file.""" + target, attack_obj = common_setup _ = create_single_config_file() attack_obj.attack(target) print(attack_obj) os.remove("test_single_config.json") -def test_multiple_attacks_cmd(): - """Tests running multiple attacks (MIA and AIA) on the nursery data - with an added continuous feature. - """ - target, _ = common_setup() +def test_multiple_attacks_cmd(common_setup): + """Test multiple attacks (MIA and AIA) with a continuous feature.""" + target, _ = common_setup target.save(path=os.path.join("tests", "test_multiple_target")) _ = create_single_config_file() @@ -197,14 +168,3 @@ def test_multiple_attacks_cmd(): "--attack-config-json-file-name test_single_config.json " f"--attack-target-folder-path {multiple_target} " ) - - -def test_cleanup(): - """Tidies up any files created.""" - files_made = ( - "test_single_config.json", - "outputs_multiple_attacks", - os.path.join("tests", "test_multiple_target"), - ) - for fname in files_made: - cleanup_file(fname) diff --git a/tests/attacks/test_structural_attack.py b/tests/attacks/test_structural_attack.py index 20c07d5b..263a061d 100644 --- a/tests/attacks/test_structural_attack.py +++ b/tests/attacks/test_structural_attack.py @@ -1,6 +1,4 @@ -"""Test_worst_case_attack.py -Copyright (C) Jim Smith 2023 . -""" +"""Test structural attacks.""" from __future__ import annotations @@ -20,8 +18,6 @@ import aisdc.attacks.structural_attack as sa from aisdc.attacks.target import Target -from ..common import clean - def get_target(modeltype: str, **kwparams: dict) -> Target: """Loads dataset and creates target of the desired type.""" @@ -159,15 +155,15 @@ def test_unnecessary_risk(): def test_non_trees(): - """Test behaviour if model type not tree-based.""" + """Test behaviour if model type not tree-based.""" param_dict = {"probability": True} target = get_target("svc", **param_dict) myattack = sa.StructuralAttack() myattack.attack(target) # remove model target.model = None + myattack2 = sa.StructuralAttack() with pytest.raises(NotImplementedError): - myattack2 = sa.StructuralAttack() myattack2.attack(target) @@ -410,8 +406,3 @@ def test_main_example(): ] with patch.object(sys, "argv", testargs): sa.main() - - clean("dt.sav") - clean("test_output_sa") - clean("config_structural_test.json") - clean("outputs_structural") diff --git a/tests/attacks/test_worst_case_attack.py b/tests/attacks/test_worst_case_attack.py index 2bd9d83d..605fcde2 100644 --- a/tests/attacks/test_worst_case_attack.py +++ b/tests/attacks/test_worst_case_attack.py @@ -1,12 +1,9 @@ -"""Test_worst_case_attack.py -Copyright (C) Jim Smith 2022 . -""" +"""Test worst case attack.""" from __future__ import annotations import json import os -import shutil import sys from unittest.mock import patch @@ -20,15 +17,6 @@ from aisdc.attacks.target import Target -def clean_up(name): - """Removes unwanted files or directory.""" - if os.path.exists(name): - if os.path.isfile(name): - os.remove(name) - elif os.path.isdir(name): - shutil.rmtree(name) - - def test_config_file_arguments_parsin(): """Tests reading parameters from the configuration file.""" config = { @@ -353,24 +341,3 @@ def test_main(): testargs = ["prog", "run-attack"] with patch.object(sys, "argv", testargs): worst_case_attack.main() - - # wrong args - - # testargs = ["prog", "run-attack","--no-such-arg"] - # with patch.object(sys, 'argv', testargs): - # worst_case_attack.main() - - -def test_cleanup(): - """Gets rid of files created during tests.""" - names = [ - "test_output_worstcase", - "output_worstcase", - "test_worstcase_target", - "test_preds.csv", - "train_preds.csv", - "ypred_test.csv", - "ypred_train.csv", - ] - for name in names: - clean_up(name) diff --git a/tests/common.py b/tests/conftest.py similarity index 60% rename from tests/common.py rename to tests/conftest.py index 3f3ec392..c67a5528 100644 --- a/tests/common.py +++ b/tests/conftest.py @@ -1,11 +1,10 @@ """Common utility functions for testing.""" -from __future__ import annotations - import os import shutil import numpy as np +import pytest import sklearn from sklearn.datasets import fetch_openml from sklearn.model_selection import train_test_split @@ -13,23 +12,88 @@ from aisdc.attacks.target import Target - -def clean(name): - """Removes unwanted files or directory.""" - if os.path.exists(name): - if os.path.isfile(name): - os.remove(name) - elif os.path.isdir(name): - shutil.rmtree(name) - - -def get_target( # pylint: disable=too-many-locals - model: sklearn.base.BaseEstimator, -) -> Target: - """ - Wrap the model and data in a Target object. - Uses A randomly sampled 10+10% of the nursery data set. +folders = [ + "RES", + "dt.sav", + "fit.tf", + "fit2.tf", + "keras_save.tf", + "output_attribute", + "output_lira", + "output_worstcase", + "outputs_lira", + "outputs_multiple_attacks", + "outputs_structural", + "refit.tf", + "release_dir", + "safekeras.tf", + "save_test", + "test_lira_target", + "test_output_lira", + "test_output_sa", + "test_output_worstcase", + "test_worstcase_target", + "tests/test_aia_target", + "tests/test_multiple_target", + "tfsaves", + "training_artefacts", +] + +files = [ + "1024-WorstCase.png", + "2048-WorstCase.png", + "ATTACK_RESULTS09_06_2024.json", + "attack.txt", + "config.json", + "config_structural_test.json", + "dummy.pkl", + "dummy.sav", + "dummy_model.txt", + "example_filename.json", + "filename_should_be_changed.txt", + "filename_to_rewrite.json", + "results.txt", + "rf_test.pkl", + "rf_test.sav", + "safekeras.h5", + "target.json", + "test.json", + "test_data.csv", + "test_preds.csv", + "test_single_config.json", + "tests/test_config_aia_cmd.json", + "train_data.csv", + "train_preds.csv", + "unpicklable.pkl", + "unpicklable.sav", + "ypred_test.csv", + "ypred_train.csv", +] + + +@pytest.fixture(name="cleanup", autouse=True) +def _cleanup(): + """Remove created files and directories.""" + yield + for folder in folders: + try: + shutil.rmtree(folder) + except Exception: # pylint: disable=broad-exception-caught + pass + for file in files: + try: + os.remove(file) + except Exception: # pylint: disable=broad-exception-caught + pass + + +@pytest.fixture() +def get_target(request) -> Target: # pylint: disable=too-many-locals + """Wrap the model and data in a Target object. + + Uses a randomly sampled 10+10% of the nursery data set. """ + model: sklearn.BaseEstimator = request.param nursery_data = fetch_openml(data_id=26, as_frame=True) x = np.asarray(nursery_data.data, dtype=str) diff --git a/tests/attacks/test_data_interface.py b/tests/preprocessing/test_data_interface.py similarity index 75% rename from tests/attacks/test_data_interface.py rename to tests/preprocessing/test_data_interface.py index 04b8e71d..111dfdf7 100644 --- a/tests/attacks/test_data_interface.py +++ b/tests/preprocessing/test_data_interface.py @@ -5,6 +5,7 @@ import unittest import pandas as pd +import pytest from aisdc.preprocessing.loaders import UnknownDataset, get_data_sklearn @@ -15,12 +16,12 @@ class TestLoaders(unittest.TestCase): def test_iris(self): """Nursery data.""" feature_df, target_df = get_data_sklearn("iris") - self.assertIsInstance(feature_df, pd.DataFrame) - self.assertIsInstance(target_df, pd.DataFrame) + assert isinstance(feature_df, pd.DataFrame) + assert isinstance(target_df, pd.DataFrame) def test_unknown(self): """Test that a nonsense string raises the correct exception.""" - with self.assertRaises(UnknownDataset): + with pytest.raises(UnknownDataset): _, _ = get_data_sklearn("NONSENSE") def test_standard(self): @@ -28,15 +29,15 @@ def test_standard(self): feature_df, _ = get_data_sklearn("standard iris") for column in feature_df.columns: temp = feature_df[column].mean() - self.assertAlmostEqual(temp, 0.0) + assert temp == pytest.approx(0) temp = feature_df[column].std() - self.assertAlmostEqual(temp, 1.0) + assert temp == pytest.approx(1) def test_minmax(self): """Test the minmax scaling.""" feature_df, _ = get_data_sklearn("minmax iris") for column in feature_df.columns: temp = feature_df[column].min() - self.assertAlmostEqual(temp, 0.0) + assert temp == pytest.approx(0) temp = feature_df[column].max() - self.assertAlmostEqual(temp, 1.0) + assert temp == pytest.approx(1) diff --git a/tests/preprocessing/test_loaders.py b/tests/preprocessing/test_loaders.py index e38b6bb9..28450c4d 100644 --- a/tests/preprocessing/test_loaders.py +++ b/tests/preprocessing/test_loaders.py @@ -1,10 +1,4 @@ -"""Test_loaders.py -Series of functions to use with pytest to check the loaders classes -Most use just the truncated files with first five examples of each class for brevity. -Please access the datasets from the sources listed in preprocessing/loaders.py -Please acknowledge those sources in any publications. -Jim Smith 2022. -""" +"""Test data loaders.""" from __future__ import annotations @@ -210,13 +204,3 @@ def test_synth_ae(): x_df, y_df = loaders.get_data_sklearn("synth-ae-XXL", DATA_FOLDER) assert x_df.shape == (8, 16), f"x_df shape is {x_df.shape}" assert y_df.shape == (8, 1) - - -# def test_rdmp(): -# """The RDMP dataloader.""" -# x_df, y_df = loaders.get_data_sklearn("RDMP", DATA_FOLDER) - -# assert 'death' not in x_df.columns -# assert 'age' in x_df.columns - -# assert y_df.shape[1] == 1 diff --git a/tests/safemodel/test_attacks.py b/tests/safemodel/test_attacks.py index fadb7db7..12c296d1 100644 --- a/tests/safemodel/test_attacks.py +++ b/tests/safemodel/test_attacks.py @@ -1,7 +1,4 @@ -"""Jim Smith October 2022 -tests to pick up odd cases not otherwise covered -in code in the attacks folder. -""" +"""Tests to pick up odd cases not otherwise covered in code in the attacks folder.""" from __future__ import annotations @@ -24,13 +21,11 @@ def test_superclass(): with pytest.raises(NotImplementedError): my_attack.attack(target) with pytest.raises(NotImplementedError): - print(str(my_attack)) # .__str__() + print(str(my_attack)) def test_NumpyArrayEncoder(): - """Conversion routine - from reports.py. - """ + """Conversion routine from reports.py.""" i32 = np.int32(2) i64 = np.int64(2) diff --git a/tests/safemodel/test_attacks_via_safemodel.py b/tests/safemodel/test_attacks_via_safemodel.py index b0fb9d33..99e48db2 100644 --- a/tests/safemodel/test_attacks_via_safemodel.py +++ b/tests/safemodel/test_attacks_via_safemodel.py @@ -1,79 +1,79 @@ -""" -Tests attacks called via safemodel classes -uses a subsampled nursery dataset as this tests more of the attack code -currently using random forests. -""" +"""Test attacks called via safemodel classes.""" from __future__ import annotations -import shutil - import numpy as np +import pytest from aisdc.attacks import attribute_attack, likelihood_attack, worst_case_attack from aisdc.safemodel.classifiers import SafeDecisionTreeClassifier -from ..common import clean, get_target - -# pylint: disable=duplicate-code,unnecessary-dunder-call - RES_DIR = "RES" -def test_attacks_via_request_release(): - """Make vulnerable,hacked model then call request_release.""" - # build a broken model and hack it so lots of reasons to fail and be vulnerable - model = SafeDecisionTreeClassifier(random_state=1, max_depth=10, min_samples_leaf=1) - target = get_target(model) - assert target.__str__() == "nursery" - model.fit(target.x_train, target.y_train) - model.min_samples_leaf = 10 - model.request_release(path=RES_DIR, ext="pkl", target=target) - clean(RES_DIR) - - -def test_run_attack_lira(): - """Calls the lira attack via safemodel.""" - # build a model - model = SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=10) - target = get_target(model) - model.fit(target.x_train, target.y_train) - _, disclosive = model.preliminary_check() +@pytest.mark.parametrize( + "get_target", + [SafeDecisionTreeClassifier(random_state=1, max_depth=10, min_samples_leaf=1)], + indirect=True, +) +def test_attacks_via_request_release(get_target): + """Test vulnerable, hacked model then call request_release.""" + target = get_target + assert target.__str__() == "nursery" # pylint: disable=unnecessary-dunder-call + target.model.fit(target.x_train, target.y_train) + target.model.min_samples_leaf = 10 + target.model.request_release(path=RES_DIR, ext="pkl", target=target) + + +@pytest.mark.parametrize( + "get_target", + [SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=10)], + indirect=True, +) +def test_run_attack_lira(get_target): + """Test the lira attack via safemodel.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + _, disclosive = target.model.preliminary_check() assert not disclosive - print(np.unique(target.y_test, return_counts=True)) - print(np.unique(model.predict(target.x_test), return_counts=True)) - metadata = model.run_attack(target, "lira", RES_DIR, "lira_res") - clean(RES_DIR) + print(np.unique(target.model.predict(target.x_test), return_counts=True)) + metadata = target.model.run_attack(target, "lira", RES_DIR, "lira_res") assert len(metadata) > 0 # something has been added -def test_run_attack_worstcase(): - """Calls the worst case attack via safemodel.""" - model = SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=20) - target = get_target(model) - model.fit(target.x_train, target.y_train) - _, disclosive = model.preliminary_check() +@pytest.mark.parametrize( + "get_target", + [SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=20)], + indirect=True, +) +def test_run_attack_worstcase(get_target): + """Test the worst case attack via safemodel.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + _, disclosive = target.model.preliminary_check() assert not disclosive - metadata = model.run_attack(target, "worst_case", RES_DIR, "wc_res") - clean(RES_DIR) + metadata = target.model.run_attack(target, "worst_case", RES_DIR, "wc_res") assert len(metadata) > 0 # something has been added -def test_run_attack_attribute(): - """Calls the attribute attack via safemodel.""" - model = SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=10) - target = get_target(model) - model.fit(target.x_train, target.y_train) - _, disclosive = model.preliminary_check() +@pytest.mark.parametrize( + "get_target", + [SafeDecisionTreeClassifier(random_state=1, max_depth=5, min_samples_leaf=10)], + indirect=True, +) +def test_run_attack_attribute(get_target): + """Test the attribute attack via safemodel.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + _, disclosive = target.model.preliminary_check() assert not disclosive - metadata = model.run_attack(target, "attribute", RES_DIR, "attr_res") - clean(RES_DIR) + metadata = target.model.run_attack(target, "attribute", RES_DIR, "attr_res") assert len(metadata) > 0 # something has been added def test_attack_args(): - """Tests the attack arguments class.""" + """Test the attack arguments class.""" fname = "aia_example" attack_obj = attribute_attack.AttributeAttack( output_dir="output_attribute", report_name=fname @@ -97,17 +97,16 @@ def test_attack_args(): attack_obj.__dict__["foo"] = "boo" assert attack_obj.__dict__["foo"] == "boo" assert fname == attack_obj.report_name - shutil.rmtree("output_attribute") - shutil.rmtree("output_lira") - shutil.rmtree("output_worstcase") - - -def test_run_attack_unknown(): - """Calls an unknown attack via safemodel.""" - # build a model - model = SafeDecisionTreeClassifier(random_state=1, max_depth=5) - target = get_target(model) - model.fit(target.x_train, target.y_train) - metadata = model.run_attack(target, "unknown", RES_DIR, "unk") - clean(RES_DIR) + + +@pytest.mark.parametrize( + "get_target", + [SafeDecisionTreeClassifier(random_state=1, max_depth=5)], + indirect=True, +) +def test_run_attack_unknown(get_target): + """Test an unknown attack via safemodel.""" + target = get_target + target.model.fit(target.x_train, target.y_train) + metadata = target.model.run_attack(target, "unknown", RES_DIR, "unk") assert metadata["outcome"] == "unrecognised attack type requested" diff --git a/tests/safemodel/test_safedecisiontreeclassifier.py b/tests/safemodel/test_safedecisiontreeclassifier.py index a543ce18..f0302ce9 100644 --- a/tests/safemodel/test_safedecisiontreeclassifier.py +++ b/tests/safemodel/test_safedecisiontreeclassifier.py @@ -1,4 +1,4 @@ -"""This module contains unit tests for the SafeDecisionTreeClassifier.""" +"""Test SafeDecisionTreeClassifier.""" from __future__ import annotations @@ -18,7 +18,7 @@ def get_data(): - """Returns data for testing.""" + """Return data for testing.""" iris = datasets.load_iris() x = np.asarray(iris["data"], dtype=np.float64) y = np.asarray(iris["target"], dtype=np.float64) @@ -48,32 +48,31 @@ def test_decision_trees_are_equal(): # one or both untrained model3 = SafeDecisionTreeClassifier(random_state=1, max_depth=7) same, msg = decision_trees_are_equal(model1, model3) - assert same is False + assert not same assert len(msg) > 0 same, msg = decision_trees_are_equal(model3, model1) - assert same is False + assert not same assert len(msg) > 0 same, msg = decision_trees_are_equal(model3, model3) - assert same is True - # assert len(msg)>0 + assert same # different - # x2=x1+1 model3 = SafeDecisionTreeClassifier(random_state=1, max_depth=7) model3.criterion = "entropy" model3.fit(x1, y) same, msg = decision_trees_are_equal(model1, model3) print(f"diff msg = {msg}") - assert same is False + assert not same # wrong type same, _ = decision_trees_are_equal(model1, "aString") - assert same is False + assert not same def test_get_tree_k_anonymity(): - """Getting k_anonymity - 50 data points randomly split in 2, single layer + """Test getting k_anonymity. + + 50 data points randomly split in 2, single layer. so k should be ~25. """ x = np.random.rand(50, 2) @@ -84,12 +83,11 @@ def test_get_tree_k_anonymity(): model = SafeDecisionTreeClassifier(random_state=1, max_depth=1) model.fit(x, y) k = get_tree_k_anonymity(model, x) - # print(f'k={k}') assert k > 10 def test_decisiontree_unchanged(): - """SafeDecisionTreeClassifier using unchanged values.""" + """Test using unchanged values.""" x, y = get_data() model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=5) model.fit(x, y) @@ -97,11 +95,11 @@ def test_decisiontree_unchanged(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_decisiontree_safe_recommended(): - """SafeDecisionTreeClassifier using recommended values.""" + """Test using recommended values.""" x, y = get_data() model = SafeDecisionTreeClassifier( random_state=1, max_depth=10, min_samples_leaf=10 @@ -112,11 +110,11 @@ def test_decisiontree_safe_recommended(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_decisiontree_safe_1(): - """SafeDecisionTreeClassifier with safe changes.""" + """Test safe changes.""" x, y = get_data() model = SafeDecisionTreeClassifier( random_state=1, max_depth=10, min_samples_leaf=10 @@ -127,11 +125,11 @@ def test_decisiontree_safe_1(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_decisiontree_safe_2(): - """SafeDecisionTreeClassifier with safe changes.""" + """Test safe changes.""" x, y = get_data() model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=10) model.fit(x, y) @@ -139,11 +137,11 @@ def test_decisiontree_safe_2(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_decisiontree_unsafe_1(): - """SafeDecisionTreeClassifier with unsafe changes.""" + """Test unsafe changes.""" x, y = get_data() model = SafeDecisionTreeClassifier( random_state=1, max_depth=10, min_samples_leaf=10 @@ -158,24 +156,11 @@ def test_decisiontree_unsafe_1(): "min value of 5." ) assert msg == correct_msg - assert disclosive is True - - -# no longer relevant because of changed default functionality of priminary_checK() -# def test_decisiontree_unsafe_2(): -# """SafeDecisionTreeClassifier with unsafe changes - automatically fixed.""" -# x, y = get_data() -# model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=1) -# model.fit(x, y) -# assert model.score(x, y) == 0.9668874172185431 -# msg, disclosive = model.preliminary_check() -# correct_msg = "Model parameters are within recommended ranges.\n" -# assert msg == correct_msg -# assert disclosive is False + assert disclosive def test_decisiontree_save(): - """SafeDecisionTreeClassifier model saving.""" + """Test model saving.""" x, y = get_data() model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=50) model.fit(x, y) @@ -190,7 +175,6 @@ def test_decisiontree_save(): with open("dt_test.sav", "rb") as file: sav_model = joblib.load(file) assert sav_model.score(x, y) == 0.9470198675496688 - # cleanup for name in ("dt_test.pkl", "dt_test.sav"): if os.path.exists(name) and os.path.isfile(name): @@ -198,7 +182,7 @@ def test_decisiontree_save(): def test_decisiontree_hacked_postfit(): - """SafeDecisionTreeClassifier changes made to parameters after fit() called.""" + """Test changes made to parameters after fit() called.""" x, y = get_data() model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=1) model.min_samples_leaf = 1 @@ -215,7 +199,7 @@ def test_decisiontree_hacked_postfit(): correct_msg1 = reporting.get_reporting_string(name="within_recommended_ranges") print(f"correct msg1: {correct_msg1}\n", f"actual msg1: {msg1}") assert msg1 == correct_msg1 - assert disclosive is False + assert not disclosive # but on closer inspection not msg2, disclosive2 = model.posthoc_check() @@ -231,13 +215,14 @@ def test_decisiontree_hacked_postfit(): correct_msg2 = part1 + part2 + part3 + part4 print(f"Correct msg2 : {correct_msg2}\n" f"actual mesg2 : {msg2}") assert msg2 == correct_msg2 - assert disclosive2 is True + assert disclosive2 def test_data_hiding(): - """What if the hacking was really obscure - like putting something in the exceptions list - then adding data to current and saved copies. + """Test if the hacking was really obscure. + + Example: putting something in the exceptions list then adding data to + current and saved copies. """ x, y = get_data() model = SafeDecisionTreeClassifier(random_state=1, min_samples_leaf=5) diff --git a/tests/safemodel/test_safekeras2.py b/tests/safemodel/test_safekeras2.py index 5fbac49f..264c6c8a 100644 --- a/tests/safemodel/test_safekeras2.py +++ b/tests/safemodel/test_safekeras2.py @@ -1,9 +1,8 @@ -"""This module contains unit tests for SafeKerasModel.""" +"""Test SafeKerasModel.""" from __future__ import annotations import os -import shutil import warnings import numpy as np @@ -11,7 +10,7 @@ import tensorflow as tf from sklearn import datasets from sklearn.model_selection import train_test_split -from tensorflow.keras.layers import Dense, Input # pylint: disable = import-error +from tensorflow.keras.layers import Dense, Input # pylint: disable=import-error from aisdc.safemodel.classifiers import SafeKerasModel, safekeras from aisdc.safemodel.reporting import get_reporting_string @@ -30,21 +29,8 @@ RES_DIR = "RES" -def clean(): - """Removes unwanted results.""" - shutil.rmtree(RES_DIR) - - -def cleanup_file(name: str): - """Removes unwanted files or directory.""" - if os.path.exists(name) and os.path.isfile(name): # h5 - os.remove(name) - elif os.path.exists(name) and os.path.isdir(name): # tf - shutil.rmtree(name) - - def get_data(): - """Returns data for testing.""" + """Return data for testing.""" iris = datasets.load_iris() xall = np.asarray(iris["data"], dtype=np.float64) yall = np.asarray(iris["target"], dtype=np.float64) @@ -84,9 +70,7 @@ def make_small_model(num_hidden_layers=2): def check_init_completed(model: SafeKerasModel): - """Basic checks for things that happen - at end of correct init. - """ + """Test basic checks for things that happen at end of correct init.""" rightname = "KerasModel" assert ( model.model_type == rightname @@ -99,8 +83,9 @@ def check_init_completed(model: SafeKerasModel): def test_init_variants(): - """Test alternative ways of calling init - do just with single layer for speed of testing. + """Test alternative ways of calling init. + + Just with single layer for speed of testing. """ # get data X, _, _, _ = get_data() @@ -147,7 +132,7 @@ def test_init_variants(): def test_same_configs(): # pylint: disable=too-many-locals - """Check whether tests for equal configuration work.""" + """Test whether tests for equal configuration work.""" model1, X, _, _, _ = make_small_model(num_hidden_layers=1) model2, _, _, _, _ = make_small_model(num_hidden_layers=2) @@ -159,7 +144,7 @@ def test_same_configs(): # pylint: disable=too-many-locals f"model1 has {len(model1.layers)} layers, but\n" f"model2 has {len(model2.layers)} layers\n" ) - assert same1 is False, errstr + assert not same1, errstr correct_msg1 = get_reporting_string(name="different_layer_count") errstr = f"msg was: {msg1}\n" f"should be : {correct_msg1}" assert msg1 == correct_msg1, errstr @@ -170,7 +155,7 @@ def test_same_configs(): # pylint: disable=too-many-locals assert msg2 == correct_msg2, ( rf"should report {correct_msg2}\," f" but got {msg2}.\n" ) - assert same2 is True, "models are same!" + assert same2, "models are same!" # same layers, different widths input_data = Input(shape=X[0].shape) @@ -187,7 +172,7 @@ def test_same_configs(): # pylint: disable=too-many-locals check_init_completed(model1a) same3, msg3 = safekeras.same_configs(model1, model1a) errmsg = "Should report layers have different num nodes" - assert same3 is False, errmsg + assert not same3, errmsg correct_msg3 = get_reporting_string(name="layer_configs_differ", layer=1, length=1) correct_msg3 += get_reporting_string( name="param_changed_from_to", key="units", val="32", cur_val=64 @@ -197,7 +182,7 @@ def test_same_configs(): # pylint: disable=too-many-locals def test_same_weights(): # pylint : disable=too-many-locals - """Check the same weights method catches differences.""" + """Test the same weights method catches differences.""" # make models to test model1, X, _, _, _ = make_small_model(num_hidden_layers=1) model2, _, _, _, _ = make_small_model(num_hidden_layers=2) @@ -215,11 +200,11 @@ def test_same_weights(): # pylint : disable=too-many-locals # same same1, _ = safekeras.same_weights(model1, model1) - assert same1 is True + assert same1 # different num layers same2, _ = safekeras.same_weights(model1, model2) - assert same2 is False + assert not same2 # different sized layers same3, _ = safekeras.same_weights(model1, model1a) @@ -228,11 +213,11 @@ def test_same_weights(): # pylint : disable=too-many-locals f" {len(model1.layers[1].get_weights()[0][0])} units" f" but model2 has {len(model1a.layers[1].get_weights()[0][0])}.\n" ) - assert same3 is False, errstr + assert not same3, errstr def test_DP_optimizer_checks(): - """Tests the various checks that DP optimiser was used.""" + """Test the various checks that DP optimiser was used.""" # make model model1, _, _, _, _ = make_small_model(num_hidden_layers=1) loss = tf.keras.losses.CategoricalCrossentropy( @@ -249,9 +234,9 @@ def test_DP_optimizer_checks(): model, _, _, _, _ = make_small_model(num_hidden_layers=1) model.compile(loss=loss, optimizer=oktype) opt_ok, msg = safekeras.check_optimizer_allowed(model.optimizer) - assert opt_ok is True, msg + assert opt_ok, msg opt_is_dp, _ = safekeras.check_optimizer_is_DP(model.optimizer) - assert opt_is_dp is True + assert opt_is_dp # not ok optimizer model, _, _, _, _ = make_small_model(num_hidden_layers=1) @@ -259,24 +244,24 @@ def test_DP_optimizer_checks(): # reset to not DP optimizer model.optimizer = tf.keras.optimizers.get("SGD") opt_ok, msg = safekeras.check_optimizer_allowed(model1.optimizer) - assert opt_ok is False, msg + assert not opt_ok, msg opt_is_dp, msg = safekeras.check_optimizer_is_DP(model1.optimizer) - assert opt_is_dp is False, msg + assert not opt_is_dp, msg def test_DP_used(): - """Tests the various checks that DP optimiser was used.""" - # should pass aftyer model compiled **and** fitted with DP optimizer + """Test the various checks that DP optimiser was used.""" + # should pass after model compiled **and** fitted with DP optimizer model1, X, y, Xval, yval = make_small_model(num_hidden_layers=1) loss = tf.keras.losses.CategoricalCrossentropy( from_logits=False, reduction=tf.losses.Reduction.NONE ) model1.compile(loss=loss) dp_used, msg = safekeras.check_DP_used(model1.optimizer) - assert dp_used is False + assert not dp_used model1.fit(X, y, validation_data=(Xval, yval), epochs=EPOCHS, batch_size=20) dp_used, msg = safekeras.check_DP_used(model1.optimizer) - assert dp_used is True + assert dp_used # this model gets changed to non-DP by calling the superclass compile() # so should fail all the checks @@ -284,7 +269,7 @@ def test_DP_used(): super(SafeKerasModel, model2).compile(loss=loss, optimizer="SGD") model2.fit(X, y, validation_data=(Xval, yval), epochs=EPOCHS, batch_size=20) dp_used, msg = safekeras.check_DP_used(model2.optimizer) - assert dp_used is False, msg + assert not dp_used, msg def test_checkpoints_are_equal(): @@ -301,12 +286,12 @@ def test_checkpoints_are_equal(): # same arch, different weights same, msg = safekeras.check_checkpoint_equality("fit.tf", "refit.tf") - assert same is False, msg + assert not same, msg # should be same same, msg = safekeras.check_checkpoint_equality("fit.tf", "fit.tf") print(msg) - assert same is True, msg + assert same, msg # different architecture model2, X, y, Xval, yval = make_small_model(num_hidden_layers=3) @@ -316,29 +301,25 @@ def test_checkpoints_are_equal(): same, msg = safekeras.check_checkpoint_equality("fit.tf", "fit2.tf") print(msg) - assert same is False, msg + assert not same, msg # coping with trashed files - cleanup_file("fit.tf/saved_model.pb") same, msg = safekeras.check_checkpoint_equality("fit.tf", "fit2.tf") - assert same is False, msg + assert not same, msg same, msg = safekeras.check_checkpoint_equality("fit2.tf", "fit.tf") - assert same is False, msg + assert not same, msg same, msg = safekeras.check_checkpoint_equality("hello", "fit2.tf") - assert same is False + assert not same assert "Error re-loading model from" in msg same, msg = safekeras.check_checkpoint_equality("fit2.tf", "hello") - assert same is False + assert not same assert "Error re-loading model from" in msg - for name in ("fit.tf", "fit2.tf", "refit.tf"): - cleanup_file(name) - def test_load(): - """Tests the oading functionality.""" + """Test the loading functionality.""" # make a model, train then save it model, X, y, Xval, yval = make_small_model() @@ -351,21 +332,18 @@ def test_load(): # won't load with invalid names ok, _ = safekeras.load_safe_keras_model() - assert ok is False, "can't load with no model file name" + assert not ok, "can't load with no model file name" ok, _ = safekeras.load_safe_keras_model("keras_save.h5") - assert ok is False, "can only load from .tf file" + assert not ok, "can only load from .tf file" # should load fine with right name ok, reloaded_model = safekeras.load_safe_keras_model("keras_save.tf") - assert ok is True + assert ok ypred = "over-write-me" ypred = reloaded_model.predict(X) assert isinstance(ypred, np.ndarray) - cleanup_file("keras_save.tf") - cleanup_file("tfsaves") - def test_keras_model_created(): """Test keras model.""" @@ -434,7 +412,7 @@ def test_keras_model_compiled_as_DP(): def test_keras_basic_fit(): - """SafeKeras using recommended values.""" + """Test SafeKeras using recommended values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -454,7 +432,7 @@ def test_keras_basic_fit(): batch_size=X.shape[0], refine_epsilon=True, ) - assert ok is False + assert not ok # now default (False) model.fit(X, y, validation_data=(Xval, yval), epochs=EPOCHS, batch_size=20) @@ -473,7 +451,7 @@ def test_keras_basic_fit(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg, "failed check params are within range" - assert disclosive is False, "failed check disclosive is false" + assert not disclosive, "failed check disclosive is false" def test_keras_save_actions(): @@ -490,27 +468,19 @@ def test_keras_save_actions(): # start with .tf and .h5 which should work names = ("safekeras.tf", "safekeras.h5") for name in names: - # clear existing files - cleanup_file(name) # save file model.save(name) assert os.path.exists(name), f"Failed test to save model as {name}" - # clean up - cleanup_file(name) # now other versions which should not names = ("safekeras.sav", "safekeras.pkl", "randomfilename", "undefined") for name in names: - cleanup_file(name) model.save(name) - assert os.path.exists(name) is False, f"Failed test NOT to save model as {name}" - cleanup_file(name) - # cleeanup - cleanup_file("tfsaves") + assert not os.path.exists(name), f"Failed test NOT to save model as {name}" def test_keras_unsafe_l2_norm(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -543,11 +513,11 @@ def test_keras_unsafe_l2_norm(): "min value of 1.0." ) assert msg == correct_msg, "failed check correct warning message" - assert disclosive is True, "failed check disclosive is True" + assert disclosive, "failed check disclosive is True" def test_keras_unsafe_noise_multiplier(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -581,11 +551,11 @@ def test_keras_unsafe_noise_multiplier(): ) assert msg == correct_msg, "failed check params are within range" - assert disclosive is True, "failed check disclosive is True" + assert disclosive, "failed check disclosive is True" def test_keras_unsafe_min_epsilon(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -618,11 +588,11 @@ def test_keras_unsafe_min_epsilon(): ) assert msg == correct_msg, "failed check correct warning message" - assert disclosive is True, "failed check disclosive is True" + assert disclosive, "failed check disclosive is True" def test_keras_unsafe_delta(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -654,11 +624,11 @@ def test_keras_unsafe_delta(): "- parameter delta = 1e-06 identified as less than the recommended min value of 1e-05." ) assert msg == correct_msg, "failed check params are within range" - assert disclosive is True, "failed check disclosive is True" + assert disclosive, "failed check disclosive is True" def test_keras_unsafe_batch_size(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -687,11 +657,11 @@ def test_keras_unsafe_batch_size(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg, "failed check params are within range" - assert disclosive is False, "failed check disclosive is false" + assert not disclosive, "failed check disclosive is false" def test_keras_unsafe_learning_rate(): - """SafeKeras using unsafe values.""" + """Test SafeKeras using unsafe values.""" model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -721,7 +691,7 @@ def test_keras_unsafe_learning_rate(): correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg, "failed check warning message incorrect" - assert disclosive is False, "failed check disclosive is false" + assert not disclosive, "failed check disclosive is false" def test_create_checkfile(): @@ -743,26 +713,23 @@ def test_create_checkfile(): # check save file model.save(name) assert os.path.exists(name), f"Failed test to save model as {name}" - clean() # check release model.request_release(path=RES_DIR, ext=ext) assert os.path.exists(name), f"Failed test to save model as {name}" name = os.path.normpath(f"{RES_DIR}/target.json") assert os.path.exists(name), "Failed test to save target.json" - clean() # now other versions which should not exts = ("sav", "pkl", "undefined") for ext in exts: - name = os.path.normpath(f"{RES_DIR}/model.{ext}") + name = os.path.normpath(f"{RES_DIR}/cfmodel.{ext}") os.makedirs(os.path.dirname(name), exist_ok=True) model.save(name) - assert os.path.exists(name) is False, f"Failed test NOT to save model as {name}" - clean() + assert not os.path.exists(name), f"Failed test NOT to save model as {name}" def test_posthoc_check(): - """Testing the posthoc checking function.""" + """Test the posthoc checking function.""" # make a model, train then save it model, X, y, Xval, yval = make_small_model() loss = tf.keras.losses.CategoricalCrossentropy( @@ -773,20 +740,11 @@ def test_posthoc_check(): # should be ok _, disclosive = model.posthoc_check() - assert disclosive is False, "base config in tests should be ok" + assert not disclosive, "base config in tests should be ok" # change optimizer and some other settings # in way that stresses lots of routes - cleanup_file("tfsaves/fit_model.tf") model.epochs = 1000 model.optimizer = tf.keras.optimizers.get("SGD") _, disclosive = model.posthoc_check() - assert disclosive is True, "should pick up optimizer changed" - - cleanup_file("keras_save.tf") - cleanup_file("tfsaves") - - -def test_final_cleanup(): - """Clean up any files let around by other tests.""" - cleanup_file("tfsaves") + assert disclosive, "should pick up optimizer changed" diff --git a/tests/safemodel/test_safemodel.py b/tests/safemodel/test_safemodel.py index e2761c8e..e6e2721d 100644 --- a/tests/safemodel/test_safemodel.py +++ b/tests/safemodel/test_safemodel.py @@ -1,4 +1,4 @@ -"""Tests for fnctionality in super class.""" +"""Test safemodel super class.""" from __future__ import annotations @@ -14,8 +14,6 @@ from aisdc.safemodel.reporting import get_reporting_string from aisdc.safemodel.safemodel import SafeModel -from ..common import clean - notok_start = get_reporting_string(name="warn_possible_disclosure_risk") ok_start = get_reporting_string(name="within_recommended_ranges") @@ -43,7 +41,7 @@ def predict(self, x: np.ndarray): # pragma: no cover def get_data(): - """Returns data for testing.""" + """Return data for testing.""" iris = datasets.load_iris() x = np.asarray(iris["data"], dtype=np.float64) y = np.asarray(iris["target"], dtype=np.float64) @@ -52,13 +50,11 @@ def get_data(): return x, y -class SafeDummyClassifier( - SafeModel, DummyClassifier -): # pylint:disable=too-many-instance-attributes +class SafeDummyClassifier(SafeModel, DummyClassifier): # pylint:disable=too-many-instance-attributes """Privacy protected dummy classifier.""" def __init__(self, **kwargs) -> None: - """Creates model and applies constraints to params.""" + """Create model and applies constraints to params.""" SafeModel.__init__(self) self.basemodel_paramnames = ( "at_least_5f", @@ -84,17 +80,17 @@ def __init__(self, **kwargs) -> None: self.newthing = ["myStringKey", "aString", "myIntKey", "42"] def set_params(self, **kwargs): # pragma: no cover - """Sets params.""" - for key, val in kwargs.items(): # pylint:disable=unused-variable + """Set params.""" + for _, val in kwargs.items(): self.key = val # pylint:disable=attribute-defined-outside-init - def fit(self, x: np.ndarray, y: np.ndarray): + def fit(self, x: np.ndarray, y: np.ndarray): # noqa: ARG002 """Dummy fit.""" self.saved_model = copy.deepcopy(self.__dict__) def test_params_checks_ok(): - """Test parameter checks ok.""" + """Test parameter checks ok.""" model = SafeDummyClassifier() correct_msg = ok_start @@ -103,16 +99,16 @@ def test_params_checks_ok(): f"exactly_boo is {model.exactly_boo} with type{type(model.exactly_boo).__name__}" ) assert msg == ok_start, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive def test_params_checks_too_low(): - """Test parameter checks too low.""" + """Test parameter checks too low.""" model = SafeDummyClassifier() model.at_least_5f = 4.0 msg, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start + get_reporting_string( name="less_than_min_value", key="at_least_5f", @@ -123,12 +119,12 @@ def test_params_checks_too_low(): def test_params_checks_too_high(): - """Test parameter checks too high.""" + """Test parameter checks too high.""" model = SafeDummyClassifier() model.at_most_5i = 6 msg, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start + get_reporting_string( name="greater_than_max_value", key="at_most_5i", cur_val=model.at_most_5i, val=5 ) @@ -136,12 +132,12 @@ def test_params_checks_too_high(): def test_params_checks_not_equal(): - """Test parameter checks not equal.""" + """Test parameter checks not equal.""" model = SafeDummyClassifier() model.exactly_boo = "foo" msg, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start + get_reporting_string( name="different_than_fixed_value", key="exactly_boo", @@ -152,14 +148,14 @@ def test_params_checks_not_equal(): def test_params_checks_wrong_type_str(): - """Test parameter checks wrong type - strings given.""" + """Test parameter checks wrong type - strings given.""" model = SafeDummyClassifier() model.at_least_5f = "five" model.at_most_5i = "five" msg, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start correct_msg += get_reporting_string( name="different_than_recommended_type", @@ -190,14 +186,14 @@ def test_params_checks_wrong_type_str(): def test_params_checks_wrong_type_float(): - """Test parameter checks wrong_type_float.""" + """Test parameter checks wrong_type_float.""" model = SafeDummyClassifier() - model.exactly_boo = 5.0 - model.at_most_5i = 5.0 + model.exactly_boo = 5 + model.at_most_5i = 5 _, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start correct_msg += get_reporting_string( @@ -215,14 +211,14 @@ def test_params_checks_wrong_type_float(): def test_params_checks_wrong_type_int(): - """Test parameter checks wrong_type_intt.""" + """Test parameter checks wrong_type_intt.""" model = SafeDummyClassifier() model.exactly_boo = 5 model.at_least_5f = 5 msg, disclosive = model.preliminary_check() - assert disclosive is True + assert disclosive correct_msg = notok_start correct_msg += get_reporting_string( @@ -247,7 +243,7 @@ def test_params_checks_wrong_type_int(): def test_check_unknown_param(): - """Checks handling of malformed json rule.""" + """Test handling of malformed json rule.""" # pylint:disable=protected-access,no-member model = SafeDummyClassifier() _, _ = model.preliminary_check() @@ -260,19 +256,20 @@ def test_check_unknown_param(): cur_val="boo", ) assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive def test_check_model_param_or(): - """Tests or conditions in rules.json - the and condition is tested by the decision tree tests. + """Test or conditions in rules.json. + + The and condition is tested by the decision tree tests. """ # ok model = SafeDummyClassifier() msg, disclosive = model.preliminary_check() correct_msg = ok_start assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive part1 = get_reporting_string( name="different_than_fixed_value", @@ -292,25 +289,25 @@ def test_check_model_param_or(): correct_msg = ok_start + part1 msg, disclosive = model.preliminary_check() assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive # or branch 2 model = SafeDummyClassifier(keyB=False) correct_msg = ok_start + part2 msg, disclosive = model.preliminary_check() assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive # fail or model = SafeDummyClassifier(keyA=False, keyB=False) correct_msg = notok_start + part1 + part2 msg, disclosive = model.preliminary_check() assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is True + assert disclosive def test_saves(): - """Checks that save functions as expected.""" + """Test that save functions as expected.""" model = SafeDummyClassifier() x, y = get_data() model.fit(x, y) @@ -332,14 +329,9 @@ def test_saves(): model.square = lambda x: x * x # pylint: disable=attribute-defined-outside-init model.save("unpicklable.sav") - # cleanup - for name in ("dummy.pkl", "dummy.sav", "unpicklable.pkl", "unpicklable.sav"): - if os.path.exists(name) and os.path.isfile(name): - os.remove(name) - def test_loads(): - """Basic check that making, changing,saving,loading model works.""" + """Test that making, changing, saving, loading model works.""" model = SafeDummyClassifier() x, y = get_data() model.fit(x, y) @@ -359,14 +351,9 @@ def test_loads(): model2 = joblib.load(file) assert model2.exactly_boo == "this_should_be_present" - # cleanup - for name in ("dummy.pkl", "dummy.sav"): - if os.path.exists(name) and os.path.isfile(name): - os.remove(name) - -def test__apply_constraints(): - """Tests constraints can be applied as expected.""" +def test_apply_constraints(): + """Test constraints can be applied as expected.""" # wrong type model = SafeDummyClassifier() @@ -380,22 +367,22 @@ def test__apply_constraints(): msg, _ = model.preliminary_check(verbose=True, apply_constraints=True) - assert model.at_least_5f == 5.0 + assert model.at_least_5f == 5 assert model.at_most_5i == 5 assert model.exactly_boo == "boo" # checks that type changes happen correctly model = SafeDummyClassifier() - model.at_least_5f = int(6.0) - model.at_most_5i = float(4.2) + model.at_least_5f = 6 + model.at_most_5i = 4.2 model.exactly_boo = "five" - assert model.at_least_5f == int(6) + assert model.at_least_5f == 6 assert model.at_most_5i == 4.2 assert model.exactly_boo == "five" msg, _ = model.preliminary_check(verbose=True, apply_constraints=True) - assert model.at_least_5f == 6.0 + assert model.at_least_5f == 6 assert model.at_most_5i == 4 assert model.exactly_boo == "boo" @@ -403,7 +390,7 @@ def test__apply_constraints(): correct_msg += get_reporting_string( name="different_than_recommended_type", key="at_least_5f", - cur_val=int(6), + cur_val=6, val="float", ) correct_msg += get_reporting_string( @@ -457,9 +444,7 @@ def test__apply_constraints(): def test_get_saved_model_exception(): - """Tests the exception handling - in get_current_and_saved_models(). - """ + """Test the exception handling in get_current_and_saved_models().""" model = SafeDummyClassifier() # add generator which can't be pickled or copied @@ -474,9 +459,9 @@ def test_get_saved_model_exception(): def test_generic_additional_tests(): - """Checks the class generic additional tests - for this purpose SafeDummyClassifier() - defines + """Test the class generic additional tests. + + For this purpose SafeDummyClassifier() defines self.newthing = {"myStringKey": "aString", "myIntKey": 42}. """ model = SafeDummyClassifier() @@ -487,7 +472,7 @@ def test_generic_additional_tests(): msg, disclosive = model.posthoc_check() correct_msg = "" assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is False + assert not disclosive # different lengths model.saved_model["newthing"] += ("extraA",) @@ -500,7 +485,7 @@ def test_generic_additional_tests(): correct_msg = "Warning: different counts of values for parameter newthing.\n" assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is True + assert disclosive # different thing in list model.newthing += ("extraB",) @@ -514,11 +499,11 @@ def test_generic_additional_tests(): f'{model.saved_model["newthing"]}' ) assert msg == correct_msg, f"Correct msg:\n{correct_msg}\nActual msg:\n{msg}\n" - assert disclosive is True + assert disclosive def test_request_release_without_attacks(): - """Checks requestrelease code works and check the content of the json file.""" + """Test request release works and check the content of the json file.""" model = SafeDummyClassifier() x, y = get_data() model.fit(x, y) @@ -556,5 +541,3 @@ def test_request_release_without_attacks(): "reason": reason, "timestamp": model.timestamp, } in json_data["safemodel"] - - clean(RES_DIR) diff --git a/tests/safemodel/test_saferandomforestclassifier.py b/tests/safemodel/test_saferandomforestclassifier.py index acc8b6ec..e989d960 100644 --- a/tests/safemodel/test_saferandomforestclassifier.py +++ b/tests/safemodel/test_saferandomforestclassifier.py @@ -1,9 +1,8 @@ -"""This module contains unit tests for the SafeRandomForestClassifier.""" +"""Tests for the SafeRandomForestClassifier.""" from __future__ import annotations import copy -import os import pickle import joblib @@ -28,11 +27,9 @@ def fit(self, x: np.ndarray, y: np.ndarray): def predict(self, x: np.ndarray): """Predict all ones.""" - # return np.ones(x.shape[0]) - def get_data(): - """Returns data for testing.""" + """Return data for testing.""" iris = datasets.load_iris() x = np.asarray(iris["data"], dtype=np.float64) y = np.asarray(iris["target"], dtype=np.float64) @@ -42,7 +39,7 @@ def get_data(): def test_randomforest_unchanged(): - """SafeRandomForestClassifier using recommended values.""" + """Test using recommended values.""" x, y = get_data() model = SafeRandomForestClassifier( random_state=1, n_estimators=5, min_samples_leaf=5 @@ -52,11 +49,11 @@ def test_randomforest_unchanged(): msg, disclosive = model.preliminary_check() correct_msg = get_reporting_string(name="within_recommended_ranges") assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is False + assert not disclosive def test_randomforest_recommended(): - """SafeRandomForestClassifier using recommended values.""" + """Test using recommended values.""" x, y = get_data() model = SafeRandomForestClassifier(random_state=1, n_estimators=5) model.min_samples_leaf = 6 @@ -64,11 +61,11 @@ def test_randomforest_recommended(): msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is False + assert not disclosive def test_randomforest_unsafe_1(): - """SafeRandomForestClassifier with unsafe changes.""" + """Test with unsafe changes.""" x, y = get_data() model = SafeRandomForestClassifier( random_state=1, n_estimators=5, min_samples_leaf=5 @@ -82,11 +79,11 @@ def test_randomforest_unsafe_1(): "fixed value of True." ) assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is True + assert disclosive def test_randomforest_unsafe_2(): - """SafeRandomForestClassifier with unsafe changes.""" + """Test with unsafe changes.""" model = SafeRandomForestClassifier(random_state=1, n_estimators=5) model.bootstrap = True model.min_samples_leaf = 2 @@ -97,11 +94,11 @@ def test_randomforest_unsafe_2(): "min value of 5." ) assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is True + assert disclosive def test_randomforest_unsafe_3(): - """SafeRandomForestClassifier with unsafe changes.""" + """Test with unsafe changes.""" model = SafeRandomForestClassifier(random_state=1, n_estimators=5) model.bootstrap = False model.min_samples_leaf = 2 @@ -114,11 +111,11 @@ def test_randomforest_unsafe_3(): "min value of 5." ) assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is True + assert disclosive def test_randomforest_save(): - """SafeRandomForestClassifier model saving.""" + """Test model saving.""" x, y = get_data() model = SafeRandomForestClassifier( random_state=1, n_estimators=5, min_samples_leaf=5 @@ -135,14 +132,9 @@ def test_randomforest_save(): sav_model = joblib.load(file) assert sav_model.score(x, y) == EXPECTED_ACC - # cleanup - for name in ("rf_test.pkl", "rf_test.sav"): - if os.path.exists(name) and os.path.isfile(name): - os.remove(name) - def test_randomforest_hacked_postfit(): - """SafeRandomForestClassifier changes made to parameters after fit() called.""" + """Test changes made to parameters after fit() called.""" x, y = get_data() model = SafeRandomForestClassifier( random_state=1, n_estimators=5, min_samples_leaf=5 @@ -154,30 +146,22 @@ def test_randomforest_hacked_postfit(): msg, disclosive = model.preliminary_check() correct_msg = get_reporting_string(name="within_recommended_ranges") assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is False + assert not disclosive # but more detailed analysis says not msg2, disclosive2 = model.posthoc_check() part1 = get_reporting_string(name="basic_params_differ", length=1) part2 = get_reporting_string( name="param_changed_from_to", key="bootstrap", val=False, cur_val=True ) - part3 = "" # get_reporting_string( - # name="param_changed_from_to", - # key="estimator", - # val="DecisionTreeClassifier()", - # cur_val="DecisionTreeClassifier()", - # ) + part3 = "" correct_msg2 = part1 + part2 + part3 - # print(f'Correct: {correct_msg2}\n Actual: {msg2}') assert msg2 == correct_msg2, f"{msg2}\n should be {correct_msg2}" - assert disclosive2 is True + assert disclosive2 def test_not_fitted(): - """Posthoc_check() called on unfitred model - could have anything injected in classifier parameters. - """ + """Test Posthoc_check() called on unfitted model.""" unfitted_model = SafeRandomForestClassifier(random_state=1, n_estimators=5) # not fitted @@ -190,9 +174,7 @@ def test_not_fitted(): def test_randomforest_modeltype_changed(): - """Model type has been changed after fit() - in this this case to hide some data. - """ + """Test model type has been changed after fit().""" x, y = get_data() model = SafeRandomForestClassifier(random_state=1, n_estimators=5) correct_msg = "" @@ -205,7 +187,6 @@ def test_randomforest_modeltype_changed(): model.estimators_[i] = x[i, :] msg, disclosive = model.posthoc_check() - # correct_msg += get_reporting_string(name="basic_params_differ",length=1) correct_msg = get_reporting_string(name="forest_estimators_differ", idx=5) correct_msg += get_reporting_string( name="param_changed_from_to", @@ -213,17 +194,14 @@ def test_randomforest_modeltype_changed(): val="DecisionTreeClassifier()", cur_val="DummyClassifier()", ) - # correct_msg += ("structure estimator has 1 differences: [('change', '', " - # "(DecisionTreeClassifier(), DecisionTreeClassifier()))]" - # ) print(f"Correct: {correct_msg} Actual: {msg}") assert msg == correct_msg, f"{msg}\n should be {correct_msg}" - assert disclosive is True, "should have been flagged as disclosive" + assert disclosive, "should have been flagged as disclosive" def test_randomforest_hacked_postfit_trees_removed(): - """Tests various combinations of removing trees.""" + """Test various combinations of removing trees.""" x, y = get_data() model = SafeRandomForestClassifier(random_state=1, n_estimators=5) # code that checks estimators_ : one other or both missing or different number or size @@ -233,7 +211,6 @@ def test_randomforest_hacked_postfit_trees_removed(): the_estimators = model.__dict__.pop("estimators_") msg, disclosive = model.posthoc_check() correct_msg = get_reporting_string(name="current_item_removed", item="estimators_") - # print(f'Correct: {correct_msg} Actual: {msg}') assert disclosive, "should be flagged as disclosive" assert msg == correct_msg, f"{msg}\n should be {correct_msg}" @@ -241,7 +218,6 @@ def test_randomforest_hacked_postfit_trees_removed(): _ = model.saved_model.pop("estimators_") msg, disclosive = model.posthoc_check() correct_msg = get_reporting_string(name="both_item_removed", item="estimators_") - # print(f'Correct: {correct_msg} Actual: {msg}') assert disclosive, "should be flagged as disclosive" assert msg == correct_msg, f"{msg}\n should be {correct_msg}" @@ -249,13 +225,12 @@ def test_randomforest_hacked_postfit_trees_removed(): model.estimators_ = the_estimators msg, disclosive = model.posthoc_check() correct_msg = get_reporting_string(name="saved_item_removed", item="estimators_") - # print(f'Correct: {correct_msg} Actual: {msg}') assert disclosive, "should be flagged as disclosive" assert msg == correct_msg, f"{msg}\n should be {correct_msg}" def test_randomforest_hacked_postfit_trees_swapped(): - """Trees swapped with those from a different random forest.""" + """Test trees swapped with those from a different random forest.""" x, y = get_data() model = SafeRandomForestClassifier(random_state=1, n_estimators=5) diffsizemodel = SafeRandomForestClassifier( @@ -278,20 +253,14 @@ def test_randomforest_hacked_postfit_trees_swapped(): name="param_changed_from_to", key="max_depth", val="None", cur_val="2" ) part3 = get_reporting_string(name="forest_estimators_differ", idx=5) - part4 = "" # get_reporting_string( - # name="param_changed_from_to", - # key="estimator", - # val="DecisionTreeClassifier()", - # cur_val="DecisionTreeClassifier()", - # ) + part4 = "" correct_msg = part1 + part2 + part3 + part4 - # print(f'Correct:\n{correct_msg} Actual:\n{msg}') assert msg == correct_msg, f"{msg}\n should be {correct_msg}" assert disclosive, "should be flagged as disclosive" def test_randomforest_hacked_postfit_moretrees(): - """Trees added after fit.""" + """Test trees added after fit.""" x, y = get_data() model = SafeRandomForestClassifier(random_state=1, n_estimators=5) diffsizemodel = SafeRandomForestClassifier(random_state=1, n_estimators=10) @@ -309,13 +278,7 @@ def test_randomforest_hacked_postfit_moretrees(): name="param_changed_from_to", key="n_estimators", val="5", cur_val="10" ) part3 = get_reporting_string(name="different_num_estimators", num1=10, num2=5) - part4 = "" # get_reporting_string( - # name="param_changed_from_to", - # key="estimator", - # val="DecisionTreeClassifier()", - # cur_val="DecisionTreeClassifier()", - # ) + part4 = "" correct_msg = part1 + part2 + part3 + part4 - # print(f'Correct:\n{correct_msg} Actual:\n{msg}') assert msg == correct_msg, f"{msg}\n should be {correct_msg}" assert disclosive, "should be flagged as disclosive" diff --git a/tests/safemodel/test_safesvc.py b/tests/safemodel/test_safesvc.py index b4704bfd..864b68e7 100644 --- a/tests/safemodel/test_safesvc.py +++ b/tests/safemodel/test_safesvc.py @@ -5,6 +5,7 @@ import unittest import numpy as np +import pytest from sklearn import datasets from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC @@ -13,7 +14,7 @@ def get_data(): - """Returns data for testing.""" + """Return data for testing.""" cancer = datasets.load_breast_cancer() x = np.asarray(cancer["data"], dtype=np.float64) y = np.asarray(cancer["target"], dtype=np.float64) @@ -38,26 +39,26 @@ def test_run(self): sv_predictions = svc.predict(test_features) # Check that the two models have equal shape - self.assertTupleEqual(dp_predictions.shape, sv_predictions.shape) + assert dp_predictions.shape == sv_predictions.shape dp_predprob = dpsvc.predict_proba(test_features) sv_predprob = svc.predict_proba(test_features) # Check that the two models have equal shape - self.assertTupleEqual(dp_predprob.shape, sv_predprob.shape) + assert dp_predprob.shape == sv_predprob.shape def test_svc_recommended(self): - """SafeSupportVectorClassifier using recommended values.""" + """Test using recommended values.""" x, y = get_data() model = SafeSVC(gamma=1.0) model.fit(x, y) msg, disclosive = model.preliminary_check() correct_msg = "Model parameters are within recommended ranges.\n" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_svc_khat(self): - """SafeSupportVectorClassifier khat method.""" + """Test khat method.""" x, y = get_data() model = SafeSVC(gamma=1.0) model.fit(x, y) @@ -66,28 +67,25 @@ def test_svc_khat(self): _ = model.k_hat_svm(x, y_matrix) def test_svc_wrongdata(self): - """SafeSupportVectorClassifier with wrong datatypes.""" + """Test with wrong datatypes.""" x, y = get_data() model = SafeSVC(gamma=1.0) # wrong y datatype - with self.assertRaises(Exception) as context: + with pytest.raises(Exception, match="DPSCV needs np.ndarray inputs"): model.fit(x, 1) - self.assertTrue("DPSCV needs np.ndarray inputs" in str(context.exception)) # wrong x datatype - with self.assertRaises(Exception) as context: + with pytest.raises(Exception, match="DPSCV needs np.ndarray inputs"): model.fit(1, 1) - self.assertTrue("DPSCV needs np.ndarray inputs" in str(context.exception)) # wrong label values yplus = y + 5 errstr = "DP SVC can only handle binary classification" - with self.assertRaises(Exception) as context: + with pytest.raises(Exception, match=errstr): model.fit(x, yplus) - self.assertTrue(errstr in str(context.exception)) def test_svc_gamma_zero(self): - """SafeSupportVectorClassifier still makes predictions if we provide daft params.""" + """Test predictions if we provide daft params.""" x, y = get_data() model = SafeSVC(gamma=0.0, eps=0.0) model.fit(x, y) @@ -95,14 +93,14 @@ def test_svc_gamma_zero(self): assert len(predictions) == len(x) def test_svc_gamma_auto(self): - """SafeSupportVectorClassifier still makes predictions if we provide daft params.""" + """Test predictions if we provide daft params.""" x, y = get_data() model = SafeSVC(gamma="auto") model.fit(x, y) assert model.gamma == 1.0 / x.shape[1] def test_svc_setparams(self): - """SafeSupportVectorClassifier using unchanged values.""" + """Test using unchanged values.""" x, y = get_data() model = SafeSVC(gamma=1.0) model.fit(x, y) @@ -121,7 +119,7 @@ def test_svc_setparams(self): # should log to file mewssage "Unsupported parameter: foo" def test_svc_unchanged(self): - """SafeSupportVectorClassifier using unchanged values.""" + """Test using unchanged values.""" x, y = get_data() model = SafeSVC(gamma=1.0) model.fit(x, y) @@ -129,10 +127,10 @@ def test_svc_unchanged(self): msg, disclosive = model.posthoc_check() correct_msg = "" assert msg == correct_msg - assert disclosive is False + assert not disclosive def test_svc_nonstd_params_changed_postfit(self): - """SafeSupportVectorClassifier with params changed after fit.""" + """Test with params changed after fit.""" x, y = get_data() model = SafeSVC(gamma=1.0) model.fit(x, y) @@ -147,4 +145,4 @@ def test_svc_nonstd_params_changed_postfit(self): ) assert msg == correct_msg - assert disclosive is True + assert disclosive diff --git a/tests/safemodel/test_safetf.py b/tests/safemodel/test_safetf.py index a1d851a9..d140659f 100644 --- a/tests/safemodel/test_safetf.py +++ b/tests/safemodel/test_safetf.py @@ -1,6 +1,4 @@ -"""Jim Smith 2022 - Not currently implemented. -""" +"""Tests for safetf.""" from __future__ import annotations @@ -10,7 +8,7 @@ def test_Safe_tf_DPModel_l2_and_noise(): - """Tests user is informed this is not implemented yet.""" + """Test user is informed this is not implemented yet.""" with pytest.raises(NotImplementedError): # with values for the l2 and noise params safetf.Safe_tf_DPModel(1.5, 2.0, True)