From 1a81c67a96783d149db9c3970f679d78eab87fcb Mon Sep 17 00:00:00 2001 From: Roman Bredehoft Date: Thu, 14 Sep 2023 14:55:17 +0200 Subject: [PATCH] chore: only run tests that changed in regular CI if nothing else did --- .github/workflows/continuous-integration.yaml | 99 +++++++++++++------ .../FashionMNIST_quant_state_dict.pt | 0 .../custom_data_fp32_state_dict.pt | 0 .../custom_data_quant_state_dict.pt | 0 tests/data/{ => torch}/mnist_2b_s1_1.zip | 0 tests/data/{ => torch}/mnist_test_batch.zip | 0 .../test_p_error_binary_search.py | 5 +- tests/torch/test_compile_torch.py | 4 +- 8 files changed, 74 insertions(+), 34 deletions(-) rename tests/{ => data}/parameter_search/FashionMNIST_quant_state_dict.pt (100%) rename tests/{ => data}/parameter_search/custom_data_fp32_state_dict.pt (100%) rename tests/{ => data}/parameter_search/custom_data_quant_state_dict.pt (100%) rename tests/data/{ => torch}/mnist_2b_s1_1.zip (100%) rename tests/data/{ => torch}/mnist_test_batch.zip (100%) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index dc14cb349..4dd47a398 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -496,7 +496,9 @@ jobs: - src/** - '!src/concrete/ml/version.py' tests: - - tests/** + - 'tests/**/test_*.py' + tests_utils: + - tests/data/** - src/concrete/ml/pytest/** determinism: - tests/seeding/test_seeding.py @@ -514,20 +516,24 @@ jobs: makefile: - Makefile - # Run determinism test. The only case where this is not run is for a PR that does not modify - # anything in the source code or the determinism test file. This step is also triggered - # if any dependency has been updated or if some changes were found in the conftest.py as well - # as the Makefile + # Run determinism test if: + # - during weekly or release CI, as well as when the CI has been triggered manually (through + # GitHub's Action interface) + # - the determinism test file has been changed + # - the source code has been changed + # - any dependency has been updated + # - conftest.py has been changed + # - Makefile has been changed - name: Determinism id: determinism if: | - !( - steps.changed-files-in-pr.outcome == 'success' - && steps.changed-files-in-pr.outputs.determinism_any_changed == 'false' - && steps.changed-files-in-pr.outputs.src_any_changed == 'false' - && steps.changed-files-in-pr.outputs.dependencies_any_changed == 'false' - && steps.changed-files-in-pr.outputs.conftest_any_changed == 'false' - && steps.changed-files-in-pr.outputs.makefile_any_changed == 'false' + ( + steps.changed-files-in-pr.outcome == 'skipped' + || steps.changed-files-in-pr.outputs.determinism_any_changed == 'true' + || steps.changed-files-in-pr.outputs.src_any_changed == 'true' + || steps.changed-files-in-pr.outputs.dependencies_any_changed == 'true' + || steps.changed-files-in-pr.outputs.conftest_any_changed == 'true' + || steps.changed-files-in-pr.outputs.makefile_any_changed == 'true' ) && steps.install-deps.outcome == 'success' && steps.make-pcc.outcome == 'success' @@ -535,16 +541,20 @@ jobs: run: | make determinism - # Build the documentation if anything changed in our documentation files, in the source code - # or in the makefile + # Build the documentation if : + # - during weekly or release CI, as well as when the CI has been triggered manually (through + # GitHub's Action interface) + # - any documentation files has been changed + # - the source code has been changed + # - Makefile has been changed - name: Build docs id: build-docs if: | - !( - steps.changed-files-in-pr.outcome == 'success' - && steps.changed-files-in-pr.outputs.docs_any_changed == 'false' - && steps.changed-files-in-pr.outputs.src_any_changed == 'false' - && steps.changed-files-in-pr.outputs.makefile_any_changed == 'false' + ( + steps.changed-files-in-pr.outcome == 'skipped' + || steps.changed-files-in-pr.outputs.docs_any_changed == 'true' + || steps.changed-files-in-pr.outputs.src_any_changed == 'true' + || steps.changed-files-in-pr.outputs.makefile_any_changed == 'true' ) && steps.install-deps.outcome == 'success' && steps.make-pcc.outcome == 'success' @@ -631,10 +641,16 @@ jobs: name: py3-wheel path: dist/*.whl - # Run Pytest on a subset of our tests if the source code or our tests has changed, and if the - # current workflow does no take place in a weekly or release CI. This step is also triggered - # if any dependency has been updated or if some changes were found in the conftest.py as well - # as the Makefile + # Run Pytest on a subset of our tests if : + # - the current workflow does no take place in a weekly or release CI + # - if the CI has been triggered manually (through GitHub's Action interface) + # - the source code has been changed + # - any tests utils (pytest, data) has been changed + # - any dependency has been updated + # - conftest.py has been changed + # - Makefile has been changed + # If only some test files were changed, this step is skipped and each associated tests will be + # run individually in a following step (pytest_modified_tests_only) # If regular tests failed, a following script checks for flaky tests. If all failed tests # are known flaky tests, they are re-run. Otherwise, the step exits with status 1. # The 'bash +e {0}' is added here in order to make sure that the step does not exit directly @@ -642,13 +658,18 @@ jobs: - name: PyTest Source Code (regular) id: pytest_regular if: | - steps.changed-files-in-pr.outcome == 'success' - && ( - steps.changed-files-in-pr.outputs.src_any_changed == 'true' - || steps.changed-files-in-pr.outputs.tests_any_changed == 'true' - || steps.changed-files-in-pr.outputs.dependencies_any_changed == 'true' - || steps.changed-files-in-pr.outputs.conftest_any_changed == 'true' - || steps.changed-files-in-pr.outputs.makefile_any_changed == 'true' + ( + ( + steps.changed-files-in-pr.outcome == 'success' + && ( + steps.changed-files-in-pr.outputs.src_any_changed == 'true' + || steps.changed-files-in-pr.outputs.tests_utils_any_changed == 'true' + || steps.changed-files-in-pr.outputs.dependencies_any_changed == 'true' + || steps.changed-files-in-pr.outputs.conftest_any_changed == 'true' + || steps.changed-files-in-pr.outputs.makefile_any_changed == 'true' + ) + ) + || fromJSON(env.IS_WORKFLOW_DISPATCH) ) && steps.conformance.outcome == 'success' && !cancelled() @@ -688,6 +709,24 @@ jobs: path: failed_tests_comment.txt recreate: true + # If regular pytest step has been skipped but some changes has been detected in test files, + # meaning there was no other changed impacting our testing suite, we only need to run these + # modified tests + # Note that if pytest utils or test data are changed, the regular pytest step should have been + # triggered instead + - name: PyTest on modified tests only + id: pytest_modified_tests_only + if: | + steps.changed-files-in-pr.outcome == 'success' + && steps.pytest_regular.outcome == 'skipped' + && steps.changed-files-in-pr.outputs.tests_any_changed == 'true' + && steps.conformance.outcome == 'success' + && !cancelled() + run: | + for file in ${{ steps.changed-files-in-pr.outputs.tests_all_changed_files }}; do + make pytest_one TEST="$file" + done + # Run Pytest on all of our tests on a weekly basis - name: PyTest Source Code (weekly) id: pytest_weekly diff --git a/tests/parameter_search/FashionMNIST_quant_state_dict.pt b/tests/data/parameter_search/FashionMNIST_quant_state_dict.pt similarity index 100% rename from tests/parameter_search/FashionMNIST_quant_state_dict.pt rename to tests/data/parameter_search/FashionMNIST_quant_state_dict.pt diff --git a/tests/parameter_search/custom_data_fp32_state_dict.pt b/tests/data/parameter_search/custom_data_fp32_state_dict.pt similarity index 100% rename from tests/parameter_search/custom_data_fp32_state_dict.pt rename to tests/data/parameter_search/custom_data_fp32_state_dict.pt diff --git a/tests/parameter_search/custom_data_quant_state_dict.pt b/tests/data/parameter_search/custom_data_quant_state_dict.pt similarity index 100% rename from tests/parameter_search/custom_data_quant_state_dict.pt rename to tests/data/parameter_search/custom_data_quant_state_dict.pt diff --git a/tests/data/mnist_2b_s1_1.zip b/tests/data/torch/mnist_2b_s1_1.zip similarity index 100% rename from tests/data/mnist_2b_s1_1.zip rename to tests/data/torch/mnist_2b_s1_1.zip diff --git a/tests/data/mnist_test_batch.zip b/tests/data/torch/mnist_test_batch.zip similarity index 100% rename from tests/data/mnist_test_batch.zip rename to tests/data/torch/mnist_test_batch.zip diff --git a/tests/parameter_search/test_p_error_binary_search.py b/tests/parameter_search/test_p_error_binary_search.py index d4cc20495..fd301b25e 100644 --- a/tests/parameter_search/test_p_error_binary_search.py +++ b/tests/parameter_search/test_p_error_binary_search.py @@ -41,18 +41,19 @@ }, } +TEST_DATA_DIR = Path(__file__).parents[1] / "data" / "parameter_search" MODELS_ARGS = { "CustomModel": { "qat": { "model_class": QuantCustomModel, - "path": Path(__file__).parent / "custom_data_quant_state_dict.pt", + "path": TEST_DATA_DIR / "custom_data_quant_state_dict.pt", "params": {"n_bits": 4, "input_shape": 6, "hidden_shape": 100, "output_shape": 3}, }, "fp32": { "model_class": TorchCustomModel, "path": torch.load( - (Path(__file__).parent / "custom_data_fp32_state_dict.pt"), map_location="cpu" + (TEST_DATA_DIR / "custom_data_fp32_state_dict.pt"), map_location="cpu" ), "params": {"input_shape": 6, "hidden_shape": 100, "output_shape": 3}, }, diff --git a/tests/torch/test_compile_torch.py b/tests/torch/test_compile_torch.py index be5f50af6..867592cd7 100644 --- a/tests/torch/test_compile_torch.py +++ b/tests/torch/test_compile_torch.py @@ -788,8 +788,8 @@ def test_pretrained_mnist_qat( if not is_weekly_option: pytest.skip("Tests too long") - onnx_file_path = "tests/data/mnist_2b_s1_1.zip" - mnist_test_path = "tests/data/mnist_test_batch.zip" + onnx_file_path = "tests/data/torch/mnist_2b_s1_1.zip" + mnist_test_path = "tests/data/torch/mnist_test_batch.zip" # Load ONNX model from zip file with zipfile.ZipFile(onnx_file_path, "r") as archive_model: