From a0d76da8f3c1f32d83aa0290eaed17c51140dee0 Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:05:06 +0000 Subject: [PATCH 1/7] fixing aqlm test --- .github/workflows/self-scheduled-caller.yml | 104 +++++++++--------- src/transformers/testing_utils.py | 7 ++ .../aqlm_integration/test_aqlm.py | 3 + 3 files changed, 62 insertions(+), 52 deletions(-) diff --git a/.github/workflows/self-scheduled-caller.yml b/.github/workflows/self-scheduled-caller.yml index 75ea3bb24bc7fa..15ea29d28ef674 100644 --- a/.github/workflows/self-scheduled-caller.yml +++ b/.github/workflows/self-scheduled-caller.yml @@ -7,64 +7,64 @@ on: - cron: "17 2 * * *" push: branches: - - run_scheduled_ci* + - skip_if_aqlm_inference_not_fixed jobs: - model-ci: - name: Model CI - uses: ./.github/workflows/self-scheduled.yml - with: - job: run_models_gpu - slack_report_channel: "#transformers-ci-daily-models" - runner: daily-ci - docker: huggingface/transformers-all-latest-gpu - ci_event: Daily CI - secrets: inherit + # model-ci: + # name: Model CI + # uses: ./.github/workflows/self-scheduled.yml + # with: + # job: run_models_gpu + # slack_report_channel: "#transformers-ci-daily-models" + # runner: daily-ci + # docker: huggingface/transformers-all-latest-gpu + # ci_event: Daily CI + # secrets: inherit - torch-pipeline: - name: Torch pipeline CI - uses: ./.github/workflows/self-scheduled.yml - with: - job: run_pipelines_torch_gpu - slack_report_channel: "#transformers-ci-daily-pipeline-torch" - runner: daily-ci - docker: huggingface/transformers-pytorch-gpu - ci_event: Daily CI - secrets: inherit + # torch-pipeline: + # name: Torch pipeline CI + # uses: ./.github/workflows/self-scheduled.yml + # with: + # job: run_pipelines_torch_gpu + # slack_report_channel: "#transformers-ci-daily-pipeline-torch" + # runner: daily-ci + # docker: huggingface/transformers-pytorch-gpu + # ci_event: Daily CI + # secrets: inherit - tf-pipeline: - name: TF pipeline CI - uses: ./.github/workflows/self-scheduled.yml - with: - job: run_pipelines_tf_gpu - slack_report_channel: "#transformers-ci-daily-pipeline-tf" - runner: daily-ci - docker: huggingface/transformers-tensorflow-gpu - ci_event: Daily CI - secrets: inherit + # tf-pipeline: + # name: TF pipeline CI + # uses: ./.github/workflows/self-scheduled.yml + # with: + # job: run_pipelines_tf_gpu + # slack_report_channel: "#transformers-ci-daily-pipeline-tf" + # runner: daily-ci + # docker: huggingface/transformers-tensorflow-gpu + # ci_event: Daily CI + # secrets: inherit - example-ci: - name: Example CI - uses: ./.github/workflows/self-scheduled.yml - with: - job: run_examples_gpu - slack_report_channel: "#transformers-ci-daily-examples" - runner: daily-ci - docker: huggingface/transformers-all-latest-gpu - ci_event: Daily CI - secrets: inherit + # example-ci: + # name: Example CI + # uses: ./.github/workflows/self-scheduled.yml + # with: + # job: run_examples_gpu + # slack_report_channel: "#transformers-ci-daily-examples" + # runner: daily-ci + # docker: huggingface/transformers-all-latest-gpu + # ci_event: Daily CI + # secrets: inherit - deepspeed-ci: - name: DeepSpeed CI - uses: ./.github/workflows/self-scheduled.yml - with: - job: run_torch_cuda_extensions_gpu - slack_report_channel: "#transformers-ci-daily-deepspeed" - runner: daily-ci - docker: huggingface/transformers-pytorch-deepspeed-latest-gpu - ci_event: Daily CI - working-directory-prefix: /workspace - secrets: inherit + # deepspeed-ci: + # name: DeepSpeed CI + # uses: ./.github/workflows/self-scheduled.yml + # with: + # job: run_torch_cuda_extensions_gpu + # slack_report_channel: "#transformers-ci-daily-deepspeed" + # runner: daily-ci + # docker: huggingface/transformers-pytorch-deepspeed-latest-gpu + # ci_event: Daily CI + # working-directory-prefix: /workspace + # secrets: inherit quantization-ci: name: Quantization CI diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 49c2aefa09260e..f3c5f9572d64f3 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -345,6 +345,13 @@ def tooslow(test_case): """ return unittest.skip(reason="test is too slow")(test_case) +def skip_if_aqlm_inference_not_fixed(test_case) : + """ + Decorator marking tests for inference using aqlm models. + + These tests will be skipped till the issue from aqlm side is resolved + """ + return unittest.skip(reason="inference doesn't work with quantized aqlm models with recent torch versions")(test_case) def skip_if_not_implemented(test_func): @functools.wraps(test_func) diff --git a/tests/quantization/aqlm_integration/test_aqlm.py b/tests/quantization/aqlm_integration/test_aqlm.py index b79eae54c0c31e..860f61f01da593 100644 --- a/tests/quantization/aqlm_integration/test_aqlm.py +++ b/tests/quantization/aqlm_integration/test_aqlm.py @@ -28,6 +28,7 @@ require_torch_multi_gpu, slow, torch_device, + skip_if_aqlm_inference_not_fixed ) from transformers.utils import is_accelerate_available, is_aqlm_available, is_torch_available @@ -142,6 +143,7 @@ def test_quantized_model_conversion(self): self.assertEqual(nb_linears - 1, nb_aqlm_linear) + @skip_if_aqlm_inference_not_fixed def test_quantized_model(self): """ Simple test that checks if the quantized model is working properly @@ -158,6 +160,7 @@ def test_raise_if_non_quantized(self): with self.assertRaises(ValueError): _ = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config) + @skip_if_aqlm_inference_not_fixed def test_save_pretrained(self): """ Simple test that checks if the quantized model is working properly after being saved and loaded From a546791fcd161154304a40ae58b0b6e066afd05c Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:11:09 +0000 Subject: [PATCH 2/7] changing branch name --- .github/workflows/self-scheduled-caller.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-scheduled-caller.yml b/.github/workflows/self-scheduled-caller.yml index 15ea29d28ef674..c82b6797c31b81 100644 --- a/.github/workflows/self-scheduled-caller.yml +++ b/.github/workflows/self-scheduled-caller.yml @@ -7,7 +7,7 @@ on: - cron: "17 2 * * *" push: branches: - - skip_if_aqlm_inference_not_fixed + - skipping_aqlm_non_working_tests jobs: # model-ci: From 63f5e8760df6b6b0d5a0c23257756ab22d2b1892 Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:18:32 +0000 Subject: [PATCH 3/7] Aqlm skipping inference tests --- tests/quantization/aqlm_integration/test_aqlm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/quantization/aqlm_integration/test_aqlm.py b/tests/quantization/aqlm_integration/test_aqlm.py index 860f61f01da593..3b57ba8daa812b 100644 --- a/tests/quantization/aqlm_integration/test_aqlm.py +++ b/tests/quantization/aqlm_integration/test_aqlm.py @@ -173,7 +173,8 @@ def test_save_pretrained(self): output = model.generate(**input_ids, max_new_tokens=self.max_new_tokens) self.assertEqual(self.tokenizer.decode(output[0], skip_special_tokens=True), self.EXPECTED_OUTPUT) - + + @skip_if_aqlm_inference_not_fixed @require_torch_multi_gpu def test_quantized_model_multi_gpu(self): """ From 8b3f7919571a6a6e6e958b419a7924651dc7d626 Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:33:23 +0000 Subject: [PATCH 4/7] uncommenting ci --- .github/workflows/self-scheduled-caller.yml | 104 ++++++++++---------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/self-scheduled-caller.yml b/.github/workflows/self-scheduled-caller.yml index c82b6797c31b81..75ea3bb24bc7fa 100644 --- a/.github/workflows/self-scheduled-caller.yml +++ b/.github/workflows/self-scheduled-caller.yml @@ -7,64 +7,64 @@ on: - cron: "17 2 * * *" push: branches: - - skipping_aqlm_non_working_tests + - run_scheduled_ci* jobs: - # model-ci: - # name: Model CI - # uses: ./.github/workflows/self-scheduled.yml - # with: - # job: run_models_gpu - # slack_report_channel: "#transformers-ci-daily-models" - # runner: daily-ci - # docker: huggingface/transformers-all-latest-gpu - # ci_event: Daily CI - # secrets: inherit + model-ci: + name: Model CI + uses: ./.github/workflows/self-scheduled.yml + with: + job: run_models_gpu + slack_report_channel: "#transformers-ci-daily-models" + runner: daily-ci + docker: huggingface/transformers-all-latest-gpu + ci_event: Daily CI + secrets: inherit - # torch-pipeline: - # name: Torch pipeline CI - # uses: ./.github/workflows/self-scheduled.yml - # with: - # job: run_pipelines_torch_gpu - # slack_report_channel: "#transformers-ci-daily-pipeline-torch" - # runner: daily-ci - # docker: huggingface/transformers-pytorch-gpu - # ci_event: Daily CI - # secrets: inherit + torch-pipeline: + name: Torch pipeline CI + uses: ./.github/workflows/self-scheduled.yml + with: + job: run_pipelines_torch_gpu + slack_report_channel: "#transformers-ci-daily-pipeline-torch" + runner: daily-ci + docker: huggingface/transformers-pytorch-gpu + ci_event: Daily CI + secrets: inherit - # tf-pipeline: - # name: TF pipeline CI - # uses: ./.github/workflows/self-scheduled.yml - # with: - # job: run_pipelines_tf_gpu - # slack_report_channel: "#transformers-ci-daily-pipeline-tf" - # runner: daily-ci - # docker: huggingface/transformers-tensorflow-gpu - # ci_event: Daily CI - # secrets: inherit + tf-pipeline: + name: TF pipeline CI + uses: ./.github/workflows/self-scheduled.yml + with: + job: run_pipelines_tf_gpu + slack_report_channel: "#transformers-ci-daily-pipeline-tf" + runner: daily-ci + docker: huggingface/transformers-tensorflow-gpu + ci_event: Daily CI + secrets: inherit - # example-ci: - # name: Example CI - # uses: ./.github/workflows/self-scheduled.yml - # with: - # job: run_examples_gpu - # slack_report_channel: "#transformers-ci-daily-examples" - # runner: daily-ci - # docker: huggingface/transformers-all-latest-gpu - # ci_event: Daily CI - # secrets: inherit + example-ci: + name: Example CI + uses: ./.github/workflows/self-scheduled.yml + with: + job: run_examples_gpu + slack_report_channel: "#transformers-ci-daily-examples" + runner: daily-ci + docker: huggingface/transformers-all-latest-gpu + ci_event: Daily CI + secrets: inherit - # deepspeed-ci: - # name: DeepSpeed CI - # uses: ./.github/workflows/self-scheduled.yml - # with: - # job: run_torch_cuda_extensions_gpu - # slack_report_channel: "#transformers-ci-daily-deepspeed" - # runner: daily-ci - # docker: huggingface/transformers-pytorch-deepspeed-latest-gpu - # ci_event: Daily CI - # working-directory-prefix: /workspace - # secrets: inherit + deepspeed-ci: + name: DeepSpeed CI + uses: ./.github/workflows/self-scheduled.yml + with: + job: run_torch_cuda_extensions_gpu + slack_report_channel: "#transformers-ci-daily-deepspeed" + runner: daily-ci + docker: huggingface/transformers-pytorch-deepspeed-latest-gpu + ci_event: Daily CI + working-directory-prefix: /workspace + secrets: inherit quantization-ci: name: Quantization CI From a9f1e14e6e2ce2d4ccc68deda96fc30e6435dc45 Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:38:16 +0000 Subject: [PATCH 5/7] make style --- tests/quantization/aqlm_integration/test_aqlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/quantization/aqlm_integration/test_aqlm.py b/tests/quantization/aqlm_integration/test_aqlm.py index 3b57ba8daa812b..b0e4e68e1cec28 100644 --- a/tests/quantization/aqlm_integration/test_aqlm.py +++ b/tests/quantization/aqlm_integration/test_aqlm.py @@ -26,9 +26,9 @@ require_aqlm, require_torch_gpu, require_torch_multi_gpu, + skip_if_aqlm_inference_not_fixed, slow, torch_device, - skip_if_aqlm_inference_not_fixed ) from transformers.utils import is_accelerate_available, is_aqlm_available, is_torch_available @@ -173,7 +173,7 @@ def test_save_pretrained(self): output = model.generate(**input_ids, max_new_tokens=self.max_new_tokens) self.assertEqual(self.tokenizer.decode(output[0], skip_special_tokens=True), self.EXPECTED_OUTPUT) - + @skip_if_aqlm_inference_not_fixed @require_torch_multi_gpu def test_quantized_model_multi_gpu(self): From 851a02435ffcf8752db71d91ad88e9c14e398f7f Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 21:44:31 +0000 Subject: [PATCH 6/7] reformat --- src/transformers/testing_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index f3c5f9572d64f3..ef992a2b966582 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -345,13 +345,17 @@ def tooslow(test_case): """ return unittest.skip(reason="test is too slow")(test_case) -def skip_if_aqlm_inference_not_fixed(test_case) : + +def skip_if_aqlm_inference_not_fixed(test_case): """ Decorator marking tests for inference using aqlm models. These tests will be skipped till the issue from aqlm side is resolved """ - return unittest.skip(reason="inference doesn't work with quantized aqlm models with recent torch versions")(test_case) + return unittest.skip(reason="inference doesn't work with quantized aqlm models with recent torch versions")( + test_case + ) + def skip_if_not_implemented(test_func): @functools.wraps(test_func) From 5458765b5c0d44e0d28e2eb9af09f1759df73038 Mon Sep 17 00:00:00 2001 From: MekkCyber Date: Thu, 21 Nov 2024 22:00:22 +0000 Subject: [PATCH 7/7] fomat style --- src/transformers/testing_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index ef992a2b966582..c63caae382a963 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -352,9 +352,9 @@ def skip_if_aqlm_inference_not_fixed(test_case): These tests will be skipped till the issue from aqlm side is resolved """ - return unittest.skip(reason="inference doesn't work with quantized aqlm models with recent torch versions")( - test_case - ) + return unittest.skip( + reason="inference doesn't work with quantized aqlm models using torch.Any type with recent torch versions" + )(test_case) def skip_if_not_implemented(test_func):