From e5fcd19a20377c2996b2820efeb294bfb1b2d28d Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Wed, 20 Nov 2024 21:28:55 -0800 Subject: [PATCH 1/3] update transformers test requirements --- tools/ci_build/build.py | 16 ++++++++++++++++ .../transformers-test/requirements.txt | 7 ++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index aa1198102f978..1098d44fbe3cd 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -2074,6 +2074,21 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): if is_windows(): cwd = os.path.join(cwd, config) + # Install PyTorch which is required for transformers tests, and optional for some python tests. + if args.enable_transformers_tool_test and not args.disable_contrib_ops and not args.use_rocm: + index_url = "https://download.pytorch.org/whl/cpu" + if args.use_cuda and is_linux(): + index_url = "https://download.pytorch.org/whl/cu124" + if args.cuda_version and version_to_tuple(args.cuda_version) < (12, 0): + index_url = "https://download.pytorch.org/whl/cu118" + + run_subprocess( + [sys.executable, "-m", "pip", "install", "torch", "--index-url", index_url], + cwd=cwd, + dll_path=dll_path, + python_path=python_path, + ) + run_subprocess( [sys.executable, "onnxruntime_test_python.py"], cwd=cwd, dll_path=dll_path, python_path=python_path ) @@ -2128,6 +2143,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): dll_path=dll_path, python_path=python_path, ) + if not args.disable_contrib_ops: run_subprocess( [sys.executable, "-m", "unittest", "discover", "-s", "quantization"], cwd=cwd, dll_path=dll_path diff --git a/tools/ci_build/requirements/transformers-test/requirements.txt b/tools/ci_build/requirements/transformers-test/requirements.txt index 32c5ce7dd08d1..3519461ae7c00 100644 --- a/tools/ci_build/requirements/transformers-test/requirements.txt +++ b/tools/ci_build/requirements/transformers-test/requirements.txt @@ -1,8 +1,9 @@ # packages used by transformers python unittest (only enabled in Linux CPU CI Pipeline) packaging -protobuf==3.20.2 -numpy==1.24.0 ; python_version < '3.12' -numpy==1.26.0 ; python_version >= '3.12' +# protobuf and numpy is same as tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt +protobuf==4.21.12 +numpy==1.21.6 ; python_version < '3.9' +numpy==2.0.0 ; python_version >= '3.9' torch coloredlogs==15.0 transformers==4.38.0 From f82673cadf4df4d219d57413e8993665d00f8549 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Wed, 20 Nov 2024 23:16:02 -0800 Subject: [PATCH 2/3] refine torch install --- tools/ci_build/build.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 1098d44fbe3cd..ba1c74c659b71 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -2074,16 +2074,12 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): if is_windows(): cwd = os.path.join(cwd, config) - # Install PyTorch which is required for transformers tests, and optional for some python tests. if args.enable_transformers_tool_test and not args.disable_contrib_ops and not args.use_rocm: - index_url = "https://download.pytorch.org/whl/cpu" - if args.use_cuda and is_linux(): - index_url = "https://download.pytorch.org/whl/cu124" - if args.cuda_version and version_to_tuple(args.cuda_version) < (12, 0): - index_url = "https://download.pytorch.org/whl/cu118" - + # PyTorch is required for transformers tests, and optional for some python tests. + # Install cpu only version of torch when cuda is not enabled in Linux. + extra = [] if args.use_cuda and is_linux() else ["--index-url", "https://download.pytorch.org/whl/cpu"] run_subprocess( - [sys.executable, "-m", "pip", "install", "torch", "--index-url", index_url], + [sys.executable, "-m", "pip", "install", "torch", *extra], cwd=cwd, dll_path=dll_path, python_path=python_path, From ebb6015c0fa081d842e084e845d923a0ba221a61 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Wed, 20 Nov 2024 23:20:31 -0800 Subject: [PATCH 3/3] update comment --- tools/ci_build/requirements/transformers-test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/requirements/transformers-test/requirements.txt b/tools/ci_build/requirements/transformers-test/requirements.txt index 3519461ae7c00..01816e8c57412 100644 --- a/tools/ci_build/requirements/transformers-test/requirements.txt +++ b/tools/ci_build/requirements/transformers-test/requirements.txt @@ -1,4 +1,4 @@ -# packages used by transformers python unittest (only enabled in Linux CPU CI Pipeline) +# packages used by transformers python unittest packaging # protobuf and numpy is same as tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt protobuf==4.21.12