Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feature][refactor] Benchmark Reporting + Hub Mixin #122

Merged
merged 14 commits into from
Feb 13, 2024
Merged
4 changes: 2 additions & 2 deletions .github/workflows/check_quality.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install quality requirements
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test_api_cpu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install dependencies
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test_api_misc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CPU Intel Neural Compressor Tests
name: CLI CPU Intel Neural Compressor Tests

on:
workflow_dispatch:
Expand All @@ -12,16 +12,16 @@ concurrency:
cancel-in-progress: true

jobs:
run_cpu_neural_compressor_tests:
run_cli_cpu_neural_compressor_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install Intel Neural Compressor CPU requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CPU OnnxRuntime Tests
name: CLI CPU OnnxRuntime Tests

on:
workflow_dispatch:
Expand All @@ -12,16 +12,16 @@ concurrency:
cancel-in-progress: true

jobs:
run_cpu_onnxruntime_tests:
run_cli_cpu_onnxruntime_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CPU OpenVINO Tests
name: CLI CPU OpenVINO Tests

on:
workflow_dispatch:
Expand All @@ -12,16 +12,16 @@ concurrency:
cancel-in-progress: true

jobs:
run_cpu_openvino_tests:
run_cli_cpu_openvino_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CPU Pytorch tests
name: CLI CPU Pytorch tests

on:
workflow_dispatch:
Expand All @@ -12,16 +12,16 @@ concurrency:
cancel-in-progress: true

jobs:
run_cpu_pytorch_tests:
run_cli_cpu_pytorch_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CUDA OnnxRuntime Inference Tests
name: CLI CUDA OnnxRuntime Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_cuda_onnxruntime_inference_tests:
build_image_and_run_cli_cuda_onnxruntime_tests:
runs-on: hf-dgx-01
steps:
- name: Checkout
Expand Down Expand Up @@ -40,4 +40,4 @@ jobs:
--workdir /workspace/optimum-benchmark
--gpus '"device=0,1"'
opt-bench-cuda:11.8.0
-c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime and inference' -x"
-c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime' -x"
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CUDA Pytorch Tests
name: CLI CUDA Pytorch Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_cuda_pytorch_tests:
build_image_and_run_cli_cuda_pytorch_tests:
strategy:
fail-fast: false
matrix:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: CUDA Torch-ORT Training Tests
name: CLI CUDA Torch-ORT Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_cuda_torch_ort_training_tests:
build_image_and_run_cli_cuda_torch_ort_tests:
runs-on: hf-dgx-01
steps:
- name: Checkout
Expand Down Expand Up @@ -40,4 +40,4 @@ jobs:
--workdir /workspace/optimum-benchmark
--gpus '"device=0,1"'
opt-bench-cuda:11.8.0
-c "pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure && pytest -k 'cli and cuda and torch_ort and training' -x"
-c "pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure && pytest -k 'cli and cuda and torch_ort' -x"
4 changes: 2 additions & 2 deletions .github/workflows/test_cli_misc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
- name: Checkout
uses: actions/checkout@v3

- name: Set up Python 3.8
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.8
python-version: '3.10'

- name: Install requirements
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: ROCm OnnxRuntime Inference Tests
name: CLI ROCm OnnxRuntime Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_rocm_onnxruntime_inference_tests:
build_image_and_run_cli_rocm_onnxruntime_tests:
runs-on: hf-amd-mi210-dev
steps:
- name: Checkout
Expand Down Expand Up @@ -51,4 +51,4 @@ jobs:
--device /dev/dri/renderD129
--entrypoint /bin/bash
opt-bench-rocm-ort:5.7
-c "pip install -e .[testing,timm,diffusers] && pytest -k 'cli and rocm and onnxruntime and inference' -x"
-c "pip install -e .[testing,timm,diffusers] && pytest -k 'cli and rocm and onnxruntime' -x"
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: ROCm Pytorch Tests
name: CLI ROCm Pytorch Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_rocm_pytorch_tests:
build_image_and_run_cli_rocm_pytorch_tests:
strategy:
fail-fast: false
matrix:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: TensorRT-LLM Tests
name: CLI TensorRT-LLM Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
pull_image_and_run_tensorrt_llm_tests:
pull_image_and_run_cli_tensorrt_llm_tests:
runs-on: hf-dgx-01
steps:
- name: Checkout
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: TensorRT OnnxRuntime Inference Tests
name: CLI TensorRT OnnxRuntime Tests

on:
workflow_dispatch:
Expand All @@ -12,7 +12,7 @@ concurrency:
cancel-in-progress: true

jobs:
build_image_and_run_tensorrt_onnxruntime_tests:
build_image_and_run_cli_tensorrt_onnxruntime_tests:
runs-on: hf-dgx-01
steps:
- name: Checkout
Expand Down Expand Up @@ -40,4 +40,4 @@ jobs:
--gpus '"device=0,1"'
--entrypoint /bin/bash
opt-bench-tensorrt:22.12
-c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and tensorrt and onnxruntime and inference' -x"
-c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and tensorrt and onnxruntime' -x"
68 changes: 64 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,68 @@ style:
install:
pip install -e .

install_cpu_dev:
pip install -e .[quality,testing,openvino,onnxruntime,neural-compressor,diffusers,timm,peft]
build_docker_cpu:
docker build -f docker/cuda.dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t opt-bench-cpu:latest .

install_gpu_dev:
pip install -e .[quality,testing,onnxruntime-gpu,deepspeed,diffusers,timm,peft]
build_docker_cuda:
docker build -f docker/cuda.dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --build-arg TORCH_CUDA=cu118 --build-arg CUDA_VERSION=11.8.0 -t opt-bench-cuda:11.8.0 .

build_docker_rocm:
docker build -f docker/rocm.dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --build-arg TORCH_ROCM=rocm5.6 --build-arg ROCM_VERSION=5.6.1 -t opt-bench-rocm:5.6.1 .

test_cli_cpu_neural_compressor:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,neural-compressor] && pytest tests/ -k 'cli and cpu and neural_compressor' -x"

test_cli_cpu_openvino:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,openvino,diffusers] && pytest tests/ -k 'cli and cpu and openvino' -x"

test_cli_cpu_onnxruntime:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,onnxruntime,diffusers,timm] && pytest tests/ -k 'cli and cpu and onnxruntime' -x"

test_cli_cpu_pytorch:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,diffusers,timm] && pytest tests/ -k 'cli and cpu and pytorch' -x"

test_api_cpu:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and cpu' -x"

test_api_cuda:
docker run \
--rm \
--gpus '"device=0,1"' \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cuda:11.8.0 -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and cuda' -x"

test_api_misc:
docker run \
--rm \
--entrypoint /bin/bash \
--volume $(PWD):/workspace \
--workdir /workspace \
opt-bench-cpu:latest -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and not (cpu or cuda or rocm or tensorrt)' -x"
Loading
Loading