ARC LLM Test for Stable Version #3

Workflow file for this run

.github/workflows/llm_tests_for_stable_version_on_arc.yml at 156c232

	name: ARC LLM Test for Stable Version

	# Cancel previous runs in the PR when you push new commits
	concurrency:
	group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number \|\| github.run_id }}
	cancel-in-progress: true

	# Controls when the action will run.
	on:
	# pull_request:
	# branches: [main]
	# paths:
	# - ".github/workflows/llm_performance_tests.yml"
	# - "python/llm/test/benchmark/**"
	# - "python/llm/dev/benchmark/all-in-one/**"
	workflow_dispatch:
	workflow_call:

	# A workflow run is made up of one or more jobs that can run sequentially or in parallel
	jobs:
	llm-cpp-build:
	uses: ./.github/workflows/llm-binary-build.yml

	llm-perf-regression-test-on-arc:
	needs: llm-cpp-build
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.9"]
	runs-on: [self-hosted, llm, perf]
	env:
	OMP_NUM_THREADS: 16
	THREAD_NUM: 16
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	CSV_SAVE_PATH: '/mnt/disk1/stable_version_perf_gpu/'

	steps:
	- uses: actions/checkout@v3

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	shell: bash
	# pip install transformers_stream_generator for model internlm-chat-7b-8k
	# pip install tiktoken for model Qwen-7B-Chat-10-12
	run: \|
	python -m pip install --upgrade pip
	python -m pip install --upgrade wheel
	python -m pip install --upgrade omegaconf
	python -m pip install --upgrade pandas
	python -m pip install --upgrade einops
	python -m pip install --upgrade transformers_stream_generator
	python -m pip install --upgrade tiktoken

	- name: Download llm binary
	uses: ./.github/actions/llm/download-llm-binary

	- name: Run LLM install (all) test
	uses: ./.github/actions/llm/setup-llm-env
	with:
	extra-dependency: "xpu_2.1"

	- name: Test installed xpu version
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	bash python/llm/test/run-llm-install-tests.sh

	- name: Test on xpu (int4)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	mv python/llm/test/benchmark/stable-version-arc-perf-test-sym_int4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run.py
	# batch_size = 1
	sed -i '/batch_size/c\batch_size: 1' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/int4/batch_size_1
	rm ./*.csv
	# batch_size = 2
	sed -i '/batch_size/c\batch_size: 2' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/int4/batch_size_2
	rm ./*.csv
	# batch_size = 4
	sed -i '/batch_size/c\batch_size: 4' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/int4/batch_size_4
	rm ./*.csv
	# batch_size = 8
	sed -i '/batch_size/c\batch_size: 8' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/int4/batch_size_8
	rm ./*.csv
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f $CSV_SAVE_PATH/int4/batch_size_1 -b $CSV_SAVE_PATH/int4/batch_size_1/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/int4/batch_size_2 -b $CSV_SAVE_PATH/int4/batch_size_2/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/int4/batch_size_4 -b $CSV_SAVE_PATH/int4/batch_size_4/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/int4/batch_size_8 -b $CSV_SAVE_PATH/int4/batch_size_8/transformer_int4_gpu-results-1baseline.csv -t 5.0

	- name: Test on xpu (fp8)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	mv python/llm/test/benchmark/stable-version-arc-perf-test-fp8.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run.py
	# batch_size = 1
	sed -i '/batch_size/c\batch_size: 1' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/fp8/batch_size_1
	rm ./*.csv
	# batch_size = 2
	sed -i '/batch_size/c\batch_size: 2' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/fp8/batch_size_2
	rm ./*.csv
	# batch_size = 4
	sed -i '/batch_size/c\batch_size: 4' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/fp8/batch_size_4
	rm ./*.csv
	# batch_size = 8
	sed -i '/batch_size/c\batch_size: 8' config.yaml
	python run.py
	cp ./*.csv $CSV_SAVE_PATH/fp8/batch_size_8
	rm ./*.csv
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f $CSV_SAVE_PATH/fp8/batch_size_1 -b $CSV_SAVE_PATH/fp8/batch_size_1/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/fp8/batch_size_2 -b $CSV_SAVE_PATH/fp8/batch_size_2/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/fp8/batch_size_4 -b $CSV_SAVE_PATH/fp8/batch_size_4/transformer_int4_gpu-results-1baseline.csv -t 5.0
	python csv_to_html.py -f $CSV_SAVE_PATH/fp8/batch_size_8 -b $CSV_SAVE_PATH/fp8/batch_size_8/transformer_int4_gpu-results-1baseline.csv -t 5.0

	llm-stress-test-on-arc:
	needs: llm-perf-regression-test-on-arc
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.9"]
	runs-on: [self-hosted, llm, perf]
	env:
	OMP_NUM_THREADS: 16
	THREAD_NUM: 16
	ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
	CSV_SAVE_PATH: '/mnt/disk1/stable_version_stress_test_gpu/'

	steps:
	- uses: actions/checkout@v3

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	shell: bash
	# pip install transformers_stream_generator for model internlm-chat-7b-8k
	# pip install tiktoken for model Qwen-7B-Chat-10-12
	run: \|
	python -m pip install --upgrade pip
	python -m pip install --upgrade wheel
	python -m pip install --upgrade omegaconf
	python -m pip install --upgrade pandas
	python -m pip install --upgrade einops
	python -m pip install --upgrade transformers_stream_generator
	python -m pip install --upgrade tiktoken

	- name: Download llm binary
	uses: ./.github/actions/llm/download-llm-binary

	- name: Run LLM install (all) test
	uses: ./.github/actions/llm/setup-llm-env
	with:
	extra-dependency: "xpu_2.1"

	- name: Test installed xpu version
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	bash python/llm/test/run-llm-install-tests.sh

	- name: Test on xpu (int4)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	mv python/llm/test/benchmark/stable-version-arc-stress-test-sym_int4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py
	python run-stress-test.py
	cp ./*.csv $CSV_SAVE_PATH/int4
	rm ./*.csv
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f $CSV_SAVE_PATH/int4

	- name: Test on xpu (fp8)
	shell: bash
	run: \|
	source /opt/intel/oneapi/setvars.sh
	export USE_XETLA=OFF
	export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
	mv python/llm/test/benchmark/stable-version-arc-stress-test-fp8.yaml python/llm/dev/benchmark/all-in-one/config.yaml
	cd python/llm/dev/benchmark/all-in-one
	# hide time info
	sed -i 's/str(end - st)/"xxxxxx"/g' run-stress-test.py
	python run-stress-test.py
	cp ./*.csv $CSV_SAVE_PATH/fp8
	rm ./*.csv
	cd ../../../test/benchmark
	python -m pip install pandas==1.5.3
	python csv_to_html.py -f $CSV_SAVE_PATH/fp8

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ARC LLM Test for Stable Version #3

Workflow file

ARC LLM Test for Stable Version #3

Jobs

Run details

Workflow file for this run