-
Notifications
You must be signed in to change notification settings - Fork 3
239 lines (222 loc) · 9.1 KB
/
manual-accuary-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
name: Run accuracy test on Gaudi
on:
workflow_dispatch:
inputs:
examples:
default: "ChatQnA"
description: 'List of examples to test [AudioQnA,ChatQnA,CodeGen,FaqGen]'
required: true
type: string
datasets:
default: "en"
description: "List of services to test [en, zh]"
required: true
type: string
# for debug test
# pull_request:
# branches: [main]
# types: [opened, reopened, ready_for_review, synchronize]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-on-push
cancel-in-progress: true
jobs:
get-test-matrix:
runs-on: ubuntu-latest
outputs:
datasets: ${{ steps.get-matrix.outputs.datasets }}
examples: ${{ steps.get-matrix.outputs.examples }}
steps:
- name: Create Matrix
id: get-matrix
run: |
datasets=($(echo ${{ inputs.datasets }} | tr ',' ' '))
datasets_json=$(printf '%s\n' "${datasets[@]}" | jq -R '.' | jq -sc '.')
echo "datasets=$datasets_json" >> $GITHUB_OUTPUT
examples=($(echo ${{ inputs.examples }} | tr ',' ' '))
examples_json=$(printf '%s\n' "${examples[@]}" | jq -R '.' | jq -sc '.')
echo "examples=$examples_json" >> $GITHUB_OUTPUT
setup-acc:
runs-on: aise-acc
needs: [get-test-matrix]
strategy:
matrix:
dataset: ${{ fromJson(needs.get-test-matrix.outputs.datasets) }}
example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }}
exclude:
- example: AudioQnA
dataset: zh
- example: CodeGen
dataset: zh
- example: FaqGen
dataset: zh
fail-fast: false
steps:
- name: Clean up Working Directory
run: |
sudo rm -rf ${{github.workspace}}/* || true
docker system prune -f
docker rmi $(docker images --filter reference="100.83.111.229:5000/opea/*:latest" -q) || true
- name: Checkout out Validation
uses: actions/checkout@v4
with:
path: Validation
- name: Checkout out GenAIEval
uses: actions/checkout@v4
with:
fetch-depth: 0
repository: opea-project/GenAIEval
path: GenAIEval
- name: Checkout out GenAIExamples
uses: actions/checkout@v4
with:
repository: opea-project/GenAIExamples
path: GenAIExamples
- name: Set up environment
env:
example: ${{ matrix.example }}
run: |
set -x
conda_env_name="OPEA_acc"
export PATH=${HOME}/miniforge3/bin/:$PATH
if conda info --envs | grep -q "$conda_env_name"; then
echo "$conda_env_name exist! Recreating..."
conda env remove --name "$conda_env_name"
conda create -n ${conda_env_name} python=3.12 -y
echo "Created!"
else
conda create -n ${conda_env_name} python=3.12 -y
fi
conda_version=$(conda --version 2>&1)
version=$(echo $conda_version | grep -oP 'conda \K[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f2)
if [[ $version -ge 6 ]]; then
source activate ${conda_env_name}
else
source ${HOME}/miniforge3/etc/profile.d/conda.sh
conda activate ${conda_env_name}
fi
echo "Dataset: ${{ matrix.dataset }}"
echo "Example: ${{ matrix.example }}"
cd ${{ github.workspace }}/GenAIEval
pip install -r requirements.txt
if [[ "${example}" == "AudioQnA" ]]; then
cd ${{ github.workspace }}/GenAIExamples/AudioQnA/benchmark/accuracy
pip install -r requirements.txt
elif [[ "${example}" == "CodeGen" ]]; then
pip install -e .
elif [[ "${example}" == "FaqGen" ]]; then
pip install ragas==0.1.19
fi
- name: Evaluation Prepare
shell: bash
if: ${{ matrix.dataset }} == 'en' && ${{ matrix.dataset }} == 'ChatQnA'
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
example: ${{ matrix.example }}
dataset: ${{ matrix.dataset }}
run: |
export PATH=${HOME}/miniforge3/bin/:$PATH
conda_version=$(conda --version 2>&1)
version=$(echo $conda_version | grep -oP 'conda \K[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f2)
if [[ $version -ge 6 ]]; then
source activate OPEA_acc
else
source ${HOME}/miniforge3/etc/profile.d/conda.sh
conda activate OPEA_acc
fi
[ ! -d "acc-log" ] && mkdir acc-log
cp ${{ github.workspace }}/Validation/.github/scripts/acc_test.sh ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/
bash ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/acc_test.sh --eval_prepare "${example}" "${dataset}" | tee ${{ github.workspace }}/acc-log/${example}-${dataset}-eval_prepare.txt
- name: Launch Service
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
example: ${{ matrix.example }}
dataset: ${{ matrix.dataset }}
IMAGE_REPO: 100.83.111.229:5000/opea
run: |
export PATH=${HOME}/miniforge3/bin/:$PATH
conda_version=$(conda --version 2>&1)
version=$(echo $conda_version | grep -oP 'conda \K[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f2)
if [[ $version -ge 6 ]]; then
source activate OPEA_acc
else
source ${HOME}/miniforge3/etc/profile.d/conda.sh
conda activate OPEA_acc
fi
[ ! -d "acc-log" ] && mkdir acc-log
cp ${{ github.workspace }}/Validation/.github/scripts/acc_test.sh ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/
bash ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/acc_test.sh --launch_service "${example}" "${dataset}" | tee ${{ github.workspace }}/acc-log/${example}-${dataset}-launch_service.txt
- name: Run Evaluation
env:
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
example: ${{ matrix.example }}
dataset: ${{ matrix.dataset }}
port: 9001
run: |
set -x
export PATH=${HOME}/miniforge3/bin/:$PATH
conda_version=$(conda --version 2>&1)
version=$(echo $conda_version | grep -oP 'conda \K[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f2)
if [[ $version -ge 6 ]]; then
source activate OPEA_acc
else
source ${HOME}/miniforge3/etc/profile.d/conda.sh
conda activate OPEA_acc
fi
DPATH=${{ github.workspace }}/GenAIEval/
export PYTHONPATH=$PYTHONPATH:$DPATH
export PATH=$PATH:/bin:/usr/bin
[ ! -d "acc-log" ] && mkdir acc-log
cp ${{ github.workspace }}/Validation/.github/scripts/acc_test.sh ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/
bash ${{ github.workspace }}/GenAIExamples/${example}/benchmark/accuracy/acc_test.sh --launch_acc "${example}" "${dataset}" | tee ${{ github.workspace }}/acc-log/${example}-${dataset}-acc_test.txt
- name: Clean up container
if: always()
env:
example: ${{ matrix.example }}
run: |
docker system prune -f
cd ${{ github.workspace }}/GenAIExamples/${example}/docker_compose/intel/hpu/gaudi
docker compose stop && docker compose rm -f
docker rmi $(docker images --filter reference="100.83.111.229:5000/opea/*:latest" -q) || true
- name: Publish pipeline artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.example }}
path: ${{ github.workspace }}/acc-log/*.txt
Process-test-log:
runs-on: aise-acc
needs: [setup-acc, get-test-matrix]
strategy:
matrix:
example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }}
dataset: ${{ fromJson(needs.get-test-matrix.outputs.datasets) }}
exclude:
- example: AudioQnA
dataset: zh
- example: CodeGen
dataset: zh
- example: FaqGen
dataset: zh
fail-fast: false
steps:
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: ${{ matrix.example }}
path: ${{ github.workspace }}/acc-log/
continue-on-error: false
- name: Use the downloaded artifact
run: cat ${{ github.workspace }}/acc-log/*acc_test.txt | tee acc-log/summary.txt
- name: Publish pipeline artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
path: ${{ github.workspace }}/acc-log/summary.txt
name: acc-summary