forked from intel/neural-compressor
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathazure-pipelines.yml
58 lines (45 loc) · 4.1 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# pr:
# - main
pool:
vmImage: ubuntu-latest
variables:
PYTHON: 'python3.8'
INC_VER: '1.12'
IMAGE_NAME: 'neural-compressor'
IMAGE_TAG: '1.12'
INC_BRANCH: 'master'
stages:
- stage: Build
displayName: 'Build'
jobs:
- job: Build
displayName: 'Build'
steps:
- task: Docker@2
displayName: 'Login to repo'
inputs:
command: login
containerRegistry: inc-internal
- script: docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
displayName: 'Build Container'
- script: |
docker images | grep -i neural-compressor
displayName: 'Check docker'
# - script: |
# docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)"
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py
# displayName: 'Run python sciprt'
- script: |
docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash -c "cd examples/pytorch/nlp/huggingface_models/translation/quantization/ptq_dynamic/eager/ && pip install transformers==4.10.0 && pip install -r requirements.txt && pip install torch && sh run_tuning.sh --topology=marianmt_WMT_en_ro"
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash -c "cd examples/pytorch/nlp/huggingface_models/common && python setup.py install && cd /neural-compressor/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/ptq_static/eager && pip install -r requirements.txt && pip install torch>=1.7 transformers==4.10.0 && python -u run_clm_tune.py --model_name_or_path microsoft/DialoGPT-small --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train --do_eval --output_dir saved_results && cd saved_results && ls"
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash -c "cd examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/eager && python -m pip install -r requirements.txt && python -m pip install protobuf==3.20.0 && python -u run_summarization.py --model_name_or_path lvwerra/pegasus-samsum --do_train --do_eval --predict_with_generate --per_device_eval_batch_size 16 --output_dir saved_results --overwrite_output_dir --tune --dataset_name samsum"
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/eager/run_summarization.py --model_name_or_path lvwerra/pegasus-samsum --do_train --do_eval --predict_with_generate --per_device_eval_batch_size 16 --output_dir saved_results --overwrite_output_dir --tune
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash -c "cd examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/eager && /usr/local/bin/python3 run_summarization.py --model_name_or_path lvwerra/pegasus-samsum --do_train --do_eval --predict_with_generate --per_device_eval_batch_size 16 --output_dir saved_results --overwrite_output_dir --tune"
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash -c "examples/pytorch/nlp/huggingface_models/summarization/quantization/ptq_dynamic/eager/run_tuning.sh --topology=pegasus_samsum"
displayName: 'Tuning Summarization'
# - script: |
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python train_without_distillation.py --epochs 200 --lr 0.1 --layers 40 --widen-factor 2 --name WideResNet-40-2 --tensorboard
# displayName: 'Train teacher model'
# - script: |
# docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python main.py --epochs 200 --lr 0.02 --name MobileNetV2-0.35-distillation --teacher_model runs/WideResNet-40-2/model_best.pth.tar --tensorboard --seed 9
# displayName: 'Distillation'