-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathMakefile
92 lines (79 loc) · 2.3 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# Dynamic Ensembling Makefile
# FINE-TUNING (Training BART with in-domain data)
# summarization datadir (TODO: format of this data?)
DATADIR ?= 'data/test_dataset'
BASE_MODEL_NAME_OR_PATH ?= 'bart-large-cnn'
OUTPUT_DIR ?= 'fine-tuned-model'
N_GPU ?= 0
MAX_SOURCE_LEN ?= 512
MAX_TARGET_LEN ?= 60
TRAIN_BATCH_SIZE ?= 1
EVAL_BATCH_SIZE ?= 1
# EVALUATION ARGS
EVALUATION_DATASET ?= data/WCEP/test.jsonl
MODEL_ID ?= bart-large-cnn
MAX_ARTICLES_IN_CLUSTER ?= 5
# used for flags and additional script args
RUN_FLAGS ?=
###########
## TASKS ##
###########
.PHONY: predict
evaluate:
python transformer_decoding/evaluate.py \
--evaluation-dataset $(EVALUATION_DATASET) \
--model-id $(MODEL_ID) \
$(RUN_FLAGS)
.PHONY: evaluate
evaluate:
python transformer_decoding/evaluate.py \
--evaluation-dataset $(EVALUATION_DATASET) \
--model-id $(MODEL_ID) \
$(RUN_FLAGS)
.PHONY: fine-tune-bart
fine-tune-bart:
mkdir -p $(OUTPUT_DIR)
python transformer_decoding/finetune.py \
--data_dir $(DATADIR) \
--model_type bart \
--model_name_or_path $(BASE_MODEL_NAME_OR_PATH) \
--learning_rate 3e-5 \
--train_batch_size $(TRAIN_BATCH_SIZE) \
--eval_batch_size $(EVAL_BATCH_SIZE) \
--max_source_length $(MAX_SOURCE_LEN) \
--max_target_length $(MAX_TARGET_LEN) \
--output_dir $(OUTPUT_DIR) \
--n_gpu $(N_GPU) \
--do_train
#.PHONY: fine-tune-bart
#fine-tune-bart:
# mkdir -p $(OUTPUT_DIR)
# python bin/run_bart_sum.py \
# --data_dir $(DATADIR) \
# --model_type bart \
# --model_name_or_path $(BASE_MODEL_NAME_OR_PATH) \
# --learning_rate 3e-5 \
# --train_batch_size $(TRAIN_BATCH_SIZE) \
# --eval_batch_size $(EVAL_BATCH_SIZE) \
# --max_seq_length $(MAX_SEQ_LEN) \
# --output_dir $(OUTPUT_DIR) \
# --n_gpu $(N_GPU) \
# --do_train
resources/$(TEST_RESOURCES_VERSION):
mkdir -p ./resources
gsutil cp -r $(RESOURCES_ROOT)/$(TEST_RESOURCES_VERSION) ./resources
.PHONY: test
test: resources/$(TEST_RESOURCES_VERSION)
RESOURCES=resources/$(TEST_RESOURCES_VERSION) python -Wignore -m unittest discover
pycodestyle aylien_entity_linking
.PHONY: clean
clean:
rm -f *.pyc *.pkl *.npy
rm -rf *.egg-info
#################
## DEVELOPMENT ##
#################
.PHONY: dev
dev:
pip install -e .
pip install -r requirements.txt