-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathensemble.py
149 lines (128 loc) · 7.07 KB
/
ensemble.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
from logreg import Classifier
from reader import OffenseEvalData
from bertpredict import BertPredict
from sklearn.metrics import f1_score, accuracy_score
from collections import Counter
import argparse
def run__ensemble_classifier(args, testset):
# train the classifier on the training data
classifier = Classifier(args.data_dir, args.output_dir)
bert = BertPredict(args)
classifier.train()
processor = OffenseEvalData()
devel_indices = []
predicted_indices = []
with open(args.output_dir + "test_submission_ensemble.csv", "w") as file:
for sentence in (processor.get_test_examples(args.data_dir) if testset else processor.get_dev_examples(args.data_dir)):
char_pred = classifier.predict_one_char(sentence.text_a)
word_pred = classifier.predict_one_word(sentence.text_a)
bert_pred = bert.predict_one(sentence)
selected_label, _ = Counter([char_pred, word_pred, bert_pred]).most_common()[0]
devel_indices.append(classifier.label_index(sentence.label))
predicted_indices.append(classifier.label_index(selected_label))
file.write("%s,%s\n" % (sentence.guid, selected_label))
f1_macro = f1_score(devel_indices, predicted_indices, average="macro")
f1_micro = f1_score(devel_indices, predicted_indices, average="micro")
accuracy = accuracy_score(devel_indices, predicted_indices)
# print out performance
msg = "\n{:.1%} F1 macro {:.1%} F1 micro and {:.1%} accuracy on test data"
print(msg.format(f1_macro, f1_micro, accuracy))
def run_individual_classifier(args, testset):
# train the classifier on the training data
classifier = Classifier(args.data_dir, args.output_dir)
bert = BertPredict(args)
classifier.train()
processor = OffenseEvalData()
devel_indices = []
predicted_indices_char = []
predicted_indices_word = []
predicted_indices_bert = []
with open(args.output_dir + "test_submission_char.csv", "w") as char_file, \
open(args.output_dir + "test_submission_word.csv", "w") as word_file, \
open(args.output_dir + "test_submission_bert.csv", "w") as bert_file:
for sentence in (processor.get_test_examples(args.data_dir) if testset else processor.get_dev_examples(args.data_dir)):
char_pred = classifier.predict_one_char(sentence.text_a)
word_pred = classifier.predict_one_word(sentence.text_a)
bert_pred = bert.predict_one(sentence)
devel_indices.append(classifier.label_index(sentence.label))
predicted_indices_char.append(classifier.label_index(char_pred))
predicted_indices_word.append(classifier.label_index(word_pred))
predicted_indices_bert.append(classifier.label_index(bert_pred))
char_file.write("%s,%s\n" % (sentence.guid, char_pred))
word_file.write("%s,%s\n" % (sentence.guid, word_pred))
bert_file.write("%s,%s\n" % (sentence.guid, bert_pred))
f1_macro = f1_score(devel_indices, predicted_indices_char, average="macro")
f1_micro = f1_score(devel_indices, predicted_indices_char, average="micro")
accuracy = accuracy_score(devel_indices, predicted_indices_char)
# print out performance
msg = "\nCHAR : {:.1%} F1 macro {:.1%} F1 micro and {:.1%} accuracy on test data"
print(msg.format(f1_macro, f1_micro, accuracy))
f1_macro = f1_score(devel_indices, predicted_indices_word, average="macro")
f1_micro = f1_score(devel_indices, predicted_indices_word, average="micro")
accuracy = accuracy_score(devel_indices, predicted_indices_word)
# print out performance
msg = "\nWORD : {:.1%} F1 macro {:.1%} F1 micro and {:.1%} accuracy on test data"
print(msg.format(f1_macro, f1_micro, accuracy))
f1_macro = f1_score(devel_indices, predicted_indices_bert, average="macro")
f1_micro = f1_score(devel_indices, predicted_indices_bert, average="micro")
accuracy = accuracy_score(devel_indices, predicted_indices_bert)
# print out performance
msg = "\nBERT : {:.1%} F1 macro {:.1%} F1 micro and {:.1%} accuracy on test data"
print(msg.format(f1_macro, f1_micro, accuracy))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--data_dir",
default=None,
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--output_dir",
default=None,
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.")
## Bert Required parameters
parser.add_argument("--bert_model", type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--bert_model_dir",
default=None,
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.")
## Other parameters
parser.add_argument("--cache_dir",
default="",
type=str,
help="Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument("--max_seq_length",
default=512,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--testset',
type=bool,
default=False,
help="get predictions on test dataset")
args = parser.parse_args()
run_individual_classifier(args, args.testset)
run__ensemble_classifier(args, args.testset)
# usage
# python ensemble.py --data_dir data/ --output_dir output/ --bert_model_dir bert_trained_model/ --do_lower_case --bert_model bert-base-uncased