1
!pip install -i https://pypi.tuna.tsinghua.edu.cn/simple transformers==4.3.1
2

3
from transformers import AutoTokenizer, AutoModelForMaskedLM
4
tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
5
model = AutoModelForMaskedLM.from_pretrained("bert-base-chinese")
6

7
import torch
8
from transformers import BertTokenizer
9
from IPython.display import clear_output
10

11
PRETRAINED_MODEL_NAME = "bert-base-chinese"
12

13
# 取得此預訓練模型所使用的 tokenizer
14
tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)
15

16
clear_output()
17

18
vocab = tokenizer.vocab
19
print("字典大小：", len(vocab)) # 字典大小： 21128

1
text = "[CLS] 等到潮水 [MASK] 了，就知道谁沒穿裤子。"
2
tokens = tokenizer.tokenize(text)
3
ids = tokenizer.convert_tokens_to_ids(tokens)
4

5
print(text)
6
print(tokens[:10], '...')
7
print(ids[:10], '...')
8

9
"""
10
[CLS] 等到潮水 [MASK] 了，就知道谁沒穿裤子。
11
['[CLS]', '等', '到', '潮', '水', '[MASK]', '了', '，', '就', '知'] ...
12
[101, 5023, 1168, 4060, 3717, 103, 749, 8024, 2218, 4761] ...
13
"""
14

15

16
from transformers import BertForMaskedLM
17
# 除了 tokens 以外我們還需要辨別句子的 segment ids
18
tokens_tensor = torch.tensor([ids])  # (1, seq_len)
19
segments_tensors = torch.zeros_like(tokens_tensor)  # (1, seq_len)
20
maskedLM_model = BertForMaskedLM.from_pretrained(PRETRAINED_MODEL_NAME)
21

22
# 使用 masked LM 估計 [MASK] 位置所代表的實際 token
23
maskedLM_model.eval()
24
with torch.no_grad():
25
    outputs = maskedLM_model(tokens_tensor, segments_tensors)
26
    predictions = outputs[0] # (1, seq_len, num_hidden_units)
27
del maskedLM_model
28

29
# 將 [MASK] 位置的機率分佈取 top k 最有可能的 tokens 出來
30
masked_index = 5
31
k = 3
32
probs, indices = torch.topk(torch.softmax(predictions[0, masked_index], -1), k)
33
predicted_tokens = tokenizer.convert_ids_to_tokens(indices.tolist())
34

35
# 顯示 top k 可能的字。一般我們就是取 top 1 当做预测值
36
print("輸入 tokens ：", tokens[:10], '...')
37
print('-' * 50)
38
for i, (t, p) in enumerate(zip(predicted_tokens, probs), 1):
39
    tokens[masked_index] = t
40
    print("Top {} ({:2}%)：{}".format(i, int(p.item() * 100), tokens[:10]), '...')
41

42
"""
43
輸入 tokens ： ['[CLS]', '等', '到', '潮', '水', '[MASK]', '了', '，', '就', '知'] ...
44
--------------------------------------------------
45
Top 1 (65%)：['[CLS]', '等', '到', '潮', '水', '来', '了', '，', '就', '知'] ...
46
Top 2 ( 4%)：['[CLS]', '等', '到', '潮', '水', '过', '了', '，', '就', '知'] ...
47
Top 3 ( 4%)：['[CLS]', '等', '到', '潮', '水', '干', '了', '，', '就', '知'] ...
48
"""

1
#主要要学会使用apply+lambda
2

3
# 词汇数量
4
train['word_count']=train['tweet'].apply(lambda x:len(str(x).split(" ")))
5
train[['tweet','word_count']].head()
6

7
# 字符数量
8
train['char_count']=train['tweet'].str.len()
9
train[['tweet','char_count']].head()
10

11
# 平均字长
12
def avg_word(sentence):
13
    words=sentence.split()
14
    return (sum(len(word) for word in words)/len(words))
15

16
train['avg_word']=train['tweet'].apply(lambda x:avg_word(x))
17
train[['tweet','avg_word']].head()
18

19
# 停用词数量
20
"""
21
为节省存储空间和提高搜索效率，搜索引擎在索引页面或处理搜索请求时会自动忽略某些字或词，这些字或词即被称为Stop Words(停用词)。
22
"""
23
!pip install nltk
24
import nltk
25
nltk.download('stopwords')
26
from nltk.corpus import stopwords
27
stop=stopwords.words('english')
28

29
train['stopwords']=train['tweet'].apply(lambda sen:len([x for x in sen.split() if x in stop]))
30
train[['tweet','stopwords']].head()
31

32
# 特殊字符数量
33
train['hashtags']=train['tweet'].apply(lambda sen:len([x for x in sen.split() if x.startswith("#")]))
34
train[['tweet','hashtags']].head()
35

36
# 数字数量
37
train['numerics']=train['tweet'].apply(lambda sen:len([x for x in sen.split() if x.isdigit()]))
38
train[['tweet','numerics']].head()
39

40
# 大写字母数量
41
train['upper']=train['tweet'].apply(lambda sen:len([x for x in sen.split() if x.isupper()]))
42
train[['tweet','upper']].head()

1
# 小写转换
2
"""
3
这避免了拥有相同的多个副本。例如,当我们计算字词汇数量时,“Analytics”和“analytics”将被视为不同的单词。
4
"""
5
train['tweet']=train['tweet'].apply(lambda sen:" ".join(x.lower() for x in sen.split()))
6
train['tweet'].head()
7

8
# 去除标点符号
9
"""
10
标点符号在文本数据中不添加任何额外的信息
11
"""
12
train['tweet'] = train['tweet'].str.replace('[^\w\s]','')
13
train['tweet'].head()
14

15
# 去除停用词
16
from nltk.corpus import stopwords
17
stop=stopwords.words('english')
18
train['tweet']=train['tweet'].apply(lambda sen:" ".join(x for x in sen.split() if x not in stop))
19
train['tweet'].head()
20

21
# 去除频现词
22
freq=pd.Series(' '.join(train['tweet']).split()).value_counts()[:10]
23
freq
24
"""
25
user     17473
26
love      2647
27
ð         2511
28
day       2199
29
â         1797
30
happy     1663
31
amp       1582
32
im        1139
33
u         1136
34
time      1110
35
"""
36
freq=list(freq.index)
37
train['tweet']=train['tweet'].apply(lambda sen:' '.join(x for x in sen.split() if x not in freq))
38
train['tweet'].head()
39

40

41
# 去除稀疏词
42
freq = pd.Series(' '.join(train['tweet']).split()).value_counts()[-10:]
43
freq
44
freq = list(freq.index)
45
train['tweet'] = train['tweet'].apply(lambda x: " ".join(x for x in x.split() if x not in freq))
46
train['tweet'].head()
47

48
# 拼写校正
49
!pip install textblob
50
from textblob import TextBlob
51
train['tweet'][:5].apply(lambda x: str(TextBlob(x).correct()))
52

53
# 分词(tokenization)
54
import nltk
55
nltk.download('punkt')
56
TextBlob(train['tweet'][1]).words
57

58

59
# 词干提取(stemming)
60
"""
61
是指通过基于规则的方法去除单词的后缀，比如“ing”,“ly”，“s”等等。
62
"""
63
from nltk.stem import PorterStemmer
64
st=PorterStemmer()
65
train['tweet'][:5].apply(lambda x:" ".join([st.stem(word) for word in x.split()]))
66

67
# 词形还原(lemmatization)
68
from textblob import Word
69
train['tweet']=train['tweet'].apply(lambda x:" ".join([Word(word).lemmatize() for word in x.split()]))
70
train['tweet'].head()

1
# N-grams语言模型
2
"""
3
N-grams称为N元语言模型，是多个词语的组合，是一种统计语言模型，用来根据前(n-1)个item来预测第n个item。常见模型有一元语言模型(unigrams)、二元语言模型（bigrams ）、三元语言模型(trigrams)。
4
Unigrams包含的信息通常情况下比bigrams和trigrams少，需要根据具体应用选择语言模型，因为如果n-grams太短，这时不能捕获重要信息。另一方面，如果n-grams太长，那么捕获的信息基本上是一样的，没有差异性
5
"""
6
TextBlob(train['tweet'][0]).ngrams(2)
7

8
# 词频
9
tf1 = (train['tweet'][1:2]).apply(lambda x: pd.value_counts(x.split(" "))).sum(axis = 0).reset_index()
10
tf1.columns = ['words','tf']
11
tf1
12

13
# 逆文档频率
14
"""
15
反转文档频率(Inverse Document Frequency)，简称为IDF，其原理可以简单理解为如果一个单词在所有文档都会出现，那么可能这个单词对我们没有那么重要。
16
一个单词的IDF就是所有行数与出现该单词的行的个数的比例，最后对数。
17
"""
18
import numpy as np
19
for i,word in enumerate(tf1['words']):
20
    tf1.loc[i, 'idf'] =np.log(train.shape[0]/(len(train[train['tweet'].str.contains(word)])))
21
tf1
22

23
# TF-IDF
24
"""
25
TF-IDF=TF*IDF
26
"""
27
from sklearn.feature_extraction.text import TfidfVectorizer
28
tfidf = TfidfVectorizer(max_features=1000, lowercase=True, analyzer='word',
29
 stop_words= 'english',ngram_range=(1,1))
30
train_vect = tfidf.fit_transform(train['tweet'])
31
train_vect
32

33
# 词袋
34
"""
35
BOW，就是将文本/Query看作是一系列词的集合。由于词很多，所以咱们就用袋子把它们装起来，简称词袋。
36
"""
37
from sklearn.feature_extraction.text import CountVectorizer
38
bow = CountVectorizer(max_features=1000, lowercase=True, ngram_range=(1,1),analyzer = "word")
39
train_bow = bow.fit_transform(train['tweet'])
40
train_bow
41

42
# 情感分析
43
from textblob import TextBlob
44
testimonial = TextBlob("Textblob is amazingly simple to use. What great fun!")
45
print(testimonial.sentiment)
46
"""
47
Sentiment(polarity=0.39166666666666666, subjectivity=0.4357142857142857)
48
"""
49

50
# 词嵌入
51
from gensim.scripts.glove2word2vec import glove2word2vec
52
glove_input_file = 'glove.6B.100d.txt'
53
word2vec_output_file = 'glove.6B.100d.txt.word2vec'
54
glove2word2vec(glove_input_file, word2vec_output_file)

1
# ---------- 安装 ----------
2

3
git lfs install
4
git clone https://huggingface.co/hfl/chinese-roberta-wwm-ext
5
# if you want to clone without large files – just their pointers
6
# prepend your git clone with the following env var:
7
GIT_LFS_SKIP_SMUDGE=1
8

9
"""
10
from transformers import AutoTokenizer, AutoModelForMaskedLM
11

12
tokenizer = AutoTokenizer.from_pretrained("ckiplab/albert-tiny-chinese")
13

14
model = AutoModelForMaskedLM.from_pretrained("ckiplab/albert-tiny-chinese")
15
"""
16

17
# ---------- 导入 ----------
18

19
from transformers import AutoConfig,AutoModel,AutoTokenizer,AdamW,get_linear_schedule_with_warmup,logging
20
import torch
21
import torch.nn as nn
22
import torch.nn.functional as F
23
from torch.utils.data import TensorDataset,SequentialSampler,RandomSampler,DataLoader
24
MODEL_NAME="bert-base-chinese"
25
# MODEL_NAME="roberta-large"
26

27
# ---------- 查看配置 ----------
28
config = AutoConfig.from_pretrained(MODEL_NAME)
29
config
30

31
# ---------- tokenizer ----------
32

33
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
34
tokenizer
35
"""
36
PreTrainedTokenizerFast(name_or_path='bert-base-chinese', vocab_size=21128, model_max_len=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})
37
"""
38

39
tokenizer.all_special_ids
40
"""
41
[100, 102, 0, 101, 103]
42
"""
43

44
tokenizer.all_special_tokens
45
"""
46
['[UNK]', '[SEP]', '[PAD]', '[CLS]', '[MASK]']
47
"""
48

49
# 词汇表大小
50
tokenizer.vocab_size # 21128
51

52
# ---------- 将文本转为词汇表id 1 (encode) ----------
53
"""
54
encode(  self,  text,  text_pair,  add_special_tokens,
55
      padding,  truncation,  max_length,  stride,  return_tensors,
56
      **kwargs  ) -> List[int]
57
Converts a string to a sequence of ids (integer), using the tokenizer and vocabulary.
58
"""
59

60
text="我在北京工作"
61
token_ids=tokenizer.encode(text)
62
token_ids
63
# [101, 2769, 1762, 1266, 776, 2339, 868, 102]
64
tokenizer.convert_ids_to_tokens(token_ids)
65
# ['[CLS]', '我', '在', '北', '京', '工', '作', '[SEP]']
66

67
# 加入参数
68
token_ids=tokenizer.encode(text,padding=True,max_length=30,add_special_tokens=True)
69
token_ids
70
# [101, 2769, 1762, 1266, 776, 2339, 868, 102] 这个还是不变
71
token_ids=tokenizer.encode(text,padding="max_length",max_length=30,add_special_tokens=True)
72
token_ids
73
# [101, 2769, 1762, 1266, 776, 2339, 868, 102,0,0,0,.....] padding到30
74
token_ids=tokenizer.encode(text,padding="max_length",max_length=30,add_special_tokens=True,return_tensors='pt')
75
token_ids
76
# 返回pytorch tensor格式的列表
77

78
# ---------- 将文本转为词汇表id 2 (encode_plus) ----------
79
""" 确实是plus版本 主要是返回相关的参数多了
80
def encode_plus(   self,  text,  text_pair,  add_special_tokens,
81
      padding,  truncation,  max_length,  stride,  return_tensors,
82
      return_tensors,
83
      return_token_type_ids,
84
      return_attention_mask,
85
      return_overflowing_tokens,
86
      return_special_tokens_mask,
87
      return_offsets_mapping,
88
      return_length,
89
      verbose,
90
      **kwargs
91
  ) -> BatchEncoding:
92
"""
93

94
token_ids=tokenizer.encode_plus(
95
    text,padding="max_length",
96
    max_length=30,
97
    add_special_tokens=True,
98
    return_tensors='pt',
99
    return_token_type_ids=True,
100
    return_attention_mask=True
101
)
102
token_ids
103

104
"""
105
返回
106
1.pytorch的tensor格式id
107
2.token_type_ids
108
3.attention_mask
109

110
{
111
    'input_ids': tensor([
112
        [ 101, 2769, 1762, 1266,  776,
113
         2339,  868,  102,    0,    0,
114
            0,    0,    0,    0,    0,
115
            0,    0,    0,    0,    0,
116
            0,    0,    0,    0,    0,
117
            0,    0,    0,    0,    0]
118
  ]),
119
  'token_type_ids': tensor([
120
    [0, 0, 0, 0, 0,
121
    0, 0, 0, 0, 0,
122
    0, 0, 0, 0, 0,
123
    0, 0, 0, 0, 0,
124
    0, 0, 0, 0, 0,
125
    0, 0, 0, 0, 0]
126
  ]),
127
    'attention_mask': tensor([
128
      [1, 1, 1, 1, 1,
129
      1, 1, 1, 0, 0,
130
      0, 0, 0, 0, 0,
131
      0, 0, 0, 0, 0,
132
      0, 0, 0, 0, 0,
133
      0, 0, 0, 0, 0]
134
    ])
135
}
136
"""
137

138
# ---------- Model ----------
139
model=AutoModel.from_pretrained(MODEL_NAME)
140
model
141

142
"""
143
查看模型结构
144

145
BertModel(
146
  (embeddings): BertEmbeddings(
147
    (word_embeddings): Embedding(21128, 768, padding_idx=0)
148
    (position_embeddings): Embedding(512, 768)
149
    (token_type_embeddings): Embedding(2, 768)
150
    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
151
    (dropout): Dropout(p=0.1, inplace=False)
152
  )
153
  (encoder): BertEncoder(
154
    (layer): ModuleList(
155
      (0)-(11): BertLayer(  # 12个一样的BertLayer构成encoder
156
        (attention): BertAttention(
157
          (self): BertSelfAttention(
158
            (query): Linear(in_features=768, out_features=768, bias=True)
159
            (key): Linear(in_features=768, out_features=768, bias=True)
160
            (value): Linear(in_features=768, out_features=768, bias=True)
161
            (dropout): Dropout(p=0.1, inplace=False)
162
          )
163
          (output): BertSelfOutput(
164
            (dense): Linear(in_features=768, out_features=768, bias=True)
165
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
166
            (dropout): Dropout(p=0.1, inplace=False)
167
          )
168
        )
169
        (intermediate): BertIntermediate(
170
          (dense): Linear(in_features=768, out_features=3072, bias=True)
171
          (intermediate_act_fn): GELUActivation()
172
        )
173
        (output): BertOutput(
174
          (dense): Linear(in_features=3072, out_features=768, bias=True)
175
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
176
          (dropout): Dropout(p=0.1, inplace=False)
177
        )
178
      )
179
    )
180
  )
181
  (pooler): BertPooler(
182
    (dense): Linear(in_features=768, out_features=768, bias=True)
183
    (activation): Tanh()
184
  )
185
)
186
"""
187

188

189
# ---------- 输出 ----------
190

191
outputs=model(token_ids['input_ids'],token_ids['attention_mask'])
192

193
outputs.keys()
194
"""
195
odict_keys(['last_hidden_state', 'pooler_output'])
196
"""
197

198
last_hidden_state=
199
outputs[0].shape # last_hidden_state, torch.Size([1, 30, 768])
200
outputs[1].shape # 句子pooler_output, torch.Size([1, 30, 768])
201
outputs[0][:,0].shape # 第一个字符CLS的embedding表示 torch.Size([1, 768])
202

203
# ---------- 对Bert输出进行变换 ----------
204
config.update({
205
            'output_hidden_states':True
206
            })
207
model=AutoModel.from_pretrained(MODEL_NAME,config=config)
208
outputs=model(token_ids['input_ids'],token_ids['token_type_ids'])
209
outputs.keys()
210
"""
211
odict_keys(['last_hidden_state', 'pooler_output', 'hidden_states'])
212
"""

1
df['one_hot_labels'] = list(df[label_cols].values) # 直接将六个标签转为one hot
2
labels = list(df.one_hot_labels.values)
3
comments = list(df.comment_text.values)
4

5
tokenizer = AutoTokenizer.from_pretrained()
6
encodings = tokenizer.batch_encode_plus()
7

8
input_ids = encodings['input_ids'] # tokenized and encoded sentences
9
token_type_ids = encodings['token_type_ids'] # token type ids
10
attention_masks = encodings['attention_mask'] # attention masks
11

12
# 训练集和验证集划分
13

14
train_inputs, validation_inputs,
15
train_labels, validation_labels,
16
train_token_types, validation_token_types,
17
train_masks, validation_masks =
18
  train_test_split(
19
        input_ids, labels,
20
        token_type_ids, attention_masks,
21
        random_state=2020, test_size=0.10,
22
        stratify = labels
23
    )
24

25
label_counts = df.one_hot_labels.astype(str).value_counts()
26
one_freq = label_counts[label_counts==1].keys()
27
one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)
28

29
# Gathering single instance inputs to force into the training set after stratified split
30
one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]
31
one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]
32
one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]
33
one_freq_labels = [labels.pop(i) for i in one_freq_idxs]
34

35
# Add one frequency data to train data
36
train_inputs.extend(one_freq_input_ids)
37
train_labels.extend(one_freq_labels)
38
train_masks.extend(one_freq_attention_masks)
39
train_token_types.extend(one_freq_token_types)
40

41
# 将原始id转为 torch 张量
42
train_inputs = torch.tensor(train_inputs)
43
train_labels = torch.tensor(train_labels)
44
train_masks = torch.tensor(train_masks)
45
train_token_types = torch.tensor(train_token_types)
46

47
validation_inputs = torch.tensor(validation_inputs)
48
validation_labels = torch.tensor(validation_labels)
49
validation_masks = torch.tensor(validation_masks)
50
validation_token_types = torch.tensor(validation_token_types)
51

52
batch_size = 32
53

54
# 训练集
55
train_data = TensorDataset(train_inputs, train_masks, train_labels, train_token_types)
56
train_sampler = RandomSampler(train_data) #
57
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
58

59
validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels, validation_token_types)
60
validation_sampler = SequentialSampler(validation_data) # 按顺序遍历
61
validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)
62

63
torch.save(validation_dataloader,'validation_data_loader')
64
torch.save(train_dataloader,'train_data_loader')
65

66
from transformers import AutoModelForSequenceClassification
67
# 加载预训练模型
68
model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=num_labels)
69
# num_labels：6 默认情况2分类
70

71
paras=[para for para in model.named_parameters()]
72

73
from transformers import AdamW
74
# 对不同参数设置weight_decay_rate
75
param_optimizer = list(model.named_parameters())
76
no_decay = ['bias', 'gamma', 'beta']
77
optimizer_grouped_parameters = [
78
    {'params': [p for n, p in param_optimizer
79
                if not any(nd in n for nd in no_decay)],
80
     'weight_decay_rate': 0.01},
81
    {'params': [p for n, p in param_optimizer
82
                if any(nd in n for nd in no_decay)],
83
     'weight_decay_rate': 0.0}
84
]
85

86
optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5,correct_bias=True)
87
# 1e-5,2e-5,5e-5
88
# optimizer = AdamW(model.parameters(),lr=2e-5)  # 默认优化器
89

90
# Store our loss and accuracy for plotting
91
train_loss_set = []
92

93
# Number of training epochs (authors recommend between 2 and 4)
94
epochs = 3 # 训练轮数，15万训练集 任务比较简单的，最多设置5
95

96
# trange is a tqdm wrapper around the normal python range
97
for _ in trange(epochs, desc="Epoch"):
98
  # Training
99
  # Set our model to training mode (as opposed to evaluation mode)
100
  model.train() # 设置训练模式
101

102
  # Tracking variables
103
  tr_loss = 0 #running loss
104
  nb_tr_examples, nb_tr_steps = 0, 0
105

106
  # Train the data for one epoch
107
  for step, batch in enumerate(train_dataloader):# 遍历批数据
108
    # Add batch to GPU
109
    batch = tuple(t.to(device) for t in batch)
110
    b_input_ids, b_input_mask, b_labels, b_token_types = batch
111
    optimizer.zero_grad()
112

113
    # loss = outputs[0]
114
    # logits = outputs[1]
115

116
    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
117
    logits = outputs[0]
118
    loss_func = BCEWithLogitsLoss() # 计算损失
119
    loss = loss_func(
120
        logits.view(-1,num_labels),
121
        b_labels.type_as(logits).view(-1,num_labels)
122
    )
123
    train_loss_set.append(loss.item())# 记录loss
124

125
    # Backward pass
126
    loss.backward() # loss反向求导
127
    # Update parameters and take a step using the computed gradient
128
    optimizer.step()
129
    tr_loss += loss.item()
130
    nb_tr_examples += b_input_ids.size(0)
131
    nb_tr_steps += 1
132

133
  print("Train loss: {}".format(tr_loss/nb_tr_steps))
134

135

136
  # Validation
137

138
  # Put model in evaluation mode to evaluate loss on the validation set
139
  model.eval()
140

141
  # Variables to gather full output
142
  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]
143

144
  # Predict
145
  for i, batch in enumerate(validation_dataloader):
146
    batch = tuple(t.to(device) for t in batch)
147
    # Unpack the inputs from our dataloader
148
    b_input_ids, b_input_mask, b_labels, b_token_types = batch
149
    with torch.no_grad():
150
      # Forward pass
151
      outs = model(
152
          b_input_ids,
153
          token_type_ids=None,
154
          attention_mask=b_input_mask
155
      )
156

157
      b_logit_pred = outs[0]
158
      pred_label = torch.sigmoid(b_logit_pred)
159

160
      b_logit_pred = b_logit_pred.detach().cpu().numpy()
161
      pred_label = pred_label.to('cpu').numpy()
162
      b_labels = b_labels.to('cpu').numpy()
163

164
    tokenized_texts.append(b_input_ids)
165
    logit_preds.append(b_logit_pred)
166
    true_labels.append(b_labels)
167
    pred_labels.append(pred_label)
168

169
  # Flatten outputs
170
  pred_labels = [item for sublist in pred_labels for item in sublist]
171
  true_labels = [item for sublist in true_labels for item in sublist]
172

173
  # 计算准确率
174
  threshold = 0.50
175
  pred_bools = [pl>threshold for pl in pred_labels]
176
  true_bools = [tl==1 for tl in true_labels]
177
  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100
178
  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100
179

180
  print('F1 Validation Accuracy: ', val_f1_accuracy)
181
  print('Flat Validation Accuracy: ', val_flat_accuracy)
182

183
torch.save(model.state_dict(), 'bert_model_toxic')
184

185

186
...
187

188
# Test
189

190
test_encodings = tokenizer.batch_encode_plus(test_comments)
191
test_input_ids = test_encodings['input_ids']
192
test_token_type_ids = test_encodings['token_type_ids']
193
test_attention_masks = test_encodings['attention_mask']
194

195
# Make tensors out of data
196
test_inputs = torch.tensor(test_input_ids)
197
test_labels = torch.tensor(test_labels)
198
test_masks = torch.tensor(test_attention_masks)
199
test_token_types = torch.tensor(test_token_type_ids)
200

201
# Create test dataloader
202
test_data = TensorDataset(test_inputs, test_masks, test_labels, test_token_types)
203
test_sampler = SequentialSampler(test_data)
204
test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)
205

206
# Save test dataloader
207
torch.save(test_dataloader,'test_data_loader')
208

209
# Put model in evaluation mode to evaluate loss on the validation set
210
model.eval()
211

212
#track variables
213
logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]
214

215
# Predict
216
for i, batch in enumerate(test_dataloader):
217
  batch = tuple(t.to(device) for t in batch)
218
  # Unpack the inputs from our dataloader
219
  b_input_ids, b_input_mask, b_labels, b_token_types = batch
220
  with torch.no_grad():
221
    # Forward pass
222
    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)
223
    b_logit_pred = outs[0]
224
    pred_label = torch.sigmoid(b_logit_pred)
225

226
    b_logit_pred = b_logit_pred.detach().cpu().numpy()
227
    pred_label = pred_label.to('cpu').numpy()
228
    b_labels = b_labels.to('cpu').numpy()
229

230
  tokenized_texts.append(b_input_ids)
231
  logit_preds.append(b_logit_pred)
232
  true_labels.append(b_labels)
233
  pred_labels.append(pred_label)
234

235
# Flatten outputs
236
tokenized_texts = [item for sublist in tokenized_texts for item in sublist]
237
pred_labels = [item for sublist in pred_labels for item in sublist]
238
true_labels = [item for sublist in true_labels for item in sublist]
239
# Converting flattened binary values to boolean values
240
true_bools = [tl==1 for tl in true_labels]
241
pred_bools = [pl>0.50 for pl in pred_labels] #boolean output after thresholding
242

243
idx2label = dict(zip(range(6),label_cols))
244

245
# Getting indices of where boolean one hot vector true_bools is True so we can use idx2label to gather label names
246
true_label_idxs, pred_label_idxs=[],[]
247
for vals in true_bools:
248
  true_label_idxs.append(np.where(vals)[0].flatten().tolist())
249
for vals in pred_bools:
250
  pred_label_idxs.append(np.where(vals)[0].flatten().tolist())
251

252
# Gathering vectors of label names using idx2label
253
true_label_texts, pred_label_texts = [], []
254
for vals in true_label_idxs:
255
  if vals:
256
    true_label_texts.append([idx2label[val] for val in vals])
257
  else:
258
    true_label_texts.append(vals)
259

260
for vals in pred_label_idxs:
261
  if vals:
262
    pred_label_texts.append([idx2label[val] for val in vals])
263
  else:
264
    pred_label_texts.append(vals)
265

266
# Decoding input ids to comment text
267
comment_texts = [tokenizer.decode(text,skip_special_tokens=True,clean_up_tokenization_spaces=False) for text in tokenized_texts]
268

269
# Converting lists to df
270
comparisons_df = pd.DataFrame({'comment_text': comment_texts, 'true_labels': true_label_texts, 'pred_labels':pred_label_texts})
271
comparisons_df.to_csv('comparisons.csv')
272
comparisons_df.head()
273

274

275

276

277

278
macro_thresholds = np.array(range(1,10))/10
279
macro_thresholds
280

281
f1_results, flat_acc_results = [], []
282
for th in macro_thresholds:
283
  pred_bools = [pl>th for pl in pred_labels]
284
  test_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')
285
  test_flat_accuracy = accuracy_score(true_bools, pred_bools)
286
  f1_results.append(test_f1_accuracy)
287
  flat_acc_results.append(test_flat_accuracy)
288

289
best_macro_th = macro_thresholds[np.argmax(f1_results)] #best macro threshold value
290

291
micro_thresholds = (np.array(range(10))/100)+best_macro_th #calculating micro threshold values
292

293
f1_results, flat_acc_results = [], []
294
for th in micro_thresholds:
295
  pred_bools = [pl>th for pl in pred_labels]
296
  test_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')
297
  test_flat_accuracy = accuracy_score(true_bools, pred_bools)
298
  f1_results.append(test_f1_accuracy)
299
  flat_acc_results.append(test_flat_accuracy)
300

301
best_f1_idx = np.argmax(f1_results) #best threshold value
302

303
# Printing and saving classification report
304
print('Best Threshold: ', micro_thresholds[best_f1_idx])
305
print('Test F1 Accuracy: ', f1_results[best_f1_idx])
306
print('Test Flat Accuracy: ', flat_acc_results[best_f1_idx], '\n')
307

308
best_pred_bools = [pl>micro_thresholds[best_f1_idx] for pl in pred_labels]
309
clf_report_optimized = classification_report(true_bools,best_pred_bools, target_names=label_cols)
310
pickle.dump(clf_report_optimized, open('classification_report_optimized.txt','wb'))
311
print(clf_report_optimized)

1
def set_seed(seed):
2
    """ Set all seeds to make results reproducible """
3
    torch.manual_seed(seed)
4
    torch.cuda.manual_seed_all(seed)
5
    torch.backends.cudnn.deterministic = True
6
    torch.backends.cudnn.benchmark = False
7
    np.random.seed(seed)
8
    random.seed(seed)
9
    os.environ['PYTHONHASHSEED'] = str(seed)
10

11

12

13
A. 保证结果的可复现性  [√]
14
B. 保证模型参数的多样性
15
C. 加快模型收敛