-
Notifications
You must be signed in to change notification settings - Fork 383
/
question_classify_train.py
181 lines (160 loc) · 6.44 KB
/
question_classify_train.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#!/usr/bin/env python3
# coding: utf-8
# File: question_classify.py
# Author: lhy<[email protected],https://huangyong.github.io>
# Date: 18-11-10
import os
import numpy as np
import jieba.posseg as pseg
from keras.models import Sequential, load_model
from keras.layers import Conv1D, GlobalAveragePooling1D, MaxPooling1D, Dense, Dropout, LSTM, Bidirectional
class QuestionClassify(object):
def __init__(self):
self.label_dict = {
0:"婚姻家庭",
1:"劳动纠纷",
2:"交通事故",
3:"债权债务",
4:"刑事辩护",
5:"合同纠纷",
6:"房产纠纷",
7:"侵权",
8:"公司法",
9:"医疗纠纷",
10:"拆迁安置",
11:"行政诉讼",
12:"建设工程"
}
cur = '/'.join(os.path.abspath(__file__).split('/')[:-1])
self.train_file = os.path.join(cur, 'question_train.txt')
self.embedding_path = os.path.join(cur, 'word_vec_300.bin')
self.embdding_dict = self.load_embedding(self.embedding_path)
self.max_length = 60
self.embedding_size = 300
self.lstm_modelpath = 'model/lstm_question_classify.h5'
self.cnn_modelpath = 'model/cnn_question_classify.h5'
return
'''加载词向量'''
def load_embedding(self, embedding_path):
embedding_dict = {}
count = 0
for line in open(embedding_path):
line = line.strip().split(' ')
if len(line) < 300:
continue
wd = line[0]
vector = np.array([float(i) for i in line[1:]])
embedding_dict[wd] = vector
count += 1
if count%10000 == 0:
print(count, 'loaded')
print('loaded %s word embedding, finished'%count, )
return embedding_dict
'''对文本进行分词处理'''
def seg_sent(self, s):
wds = [i.word for i in pseg.cut(s) if i.flag[0] not in ['w', 'x']]
return wds
'''基于wordvector，通过lookup table的方式找到句子的wordvector的表示'''
def rep_sentencevector(self, sentence):
word_list = self.seg_sent(sentence)[:self.max_length]
embedding_matrix = np.zeros((self.max_length, self.embedding_size))
for index, wd in enumerate(word_list):
if wd in self.embdding_dict:
embedding_matrix[index] = self.embdding_dict.get(wd)
else:
continue
len_sent = len(word_list)
embedding_matrix = self.modify_sentencevector(embedding_matrix, len_sent)
return embedding_matrix
'''对于OOV词,通过左右词的词向量作平均,作为词向量表示'''
def modify_sentencevector(self, embedding_matrix, len_sent):
context_window = 2
for indx, vec in enumerate(embedding_matrix):
left = indx-context_window
right = indx+context_window
if left < 0:
left = 0
if right > len(embedding_matrix)-1:
right = -2
context = embedding_matrix[left:right+1]
if vec.tolist() == [0]*300 and indx < len_sent:
context_vector = context.mean(axis=0)
embedding_matrix[indx] = context_vector
return embedding_matrix
'''对数据进行onehot映射操作'''
def label_onehot(self, label):
one_hot = [0]*len(self.label_dict)
one_hot[int(label)] = 1
return one_hot
'''加载数据集'''
def load_traindata(self):
train_X = []
train_Y = []
count = 0
for line in open(self.train_file):
line = line.strip().strip().split('\t')
if len(line) < 2:
continue
count += 1
sent = line[0]
label = line[1]
sent_vector = self.rep_sentencevector(sent)
label_vector = self.label_onehot(label)
train_X.append(sent_vector)
train_Y.append(label_vector)
if count % 10000 == 0:
print('loaded %s lines'%count)
return np.array(train_X), np.array(train_Y)
'''构造CNN网络模型'''
def build_cnn_model(self):
model = Sequential()
model.add(Conv1D(64, 3, activation='relu', input_shape=(self.max_length, self.embedding_size)))
model.add(Conv1D(64, 3, activation='relu'))
model.add(MaxPooling1D(3))
model.add(Conv1D(128, 3, activation='relu'))
model.add(Conv1D(128, 3, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(0.5))
model.add(Dense(13, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
return model
'''构造LSTM网络'''
def build_lstm_model(self):
model = Sequential()
model.add(LSTM(32, return_sequences=True, input_shape=(self.max_length, self.embedding_size))) # returns a sequence of vectors of dimension 32
model.add(LSTM(32, return_sequences=True)) # returns a sequence of vectors of dimension 32
model.add(LSTM(32)) # return a single vector of dimension 32
model.add(Dense(13, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
'''训练CNN模型'''
def train_cnn(self):
X_train, Y_train, X_test, Y_test = self.split_trainset()
model = self.build_cnn_model()
model.fit(X_train, Y_train, batch_size=100, epochs=20, validation_data=(X_test, Y_test))
model.save(self.cnn_modelpath)
'''训练CNN模型'''
def train_lstm(self):
X_train, Y_train, X_test, Y_test = self.split_trainset()
model = self.build_lstm_model()
model.fit(X_train, Y_train, batch_size=100, epochs=50, validation_data=(X_test, Y_test))
model.save(self.lstm_modelpath)
'''划分数据集,按一定比例划分训练集和测试集'''
def split_trainset(self):
X, Y = self.load_traindata()
split_rate = 0.8
indx = int(len(X)*split_rate)
X_train = X[:indx]
Y_train = Y[:indx]
X_test = X[indx:]
Y_test = Y[indx:]
return X_train, Y_train, X_test, Y_test
if __name__ == '__main__':
handler = QuestionClassify()
handler.train_cnn()
handler.train_lstm()