-
Notifications
You must be signed in to change notification settings - Fork 1
/
manage.py
126 lines (81 loc) · 2.97 KB
/
manage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
import numpy as np
np.random.seed(0)
import glob
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score
negative_files = glob.glob('Negative/*.txt')
positive_files = glob.glob('Positive/*.txt')
def clean_text(text):
from re import sub
text = sub('[^ةجحخهعغفقثصضشسيىبلاآتنمكوؤرإأزدءذئطظ]', ' ', text)
text = sub(' +', ' ', text)
text = sub('[آإأ]', 'ا', text)
text = sub('ة', 'ه', text)
return text
negative_texts = []
positive_texts = []
for file in positive_files:
with open(file, 'r', encoding='utf-8') as file_to_read:
try:
text = file_to_read.read()
text = clean_text(text)
if text == "":
continue
print(text)
positive_texts.append(text)
print("-" * 10)
except UnicodeDecodeError:
continue
for file in negative_files:
with open(file, 'r', encoding='utf-8') as file_to_read:
try:
text = file_to_read.read()
text = clean_text(text)
if text == "":
continue
print(text)
negative_texts.append(text)
print("-" * 10)
except UnicodeDecodeError:
continue
print('count positive texts : ',len(positive_texts))
print('count negative texts : ',len(negative_texts))
positive_labels = [1]*len(positive_texts)
negative_labels = [0]*len(negative_texts)
all_texts = positive_texts + negative_texts
all_labels = positive_labels + negative_labels
print(len(all_labels) == len(all_texts))
all_texts, all_labels = shuffle(all_texts, all_labels)
x_train, x_test, y_train, y_test = train_test_split(all_texts, all_labels, test_size=0.20)
vectorizer = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
vectorizer.fit(x_train)
x_train = vectorizer.transform(x_train)
model = LinearSVC()
model.fit(x_train, y_train)
x_test = vectorizer.transform(x_test)
predictions = model.predict(x_test)
print("Acc svm")
print(accuracy_score(y_test, predictions))
model = MultinomialNB()
model.fit(x_train, y_train)
predictions = model.predict(x_test)
print("Acc naive_bayes")
print(accuracy_score(y_test, predictions))
import pickle
with open('model.pickle', 'wb') as file:
pickle.dump(model, file)
with open('vectorizer.pickle', 'wb') as file:
pickle.dump(vectorizer, file)
with open('model.pickle', 'rb') as file:
model = pickle.load(file)
with open('vectorizer.pickle', 'rb') as file:
vectorizer = pickle.load(file)
example_test = 'أنا سعيد جدا، كانت الرحلة رائعة'
cleaned_example_test = clean_text(example_test)
example_test_vector = vectorizer.transform([cleaned_example_test])
example_result = model.predict(example_test_vector)
print(example_result[0])