Edit preprocessing

paulshealy1 · Sep 12, 2017 · 12c186e · 12c186e
1 parent 43f7be7
commit 12c186e
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/lstm_word2vec.ipynb b/lstm_word2vec.ipynb
@@ -200,8 +200,8 @@
    ],
    "source": [
     "print('Number of reviews by class in training set')\n",
-    "print(y_train.sum(axis=0))\n",
-    "n_classes = y_train.shape[1]"
+    "print(labels.sum(axis=0))\n",
+    "n_classes = labels.shape[1]"
    ]
   },
   {
@@ -232,7 +232,9 @@
     "\n",
     "for doc in train_data[TEXT_COL]:\n",
     "    sentences = nltk.tokenize.sent_tokenize(doc)\n",
-    "    sent_lst.extend(sentences)"
+    "    for sent in sentences:\n",
+    "        word_lst = [w for w in nltk.tokenize.word_tokenize(sent) if w.isalnum()]\n",
+    "        sent_lst.append(word_lst)"
    ]
   },
   {