Final proj_IMDB.py

# %%
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Embedding, LSTM, Dense, SpatialDropout1D
from keras.callbacks import EarlyStopping
import glob
import re

# %%
# Load the IMDb dataset

file_paths_pos = glob.glob('./aclImdb/train/pos/*.txt')
file_paths_neg = glob.glob('./aclImdb/train/neg/*.txt')
file_paths_test_pos = glob.glob('./aclImdb/test/pos/*.txt')
file_paths_test_neg = glob.glob('./aclImdb/test/neg/*.txt')

trainPos = []
trainNeg = []
testPos = []
testNeg = []

for path in file_paths_pos:
    with open(path, 'r') as file:
        content = file.read()
        trainPos.append(content)

for path in file_paths_neg:
    with open(path, 'r') as file:
        content = file.read()
        trainNeg.append(content)

for path in file_paths_test_pos:
    with open(path, 'r') as file:
        content = file.read()
        testPos.append(content)

for path in file_paths_test_neg:
    with open(path, 'r') as file:
        content = file.read()
        testNeg.append(content)

trainPos = pd.DataFrame(trainPos, columns=['review'])
trainNeg = pd.DataFrame(trainNeg, columns=['review'])
testPos = pd.DataFrame(testPos, columns=['review'])
testNeg = pd.DataFrame(testNeg, columns=['review'])

trainPos['sentiment'] = 'positive'
trainNeg['sentiment'] = 'negative'
testPos['sentiment'] = 'positive'
testNeg['sentiment'] = 'negative'

imdb_data = pd.concat([trainPos, trainNeg, testPos, testNeg], ignore_index=True)
imdb_data

# %%
imdb_data.shape

# %% [markdown]
# # Preprocessing

# %%
# Cleaning the reviews

import string

def clean_text(text):
    text = text.lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub('[''"",,,]', '', text)
    text = re.sub('\n', '', text)
    return text

imdb_data['review'] = imdb_data['review'].apply(clean_text)
imdb_data.review

# %%
# Tokenize and pad sequences
max_features = 5000
tokenizer = Tokenizer(num_words=max_features, split=' ')
tokenizer.fit_on_texts(imdb_data['review'].values)
X = tokenizer.texts_to_sequences(imdb_data['review'].values)
X = pad_sequences(X)

# %%
# Encode the target labels
label_encoder = LabelEncoder()
Y = label_encoder.fit_transform(imdb_data['sentiment'].values)

# %%
# Split the data into training and testing sets
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.5, random_state=42)

# %%
# LSTM

# %%
# Build the model

embed_dim = 32
lstm_out = 32

model = Sequential()
model.add(Embedding(max_features, embed_dim, input_length=X.shape[1]))
model.add(SpatialDropout1D(0.4))
model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())

# Train the model
batch_size = 128
epochs = 3

train_history = model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=epochs, batch_size=batch_size)

# Evaluate the model
scores = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])


# %%
import matplotlib.pyplot as plt

# Plot training and validation loss
plt.plot(train_history.history['loss'], label='Training Loss')
plt.plot(train_history.history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.title('Training and Validation Loss')
plt.show()

# Plot training and validation accuracy
plt.plot(train_history.history['accuracy'], label='Training Accuracy')
plt.plot(train_history.history['val_accuracy'], label='Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.title('Training and Validation Accuracy')
plt.show()