ghana-nlp-eng-twi-data-gen-nmt-older-docker.py

import pandas as pd
import numpy as np
import string
from string import digits
import matplotlib.pyplot as plt
import re
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import Input, LSTM, Embedding, Dense
from tensorflow.keras.models import Model


# Specify data to use:

# SMALLER SET:
# lines= pd.read_table('../input/dataset-twi/en-tw.txt', names=['eng', 'twi'])
# BIGGER SET:
lines= pd.read_table('../input/datasetentwi/en-twi.txt', names=['eng', 'twi'])


# If you don't truncate sequences up to a reasonable maximum, the internal dimensions of your model will be too large

# Specify maximum sequence length
MAX_SEQ_LENGTH = 128 # very important to truncate or things will get too large and bog down the system


# check dimensions
print('shape of data:')
print(lines.shape)


# Remove all numbers from text
remove_digits = str.maketrans('', '', digits)
lines.eng=lines.eng.apply(lambda x: x.translate(remove_digits))
lines.twi=lines.twi.apply(lambda x: x.translate(remove_digits))
lines.twi = lines.twi.apply(lambda x: re.sub("[ƆɔɛƐ]", "", x))


# Remove extra spaces
lines.eng=lines.eng.apply(lambda x: x.strip())
lines.twi=lines.twi.apply(lambda x: x.strip())
lines.eng=lines.eng.apply(lambda x: re.sub(" +", " ", x))
lines.twi=lines.twi.apply(lambda x: re.sub(" +", " ", x))


# truncate to MAX_SEQ_LENGTH
lines.eng=lines.eng.apply(lambda x: " ".join(x.split(' ')[:MAX_SEQ_LENGTH]))
lines.twi=lines.twi.apply(lambda x: " ".join(x.split(' ')[:MAX_SEQ_LENGTH]))


# Add start and end tokens to target sequences
lines.twi = lines.twi.apply(lambda x : 'START_ '+ x + ' _END')


# Print a random sample of the data
print("random data sample:")
print(lines.sample(10))


# Vocabulary of English
all_eng_words=set()
for eng in lines.eng:
    for word in eng.split():
        if word not in all_eng_words:
            all_eng_words.add(word)

# Vocabulary of Twi 
all_twi_words=set()
for twi in lines.twi:
    for word in twi.split():
        if word not in all_twi_words:
            all_twi_words.add(word)


# Max Length of source sequence
lenght_list=[]
for l in lines.eng:
    lenght_list.append(len(l.split(' ')))
max_length_src = np.max(lenght_list)


# Max Length of target sequence
lenght_list=[]
for l in lines.twi:
    lenght_list.append(len(l.split(' ')))
max_length_tar = np.max(lenght_list)
max_length_tar

# Missing comments...
input_words = sorted(list(all_eng_words))
target_words = sorted(list(all_twi_words))
num_encoder_tokens = len(all_eng_words)
num_decoder_tokens = len(all_twi_words)
num_encoder_tokens, num_decoder_tokens


# Missing comments...

num_decoder_tokens += 1 # For zero padding
num_decoder_tokens


# In[16]:

input_token_index = dict([(word, i+1) for i, word in enumerate(input_words)])
target_token_index = dict([(word, i+1) for i, word in enumerate(target_words)])


# In[17]:

reverse_input_char_index = dict((i, word) for word, i in input_token_index.items())
reverse_target_char_index = dict((i, word) for word, i in target_token_index.items())


# In[18]:

lines = shuffle(lines)
lines.head(10)


# In[19]:

# Train - Test Split
X, y = lines.eng, lines.twi
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)
X_train.shape, X_test.shape


# If you wanted to save the data in pickled format:

#X_train.to_pickle('X_train.pkl')
#X_test.to_pickle('X_test.pkl')


# data generator for training
def generate_batch(X = X_train, y = y_train, batch_size = 16):
    ''' Generate a batch of data '''
    while True:
        for j in range(0, len(X), batch_size):
            encoder_input_data = np.zeros((batch_size, max_length_src),dtype='float32')
            decoder_input_data = np.zeros((batch_size, max_length_tar),dtype='float32')
            decoder_target_data = np.zeros((batch_size, max_length_tar, num_decoder_tokens),dtype='float32')
            for i, (input_text, target_text) in enumerate(zip(X[j:j+batch_size], y[j:j+batch_size])):
                for t, word in enumerate(input_text.split()):
                    encoder_input_data[i, t] = input_token_index[word] # encoder input seq
                for t, word in enumerate(target_text.split()):
                    if t<len(target_text.split())-1:
                        decoder_input_data[i, t] = target_token_index[word] # decoder input seq
                    if t>0:
                        # decoder target sequence (one hot encoded)
                        # does not include the START_ token
                        # Offset by one timestep
                        decoder_target_data[i, t - 1, target_token_index[word]] = 1.
            yield([encoder_input_data, decoder_input_data], decoder_target_data)


# Dimension of latent context vector between encoder and decoder
latent_dim = 50


# Encoder
encoder_inputs = Input(shape=(None,))
enc_emb =  Embedding(num_encoder_tokens+1, latent_dim, mask_zero = True,name="state",input_length=1)(encoder_inputs)
encoder_lstm = LSTM(latent_dim, return_state=True)
encoder_outputs, state_h, state_c = encoder_lstm(enc_emb)
# We discard `encoder_outputs` and only keep the states.
encoder_states = [state_h, state_c]


# Set up the decoder, using `encoder_states` as initial state.
decoder_inputs = Input(shape=(None,))
dec_emb_layer = Embedding(num_decoder_tokens+1, latent_dim, mask_zero = True,name="dstate",input_length=1)
dec_emb = dec_emb_layer(decoder_inputs)
# We set up our decoder to return full output sequences,
# and to return internal states as well. We don't use the
# return states in the training model, but we will use them in inference.
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(dec_emb,
                                     initial_state=encoder_states)
decoder_dense = Dense(num_decoder_tokens, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)

# Define the model that will turn
# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)


# compile model
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc'])

# load a previously trained model to continue training from a checkpoint?
LOAD_MODEL = True
if LOAD_MODEL:
    model.load_weights('trained_models/TrainedModel01.h5')

# Specify training parameters
train_samples = len(X_train)
val_samples = len(X_test)
batch_size = 32 # 32 works, 64 too large on kaggle
epochs = 4 # 50 for full length run, 2 for short run to make sure things work


# Train
model.fit_generator(generator = generate_batch(X_train, y_train, batch_size = batch_size),
                    steps_per_epoch = train_samples//batch_size,
                    epochs=epochs,
                    validation_data = generate_batch(X_test, y_test, batch_size = batch_size),
                    validation_steps = val_samples//batch_size
                   )


# Always save model!!
model.save('TrainedModel.h5')


# THE FOLLOWING CODE ALLOWS YOU TO MAKE INFERENCE

# Encode the input sequence to get the "thought vectors"
encoder_model = Model(encoder_inputs, encoder_states)

# Decoder setup
# Below tensors will hold the states of the previous time step
decoder_state_input_h = Input(shape=(latent_dim,))
decoder_state_input_c = Input(shape=(latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]

dec_emb2= dec_emb_layer(decoder_inputs) # Get the embeddings of the decoder sequence

# To predict the next word in the sequence, set the initial states to the states from the previous time step
decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=decoder_states_inputs)
decoder_states2 = [state_h2, state_c2]
decoder_outputs2 = decoder_dense(decoder_outputs2) # A dense softmax layer to generate prob dist. over the target vocabulary

# Final decoder model
decoder_model = Model(
    [decoder_inputs] + decoder_states_inputs,
    [decoder_outputs2] + decoder_states2)


# Decode sequence function
def decode_sequence(input_seq):
    # Encode the input as state vectors.
    states_value = encoder_model.predict(input_seq)
    # Generate empty target sequence of length 1.
    target_seq = np.zeros((1,1))
    # Populate the first character of target sequence with the start character.
    target_seq[0, 0] = target_token_index['START_']

    # Sampling loop for a batch of sequences
    # (to simplify, here we assume a batch of size 1).
    stop_condition = False
    decoded_sentence = ''
    while not stop_condition:
        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)

        # Sample a token
        sampled_token_index = np.argmax(output_tokens[0, -1, :])
        sampled_char = reverse_target_char_index[sampled_token_index]
        decoded_sentence += ' '+sampled_char

        # Exit condition: either hit max length
        # or find stop character.
        if (sampled_char == '_END' or
           len(decoded_sentence) > 50):
            stop_condition = True

        # Update the target sequence (of length 1).
        target_seq = np.zeros((1,1))
        target_seq[0, 0] = sampled_token_index

        # Update states
        states_value = [h, c]

    return decoded_sentence


# Define function to pull sentence from Train queue
train_gen = generate_batch(X_train, y_train, batch_size = 1)
k=-1


# Evaluate on first sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on first sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 2nd sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 3rd sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 4th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 5th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 6th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 7th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 8th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 9th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])

# Evaluate on 10th sentence
k+=1
(input_seq, actual_output), _ = next(train_gen)
decoded_sentence = decode_sequence(input_seq)
print('Input English sentence:', X_train[k:k+1].values[0])
print('Actual Twi Translation:', y_train[k:k+1].values[0][6:-4])
print('Predicted Twi Translation:', decoded_sentence[:-4])