Privacy_Disclosure_Nuhil.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Sure the Current Python Environment is Right One :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/nuhil/anaconda3/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom Data Crawler from http://www.medhelp.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import requests\n",
    "# from bs4 import BeautifulSoup\n",
    "# from os import listdir\n",
    "# import sys\n",
    "\n",
    "# def trade_spider(url):\n",
    "#     source_code = requests.get(url)\n",
    "#     plain_text = source_code.text\n",
    "#     soup = BeautifulSoup(plain_text, \"html.parser\")\n",
    "\n",
    "#     content = soup.find('div', {'class': 'post_message'})\n",
    "#     content = content.text if content else \"\"\n",
    "#     return content\n",
    "\n",
    "# # load doc into memory\n",
    "# def load_doc(filename):\n",
    "#     file_to_work = open(filename, \"r\")\n",
    "\n",
    "#     urls = []\n",
    "#     for my_line in file_to_work:\n",
    "#         urls.append(my_line)\n",
    "\n",
    "#     file_to_work.close()\n",
    "#     return urls    \n",
    "\n",
    "# urls = load_doc(\"Data_Sources/Medical_Post_URLs/Divorce--Breakups.txt\")\n",
    "\n",
    "# i = 1180\n",
    "# for url in urls:\n",
    "#     if i == 2181:\n",
    "#         break\n",
    "        \n",
    "#     post = trade_spider(url)\n",
    "#     if len(post) < 1000:\n",
    "#         continue\n",
    "        \n",
    "#     with open(\"Disclose_Nondisclose/private/\"+str(i).zfill(5)+\"_divorce-breakups.txt\", \"w\") as f:\n",
    "#         f.write(post.strip())\n",
    "        \n",
    "#     i += 1 \n",
    "#     print(i)\n",
    "    \n",
    "# print(\"Done!!!\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wiki Article Processor Using https://github.com/attardi/wikiextractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import json\n",
    "# import pandas as pd\n",
    "# import numpy as np\n",
    "# import re\n",
    "\n",
    "# file = open(\"AA/wiki_XX\", encoding = \"utf8\") \n",
    "# data = file.readlines()\n",
    "# wiki = pd.DataFrame()\n",
    "\n",
    "# i = 0\n",
    "# for js in data:\n",
    "#     wikidata = json.loads(js)\n",
    "    \n",
    "#     with open(str(i).zfill(5)+\"_\"+re.sub(\"\\W+\",\"-\", wikidata['title'].strip().lower())+\".txt\", \"w\") as f:\n",
    "#         f.write(wikidata['text'].strip())\n",
    "\n",
    "#     i += 1     \n",
    "    \n",
    "# print(\"Done!!!\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Randomize File Contents in a Directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from os import listdir\n",
    "\n",
    "# # load doc into memory\n",
    "# def load_doc(filename):\n",
    "#     # open the file as read only\n",
    "#     file = open(filename, 'r')\n",
    "#     # read all text\n",
    "#     text = file.read()\n",
    "#     # close the file\n",
    "#     file.close()\n",
    "#     return text\n",
    "\n",
    "# # specify directory to load\n",
    "# directory = 'private'\n",
    "# # walk through all files in the folder\n",
    "# i = 0\n",
    "# for filename in listdir(directory):\n",
    "#     # skip files that do not have the right extension\n",
    "#     if not filename.endswith(\".txt\"):\n",
    "#         continue\n",
    "        \n",
    "#     # create the full path of the file to open\n",
    "#     path = directory + '/' + filename\n",
    "#     # load document\n",
    "#     doc = load_doc(path)\n",
    "#     with open(\"private_r/\"+str(i).zfill(5)+\"_\"+filename.split(\"_\")[1], \"w\") as f:\n",
    "#         f.write(doc.strip())\n",
    "    \n",
    "#     i += 1\n",
    "        \n",
    "# print('Done!!!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Some Global Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split files at:  4499\n",
      "Train Until: 0 - 4499\n",
      "Test From:  4500 - 4999\n"
     ]
    }
   ],
   "source": [
    "number_of_data_in_each_class = 5000\n",
    "\n",
    "# Remaining is the test dataset\n",
    "train_with_percentage = 90 \n",
    "\n",
    "split_file_at = int(number_of_data_in_each_class*(train_with_percentage/100))-1\n",
    "train_untill = split_file_at\n",
    "test_from = train_untill+1\n",
    "\n",
    "print(\"Split files at: \", split_file_at)\n",
    "print(\"Train Until: 0 -\", train_untill)\n",
    "print(\"Test From: \", test_from, \"-\", number_of_data_in_each_class-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "from os import listdir\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load doc's content into memory by it's name**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_doc(filename):\n",
    "    \"\"\"\n",
    "    Load a doc into memory by its name\n",
    "    \"\"\"\n",
    "    # open the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Turn a given doc into clean tokens**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_doc(doc):\n",
    "    \"\"\"\n",
    "    Turn a doc into clean tokens\n",
    "    \"\"\"\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', punctuation)\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    tokens = [word for word in tokens if word.isalpha()]\n",
    "    # filter out stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    tokens = [w for w in tokens if not w in stop_words]\n",
    "    # filter out short tokens\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load a doc and add to vocab\n",
    "def add_doc_to_vocab(filename, vocab):\n",
    "    # load doc\n",
    "    doc = load_doc(filename)\n",
    "    # clean doc\n",
    "    tokens = clean_doc(doc)\n",
    "    # update counts\n",
    "    vocab.update(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Process each document inside a directory and add to `vocab` counter**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load all docs in a directory\n",
    "def process_docs(directory, vocab, is_trian):\n",
    "    # walk through all files in the folder\n",
    "    for filename in listdir(directory):\n",
    "        if not filename.endswith(\".txt\"):\n",
    "            continue\n",
    "\n",
    "        # skip any posts in the test set\n",
    "        # skip any posts in the test set\n",
    "        file_number = int(filename[:5])\n",
    "        if is_trian and file_number > train_untill:\n",
    "            continue\n",
    "                            \n",
    "        if not is_trian and file_number < test_from:\n",
    "            continue               \n",
    "            \n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # add doc to vocab\n",
    "        add_doc_to_vocab(path, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Process all documents to add everything into `vocab` Counter**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary length:  108591\n",
      "Top 50 vocabulary with count:  [('The', 29597), ('like', 12249), ('people', 11260), ('also', 10782), ('He', 10096), ('It', 9679), ('In', 9620), ('one', 9359), ('time', 9208), ('would', 8477), ('get', 7993), ('Im', 7844), ('dont', 7475), ('years', 7414), ('called', 7348), ('know', 6999), ('feel', 6941), ('This', 6759), ('many', 6654), ('first', 6011), ('said', 5794), ('used', 5647), ('made', 5608), ('They', 5462), ('things', 5191), ('want', 5173), ('even', 5165), ('two', 5146), ('go', 4974), ('back', 4896), ('make', 4716), ('could', 4607), ('year', 4576), ('life', 4570), ('really', 4525), ('much', 4485), ('help', 4411), ('think', 4409), ('never', 4284), ('day', 4278), ('My', 4257), ('She', 4120), ('started', 4042), ('still', 3952), ('way', 3866), ('going', 3847), ('told', 3786), ('different', 3764), ('work', 3660), ('There', 3579)]\n"
     ]
    }
   ],
   "source": [
    "# define vocab\n",
    "vocab = Counter()\n",
    "# add all docs to vocab\n",
    "process_docs('Data/Disclose_Nondisclose/public', vocab, True)\n",
    "process_docs('Data/Disclose_Nondisclose/private', vocab, True)\n",
    "\n",
    "# print the size of the vocab\n",
    "print(\"Vocabulary length: \", len(vocab))\n",
    "# print the top words in the vocab\n",
    "print(\"Top 50 vocabulary with count: \", vocab.most_common(50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Remove tokens that appeared less than twice**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens that appeared more than twice:  53366\n"
     ]
    }
   ],
   "source": [
    "# keep tokens with a min occurrence\n",
    "min_occurane = 2\n",
    "tokens = [k for k,c in vocab.items() if c >= min_occurane]\n",
    "print(\"Tokens that appeared more than twice: \", len(tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save final vocab/tokens to a file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_list(lines, filename):\n",
    "    # convert lines to a single blob of text\n",
    "    data = '\\n'.join(lines)\n",
    "    # open file\n",
    "    file = open(filename, 'w')\n",
    "    # write text\n",
    "    file.write(data)\n",
    "    # close file\n",
    "    file.close()\n",
    "\n",
    "# save tokens to a vocabulary file\n",
    "save_list(tokens, 'Data/Parsed_Vocab/vocab.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN with Trained Word Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/nuhil/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from string import punctuation\n",
    "from os import listdir\n",
    "from numpy import array\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Embedding\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "\n",
    "from keras.layers import Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load doc into memory\n",
    "def load_doc(filename):\n",
    "    # open the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Similar to the above `clean_doc` function but it filters out tokens that are not in `vocab`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc(doc, vocab):\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', punctuation)\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    \n",
    "    # filter out tokens not in vocab\n",
    "    tokens = [w for w in tokens if w in vocab]\n",
    "    tokens = ' '.join(tokens)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Similar to the above `process_docs` function but it does not save tokens to a file, rather keeps in memory named `documents`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load all docs in a directory\n",
    "def process_docs(directory, vocab, is_trian):\n",
    "    documents = list()\n",
    "    # walk through all files in the folder\n",
    "    for filename in listdir(directory):\n",
    "        if not filename.endswith(\".txt\"):\n",
    "            continue        \n",
    "            \n",
    "        # skip any posts in the test set\n",
    "        file_number = int(filename[:5])\n",
    "        if is_trian and file_number > train_untill:\n",
    "            continue\n",
    "                            \n",
    "        if not is_trian and file_number < test_from:\n",
    "            continue                        \n",
    "            \n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # load the doc\n",
    "        doc = load_doc(path)\n",
    "        # clean doc\n",
    "        tokens = clean_doc(doc, vocab)\n",
    "        \n",
    "        # add to list\n",
    "        documents.append(tokens)\n",
    "    return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total vocabulary:  53366\n"
     ]
    }
   ],
   "source": [
    "# load the vocabulary\n",
    "vocab_filename = 'Data/Parsed_Vocab/vocab.txt'\n",
    "vocab = load_doc(vocab_filename)\n",
    "vocab = vocab.split()\n",
    "vocab = set(vocab)\n",
    "print(\"Total vocabulary: \", len(vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load all training posts**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Training docs:  9000\n"
     ]
    }
   ],
   "source": [
    "positive_docs = process_docs('Data/Disclose_Nondisclose/private', vocab, True)\n",
    "negative_docs = process_docs('Data/Disclose_Nondisclose/public', vocab, True)\n",
    "train_docs = negative_docs + positive_docs\n",
    "\n",
    "# print(type(train_docs))\n",
    "print(\"Total number of Training docs: \", len(train_docs))\n",
    "\n",
    "# create the tokenizer\n",
    "tokenizer = Tokenizer()\n",
    "# fit the tokenizer on the documents\n",
    "tokenizer.fit_on_texts(train_docs)\n",
    "\n",
    "# sequence encode\n",
    "encoded_docs = tokenizer.texts_to_sequences(train_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Padding to keep all docs in a certain size. Also prepare training data set and associated labels as Array**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Training labels:  9000\n"
     ]
    }
   ],
   "source": [
    "# pad sequences\n",
    "# get the biggest post as per its contents\n",
    "max_length = max([len(s.split()) for s in train_docs])\n",
    "\n",
    "# define training data\n",
    "Xtrain = pad_sequences(encoded_docs, maxlen=max_length, padding='post')\n",
    "\n",
    "# define training labels\n",
    "# put 0s for the first <split_file_at> entries and 1s for last <split_file_at> entries.\n",
    "# Because, in the <train_docs> list we have the public docs first\n",
    "# and private docs later.\n",
    "# From now on we are assuming 0 for public data, 1 for private data\n",
    "ytrain = array([0 for _ in range((train_untill+1))] + [1 for _ in range((train_untill+1))])\n",
    "print(\"Total number of Training labels: \", len(ytrain))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Padding to keep all docs in a certain size. Also prepare test data set and associated labels as Array**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Test docs:  1000\n",
      "Total number of Test labels:  1000\n"
     ]
    }
   ],
   "source": [
    "# load all test posts\n",
    "positive_docs = process_docs('Data/Disclose_Nondisclose/private', vocab, False)\n",
    "negative_docs = process_docs('Data/Disclose_Nondisclose/public', vocab, False)\n",
    "test_docs = negative_docs + positive_docs\n",
    "print(\"Total number of Test docs: \", len(test_docs))\n",
    "\n",
    "# sequence encode\n",
    "encoded_docs = tokenizer.texts_to_sequences(test_docs)\n",
    "# pad sequences\n",
    "Xtest = pad_sequences(encoded_docs, maxlen=max_length, padding='post')\n",
    "# define test labels\n",
    "ytest = array([0 for _ in range((number_of_data_in_each_class - test_from))] + [1 for _ in range((number_of_data_in_each_class - test_from))])\n",
    "print(\"Total number of Test labels: \", len(ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size (largest integer value):  44394\n"
     ]
    }
   ],
   "source": [
    "# define vocabulary size (largest integer value)\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "print(\"Vocabulary size (largest integer value): \", vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the final ANN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 3589, 100)         4439400   \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 3582, 32)          25632     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 1791, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 57312)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                573130    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 11        \n",
      "=================================================================\n",
      "Total params: 5,038,173\n",
      "Trainable params: 5,038,173\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size, 100, input_length=max_length))\n",
    "model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='relu'))\n",
    "\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Compile the ANN and Evaluate on top of test data set**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " - 161s - loss: 0.0916 - acc: 0.9674\n",
      "Epoch 2/10\n",
      " - 161s - loss: 0.0075 - acc: 0.9994\n",
      "Epoch 3/10\n",
      " - 156s - loss: 0.0049 - acc: 0.9996\n",
      "Epoch 4/10\n",
      " - 155s - loss: 0.0052 - acc: 0.9997\n",
      "Epoch 5/10\n",
      " - 155s - loss: 0.0053 - acc: 0.9999\n",
      "Epoch 6/10\n",
      " - 156s - loss: 0.0043 - acc: 0.9998\n",
      "Epoch 7/10\n",
      " - 157s - loss: 0.0049 - acc: 0.9998\n",
      "Epoch 8/10\n",
      " - 158s - loss: 0.0042 - acc: 0.9999\n",
      "Epoch 9/10\n",
      " - 158s - loss: 0.0044 - acc: 0.9997\n",
      "Epoch 10/10\n",
      " - 158s - loss: 0.0037 - acc: 0.9999\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3d94049278>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# fit network\n",
    "model.fit(Xtrain, ytrain, epochs=10, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 99.700000\n"
     ]
    }
   ],
   "source": [
    "# evaluate\n",
    "loss, acc = model.evaluate(Xtest, ytest, verbose=0)\n",
    "print('Test Accuracy: %f' % (acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc(doc):\n",
    "    \"\"\"\n",
    "    Turn a doc into clean tokens\n",
    "    \"\"\"\n",
    "\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', punctuation)\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    tokens = [word for word in tokens if word.isalpha()]\n",
    "    # filter out stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    tokens = [w for w in tokens if not w in stop_words]\n",
    "    # filter out short tokens\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    tokens = ' '.join(tokens)\n",
    "    return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_new_doc(path):\n",
    "    \"\"\"\n",
    "    Process a single doc with post\n",
    "    \"\"\"\n",
    "    # define a new list\n",
    "    documents = list()\n",
    "    \n",
    "    # load doc\n",
    "    doc = load_doc(path)\n",
    "    # clean doc\n",
    "    tokens = clean_doc(doc)\n",
    "    # add to list\n",
    "    documents.append(tokens)\n",
    "    \n",
    "    return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict_privacy(path, max_length, tokenizer, model):\n",
    "    \"\"\"\n",
    "    Give a new unseen doc to predict it's privacy\n",
    "    \"\"\"\n",
    "    new_doc = process_new_doc(path)\n",
    "    # sequence encode\n",
    "    # To Do:\n",
    "    # 1. Will check the current <tokentizer> in memory\n",
    "    encoded_doc = tokenizer.texts_to_sequences(new_doc)\n",
    "    \n",
    "    # pad sequences\n",
    "    max_length = max_length\n",
    "    Xpredict = pad_sequences(encoded_doc, maxlen=max_length, padding='post')\n",
    "    \n",
    "    # prediction\n",
    "    ypredict = model.predict(Xpredict, verbose=1)\n",
    "    print(\"Privacy Score: {0} \\nRounded To: {1}\".format(ypredict, round(ypredict[0,0])))\n",
    "    \n",
    "    return round(ypredict[0,0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict_privacy_by_text(text, max_length, tokenizer, model):\n",
    "    # define a new list\n",
    "    documents = list()\n",
    "    # clean doc\n",
    "    tokens = clean_doc(text)\n",
    "    # add to list\n",
    "    documents.append(tokens)\n",
    "    \n",
    "    # sequence encode\n",
    "    # To Do:\n",
    "    # 1. Will check the current <tokentizer> in memory\n",
    "    encoded_doc = tokenizer.texts_to_sequences(documents)\n",
    "    \n",
    "    # pad sequences\n",
    "    max_length = max_length\n",
    "    Xpredict = pad_sequences(encoded_doc, maxlen=max_length, padding='post')\n",
    "    \n",
    "    # prediction\n",
    "    ypredict = model.predict(Xpredict, verbose=0)\n",
    "    print(\"Privacy Score: {0} \\nRounded To: {1}\".format(ypredict, round(ypredict[0,0])))\n",
    "    \n",
    "    return round(ypredict[0,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Give any custom text file to predict the privacy of its content as a whole**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 33ms/step\n",
      "Privacy Score: [[  2.78260650e-05]] \n",
      "Rounded To: 0.0\n",
      "\n",
      "Content is Public\n"
     ]
    }
   ],
   "source": [
    "import nltk.data\n",
    "sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "\n",
    "new_doc = 'Data/Disclose_Nondisclose/predict/my-content.txt'\n",
    "print(\"\\nContent is Public\" if predict_privacy(new_doc, max_length, tokenizer, model) == 0.0 else \"\\n\\x1b[31mContent is Private\\x1b[0m\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Evaluate each sentence separately**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The son of a landowner from Tanabe, Ueshiba studied a number of martial arts in his youth, and served in the Japanese Army during the Russo-Japanese War.\n",
      "\n",
      "Privacy Score: [[ 0.1541281]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "After being discharged in 1907, he moved to Hokkaidō as the head of a pioneer settlement; here he met and studied with Takeda Sōkaku, the founder of Daitō-ryū Aiki-jūjutsu.\n",
      "\n",
      "Privacy Score: [[ 0.46368957]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "On leaving Hokkaido in 1919, Ueshiba joined the Ōmoto-kyō movement, a Shinto sect, in Ayabe, where he served as a martial arts instructor and opened his first dojo.\n",
      "\n",
      "Privacy Score: [[ 0.32375118]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "He accompanied the head of the Ōmoto-kyō group, Onisaburo Deguchi, on an expedition to Mongolia in 1924, where they were captured by Chinese troops and returned to Japan.\n",
      "\n",
      "Privacy Score: [[ 0.16943543]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "The following year, he had a profound spiritual experience, stating that, \"a golden spirit sprang up from the ground, veiled my body, and changed my body into a golden one.\"\n",
      "\n",
      "Privacy Score: [[ 0.03768792]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "After this experience, his martial arts skill appeared to be greatly increased.\n",
      "\n",
      "Privacy Score: [[ 0.49919331]] \n",
      "Rounded To: 0.0\n",
      "Sentence is Public\n",
      "--------------------------------------------------\n",
      "I had a girl friend and now I am in depression.\n",
      "\n",
      "Privacy Score: [[ 0.62810796]] \n",
      "Rounded To: 1.0\n",
      "\u001b[31mSentence is Private\u001b[0m\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "with open(new_doc) as f:\n",
    "    text = f.read()\n",
    "    sentences = sent_detector.tokenize(text.strip())\n",
    "        \n",
    "for sentence in sentences:\n",
    "    print(sentence+\"\\n\")\n",
    "    print(\"Sentence is Public\" if predict_privacy_by_text(sentence, max_length, tokenizer, model) == 0.0 else \"\\x1b[31mSentence is Private\\x1b[0m\")\n",
    "    print(\"-\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}