Streamlit Deployment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wuTkdpQuI73-"
   },
   "source": [
    "## Streamlit Web Deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gBV4xa4hl7CB"
   },
   "outputs": [],
   "source": [
    "!pip install -q tf-models-official==2.3.0\n",
    "!pip install streamlit\n",
    "!pip install pyngrok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "t7c5ErqJH7Kj"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Gqisxpv28lJ1"
   },
   "outputs": [],
   "source": [
    "%%writefile utilss.py\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "import re\n",
    "import string\n",
    "import random\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "from nltk.tag import pos_tag\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "import tweepy\n",
    "consumerKey = \"VEyxpXLGHG9USYhM7spHVKl36\"\n",
    "consumerSecret = \"FG61nlBuLR7mb6UCPGxHH4UdMqwYNwL6aFhDt9gQJcaChblOkL\"\n",
    "accessToken = \"1142865475459846145-5VQ9CRY7iRlneurWdNzwHmT4Y9k6L1\"\n",
    "accessTokenSecret = \"iAkL3XWrsBdBQWn2eH8ifqnjoWkvBF5EHvJ1SsH6EcfLB\"\n",
    "\n",
    "authenticate = tweepy.OAuthHandler(consumerKey, consumerSecret) \n",
    "authenticate.set_access_token(accessToken, accessTokenSecret) \n",
    "api = tweepy.API(authenticate, wait_on_rate_limit = True)\n",
    "from official.modeling import tf_utils\n",
    "from official import nlp\n",
    "from official.nlp import bert\n",
    "\n",
    "# Load the required submodules\n",
    "import official.nlp.bert.bert_models\n",
    "import official.nlp.bert.configs\n",
    "import official.nlp.bert.tokenization as tokenization\n",
    "import PIL\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import io\n",
    "import tensorflow_hub as hub\n",
    "\n",
    "from keras.layers import Input, Dropout, Dense, Activation\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'\n",
    "bert_layer = hub.KerasLayer(module_url, trainable=True)\n",
    "\n",
    "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
    "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
    "tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import streamlit as st\n",
    "\n",
    "def bert_encode(texts, tokenizer, max_len=512):\n",
    "    all_tokens = []\n",
    "    all_masks = []\n",
    "    all_segments = []\n",
    "    \n",
    "    for text in texts:\n",
    "        text = tokenizer.tokenize(text)\n",
    "            \n",
    "        text = text[:max_len-2]\n",
    "        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n",
    "        pad_len = max_len - len(input_sequence)\n",
    "        \n",
    "        tokens = tokenizer.convert_tokens_to_ids(input_sequence) + [0] * pad_len\n",
    "        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n",
    "        segment_ids = [0] * max_len\n",
    "        \n",
    "        all_tokens.append(tokens)\n",
    "        all_masks.append(pad_masks)\n",
    "        all_segments.append(segment_ids)\n",
    "    \n",
    "    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)\n",
    "\n",
    "model_lonely = keras.models.load_model('/content/drive/MyDrive/Utrack_Models/Utrack_Lonely')\n",
    "\n",
    "def Show_Recent_Tweets(raw_text):\n",
    "  posts = api.user_timeline(screen_name=raw_text, count = 100, lang =\"en\", tweet_mode=\"extended\")   \n",
    "  def get_tweets():\n",
    "    column_names = ['tweet', 'time']\n",
    "    user = pd.DataFrame(columns =column_names)\n",
    "    \n",
    "    tweet_time = []\n",
    "    tweet_text = []\n",
    "    for info in posts[:100]:\n",
    "      tweet_time.append(info.created_at)\n",
    "      tweet_text.append(info.full_text)\n",
    "    \n",
    "    user['time'] = tweet_time\n",
    "    user['tweet'] = tweet_text\n",
    "    \n",
    "    return user\n",
    " \n",
    "  recent_tweets=get_tweets()        \n",
    "  return recent_tweets\n",
    " \n",
    "def tokenize_tweets(clown) :\n",
    "    tweets = clown.tweet.tolist()\n",
    "    tokenizer = WordPunctTokenizer() \n",
    "    cleaned = []\n",
    "    for i in range(0, len(tweets)):\n",
    "        text = tweets[i]\n",
    "        text = re.sub('^https?://.*[rn]*','', text)\n",
    "        text = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', text)\n",
    "        text = re.sub(\"(@[A-Za-z0-9_]+)\",\"\", text)\n",
    "        text = re.sub(\"([^\\w\\s])\", \"\", text)\n",
    "        text = re.sub(\"^RT\", \"\", text)\n",
    "        text = tokenizer.tokenize(text)\n",
    "        element = [text]\n",
    "        cleaned.append(element)\n",
    "    return cleaned\n",
    "\n",
    "def lemmatize_sentence(tweet_tokens, stop_words = ()):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    cleaned_tokens = []\n",
    "    for token, tag in pos_tag(tweet_tokens):\n",
    "        if tag.startswith('NN'):\n",
    "            pos = 'n'\n",
    "        elif tag.startswith('V'):\n",
    "            pos = 'v'\n",
    "        else:\n",
    "            pos = 'a'\n",
    "        token = lemmatizer.lemmatize(token, pos)\n",
    "        if len(token) > 0 and token not in string.punctuation and token.lower() not in stop_words:\n",
    "            cleaned_tokens.append(token.lower())\n",
    "    return cleaned_tokens\n",
    "\n",
    "def create_lemmatized_sent(words):\n",
    "    cleaned = []\n",
    "    stop_words = stopwords.words('english')\n",
    "    for i in range(0, len(words)):\n",
    "        sent = lemmatize_sentence(words[i][0], stop_words)\n",
    "        if len(sent) >= 0:\n",
    "            element = [sent]\n",
    "            cleaned.append(element)\n",
    "    return cleaned\n",
    "\n",
    "def remove_emoji(string):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', string)\n",
    "\n",
    "def write_sent(clown, sent):\n",
    "    cleaned = []\n",
    "    for i in sent:\n",
    "        s = \"\"\n",
    "        for j in i[0]:\n",
    "            j = str(j)\n",
    "            j = j + \" \"\n",
    "            s = s + j\n",
    "        s = remove_emoji(s)\n",
    "        element = [s]\n",
    "        cleaned.append(element)\n",
    "    df = pd.DataFrame(cleaned,columns = ['text'])\n",
    "    df1 = clown\n",
    "    df1 = df1['time']\n",
    "    big = pd.concat([df, df1], axis = 1)\n",
    "    return big\n",
    "  \n",
    "def import_and_predict(df, model):\n",
    "  max_len = 150\n",
    "  \n",
    "  test_input = bert_encode(df[\"text\"].values, tokenizer, max_len=max_len)\n",
    "  prediction = model.predict(test_input)\n",
    "\n",
    "  return prediction\n",
    "\n",
    "\n",
    "def output_dataframe(df, prediction):\n",
    "  df2 = pd.concat([df, prediction], axis = 1)\n",
    "  return df2\n",
    "\n",
    "def visualisation(n, file):\n",
    "  #constructing data\n",
    "  df = file\n",
    "  # df.drop(['Unnamed: 0','Unnamed: 0.1','sigmoid_predictions', 'normalised_predictions'], axis=1, inplace=True)\n",
    "  jscolumn = df.predictions\n",
    "\n",
    "  df['final'] = jscolumn\n",
    "  df['perc'] = 100*(df.final)\n",
    "  new_df = df.drop(columns = ['predictions','final'])\n",
    "  plt.figure(figsize=(40,15))\n",
    "  n = int(input())\n",
    "  temp_df = new_df[:n]\n",
    "  final_df  = temp_df.iloc[::-1]\n",
    "  sns.lineplot(x='time', y='perc', data=final_df, linewidth=7, color = 'red')\n",
    "  plt.title(\"Mental State vs Date\", fontsize= 40,fontweight='bold')\n",
    "  # plt.xticks(rotation=90)\n",
    "  sns.set_style('white')\n",
    "\n",
    "  plt.xlabel('Month',fontsize=30,fontweight='bold')\n",
    "  plt.xticks(fontsize=20,rotation=90)\n",
    "  plt.ylabel('Percentage',fontsize=30,fontweight='bold')\n",
    "  plt.yticks(fontsize=25)\n",
    "  plt.grid(axis='y', alpha=0.5)\n",
    "\n",
    "  st.pyplot()\n",
    "\n",
    "  monthdict = {\"January\":1, \"February\":2, \"March\":3, \"April\":4, \"May\":5, \"June\":6, \"July\":7, \"August\":8,\n",
    "                      \"September\":9, \"October\":10, \"November\":11, \"December\":12}\n",
    "  values= []\n",
    "  for month in df.months.unique():\n",
    "    dftempo = df[pd.to_datetime(df['time']).dt.month == monthdict[month]]\n",
    "    values.append(jsmean(dftempo.final))\n",
    "      \n",
    "  plt.figure(figsize=(15,10))\n",
    "  x= df.months.unique()\n",
    "  height = 100*np.array(values)\n",
    "  plt.bar(x, height, width=0.5, bottom=None, align='center', color=['#78C850',  # Grass\n",
    "                      '#f20a53',  # Fire\n",
    "                      '#6890F0',  # Water\n",
    "                      '#A8B820',  # Bug\n",
    "                      '#A8A878',  # Normal\n",
    "                      '#A040A0',  # Poison\n",
    "                      '#F8D030',  # Electric\n",
    "                      '#E0C068',  # Ground\n",
    "                      '#EE99AC',  # Fairy\n",
    "                      '#C03028',  # Fighting\n",
    "                      '#6cf5d3',                                                               \n",
    "                      '#561191'\n",
    "                    ])\n",
    "\n",
    "\n",
    "  sns.set_style('white')\n",
    "\n",
    "\n",
    "  plt.xlabel('Month',fontsize=15,fontweight='bold')\n",
    "  plt.xticks(fontsize=15,rotation=45)\n",
    "  plt.ylabel('Percentage',fontsize=15,fontweight='bold')\n",
    "  plt.yticks(fontsize=15)\n",
    "  plt.grid(axis='y', alpha=0.5)\n",
    "  plt.title('Average Percentage across months', fontsize=20)\n",
    "\n",
    "  st.pyplot()\n",
    "\n",
    "  df['months'] = df['time'].dt.month_name()\n",
    "  plt.figure(figsize=(10,5))\n",
    "  sns.set_style('white')\n",
    "  sns.swarmplot(x='months', y='perc', data=df.iloc[::-1])\n",
    "  #plt.xticks(rotation=90);\n",
    "  plt.xlabel('Month',fontsize=15,fontweight='bold')\n",
    "  plt.xticks(fontsize=15,rotation=0)\n",
    "  plt.ylabel('Percentage',fontsize=15,fontweight='bold')\n",
    "  plt.yticks(fontsize=15)\n",
    "  plt.title('Percentage across months', fontsize=20)\n",
    "  plt.grid(axis='y', alpha=0.5)\n",
    "\n",
    "  st.pyplot()\n",
    "\n",
    "  plt.figure(figsize=(10,6))\n",
    " \n",
    "  sns.violinplot(x='months',\n",
    "                y='perc', \n",
    "                data=df.iloc[::-1], \n",
    "                inner=None)\n",
    "  \n",
    "  sns.swarmplot(x='months', \n",
    "                y='perc', \n",
    "                data=df.iloc[::-1], \n",
    "                color='k',\n",
    "                alpha=1) \n",
    "  plt.grid(axis='y', alpha=0.5)\n",
    "  plt.xlabel('Month',fontsize=15,fontweight='bold')\n",
    "  plt.xticks(fontsize=15,rotation=0)\n",
    "  plt.ylabel('Percentage',fontsize=15,fontweight='bold')\n",
    "  plt.yticks(fontsize=15)\n",
    "  plt.title('Percentage across months', fontsize=20)\n",
    "\n",
    "  st.pyplot()\n",
    "\n",
    "  def make_pie(sizes, text, colors):\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    sizes = [100-100*jsmean(df['final']), 100*jsmean(df['final'])]\n",
    "    text = round(jsmean(df['final'])*100,2)\n",
    "    col = [[i/255. for i in c] for c in colors]\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.axis('equal')\n",
    "    width = 0.30\n",
    "    kwargs = dict(colors=col, startangle=90)\n",
    "    outside, _ = ax.pie(sizes, radius=1, pctdistance=1-width/2,**kwargs)\n",
    "    plt.setp( outside, width=width, edgecolor='white')\n",
    "\n",
    "    kwargs = dict(size=20, fontweight='bold', va='center')\n",
    "    ax.text(0, 0, text, ha='center', **kwargs)\n",
    "    plt.show()\n",
    "\n",
    "  c2 = (226,33,0)\n",
    "  c1 = (40,133,4)\n",
    "\n",
    "  make_pie([257,90], round(df['perc'].mean(), 2),[c1,c2])\n",
    "  \n",
    "  st.pyplot()\n",
    "\n",
    "def probability_out(x):\n",
    "  n=len(x)\n",
    "  for i in range(n):\n",
    "    if(x.iloc[i,0]<0): \n",
    "      x.iloc[i,0]=0\n",
    "    if(x.iloc[i,0]>=0 and x.iloc[i,0]<=1):\n",
    "      x.iloc[i,0]=np.sin(x.iloc[i,0])\n",
    "    if(x.iloc[i,0]>1):\n",
    "      x.iloc[i,0]=(np.log(x.iloc[i,0])+(np.pi)*(np.pi)*(np.sin(1)))/((np.pi)**2)\n",
    "    if(x.iloc[i,0]>1):\n",
    "      x.iloc[i,0]=1\n",
    "  return x\n",
    "\n",
    "def tweets_conclusion(df):\n",
    "  #compute weights\n",
    "  def weight(x):\n",
    "    return (np.exp(x)-1)/(np.exp(1)-1)\n",
    "\n",
    "  def jsmean(arr):\n",
    "    num = 0\n",
    "    den = 0\n",
    "    for i in arr:\n",
    "        den = den + weight(i)\n",
    "        num = num + i*weight(i)\n",
    "    return (num/den)[0]\n",
    "\n",
    "  new_df = df.values\n",
    "  return jsmean(new_df)\n",
    "\n",
    "def combine_all(user_name):\n",
    "  #preprocessing input data\n",
    "  raw_text = user_name\n",
    "  recent_tweets=Show_Recent_Tweets(raw_text)\n",
    "  words = tokenize_tweets(recent_tweets)\n",
    "  sent = create_lemmatized_sent(words)\n",
    "  df = write_sent(recent_tweets, sent)\n",
    "  #loading models\n",
    "  \n",
    "  us = f\"Setting up models for analysing the profile of **{api.get_user(screen_name=raw_text).name}**\"\n",
    "  st.markdown(us)\n",
    "\n",
    "  st.text(\"Loading the model\")\n",
    "  model_lonely = keras.models.load_model('/content/drive/MyDrive/Utrack_Models/Utrack_Lonely')\n",
    "  model_stress = keras.models.load_model('/content/drive/MyDrive/Utrack_Models/Utrack_Stress')\n",
    "  model_anxiety = keras.models.load_model('/content/drive/MyDrive/Utrack_Models/Utrack_Anxiety')\n",
    "  \n",
    "  intro = f\"Twitter Bio of the user =>  **{api.get_user(screen_name=raw_text).description}**\"\n",
    "  st.markdown(intro)\n",
    "\n",
    "  bio = f\"User lives in **{api.get_user(screen_name=raw_text).location}**\"\n",
    "  st.markdown(bio)\n",
    "\n",
    "  fol = f\"Number of Followers of the user => **{api.get_user(screen_name=raw_text).followers_count}**\"\n",
    "  st.markdown(fol)\n",
    "\n",
    "  st.text(\"Hold Up!! Working on Predictions...\")\n",
    "\n",
    "  prediction_lonely = import_and_predict(df, model_lonely)\n",
    "  prediction_stress = import_and_predict(df, model_stress)\n",
    "  prediction_anxiety = import_and_predict(df, model_anxiety)\n",
    "\n",
    "  st.text(\"Predictions Done\")\n",
    "  \n",
    "  col1, col2, col3 = st.beta_columns(3)\n",
    "\n",
    "  prediction_lonely = pd.DataFrame(prediction_lonely, columns = ['Loneliness'])\n",
    "  prediction_stress = pd.DataFrame(prediction_stress, columns = ['Stress'])\n",
    "  prediction_anxiety = pd.DataFrame(prediction_anxiety, columns = ['Anxiety'])\n",
    "  \n",
    "  prediction_lonely = probability_out(prediction_lonely)\n",
    "  prediction_stress = probability_out(prediction_stress)\n",
    "  prediction_anxiety = probability_out(prediction_anxiety)\n",
    "  \n",
    "  df_total = output_dataframe(df,prediction_lonely)\n",
    "  df_total = output_dataframe(df_total,prediction_stress)\n",
    "  df_total = output_dataframe(df_total,prediction_anxiety)\n",
    "\n",
    "  st.write(df_total)\n",
    "  df_total = df_total.rename(columns={'time':'index'}).set_index('index')\n",
    "  \n",
    "  with col1:\n",
    "    st.text(\"LONELINESS LEVELS\")\n",
    "    st.success(tweets_conclusion(prediction_lonely))\n",
    "    st.line_chart(data=df_total['Loneliness'])\n",
    "\n",
    "  with col2:\n",
    "    st.text(\"STRESS LEVELS\")\n",
    "    st.success(tweets_conclusion(prediction_stress))\n",
    "    st.line_chart(data=df_total['Stress'])\n",
    "  \n",
    "  with col3:\n",
    "    st.text(\"ANXIETY LEVELS\")\n",
    "    st.success(tweets_conclusion(prediction_anxiety))\n",
    "    st.line_chart(data=df_total['Anxiety'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ck9QVRXJ7xKG"
   },
   "outputs": [],
   "source": [
    "%%writefile app.py\n",
    "from utilss import combine_all\n",
    "import tensorflow as tf\n",
    "import streamlit as st\n",
    "from tensorflow import keras\n",
    "st.set_option('deprecation.showfileUploaderEncoding', False)\n",
    "st.set_page_config(\n",
    "     page_title=\"UTrack\",\n",
    "     layout=\"wide\"\n",
    ")\n",
    "st.title(\"UTrack\")\n",
    "\n",
    "st.subheader('*Analysing Twitter Users on Tweet-to-Tweet basis to track levels of Loneliness, Stress & Anxiety*')\n",
    "\n",
    "raw_text = st.text_input(\"Enter the exact twitter handle of the Personality (without @)\")\n",
    "st.text(raw_text)\n",
    "if raw_text == '':\n",
    "  st.text('OOPS!!!! Enter userID')\n",
    "else:\n",
    "  combine_all(raw_text)\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0f2kGi4FJGpx"
   },
   "source": [
    "## Running  localhost server for colab from ngrok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "655f1jpFACpg"
   },
   "outputs": [],
   "source": [
    "!ngrok authtoken 1pqPDOU30ORUzHtrlCA5DX7odxX_4N3in7gRue2ctUDTBYPun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mX5GlrUUFUpg"
   },
   "outputs": [],
   "source": [
    "!nohup streamlit run app.py --server.port 80 &"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### USE THE OUTPUT OF THIS CELL AS THE LINK TO THE DASHBOARD (WEBSITE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nDEH2OYwGFg7"
   },
   "outputs": [],
   "source": [
    "# USE THE OUTPUT OF THIS CELL AS THE LINK TO THE DASHBOARD (WEBSITE)\n",
    "from pyngrok import ngrok\n",
    "\n",
    "url = ngrok.connect(port=80)\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Jr1866ZnGZ9R"
   },
   "outputs": [],
   "source": [
    "!cat /content/nohup.out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "IDkeVfvaItis"
   },
   "outputs": [],
   "source": [
    "# Uncomment this only to kill the terminals.\n",
    "## ! killall ngrok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2Ek0kcFck-C2"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Arsh_Utrack_GitHub.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}