Paired+RDD+simple+case+study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "########### Problem statement ###############\n",
    "\n",
    "# We have been given two data sets - \n",
    "# 1. UserPins.txt, which represents the users old search history\n",
    "# 2. UserSearch.txt, which represents the new search interests of the users.\n",
    "\n",
    "# Now with these two sets of informations, we are trying to identify, if any user is developing \n",
    "# interest in any new topic/area.\n",
    "\n",
    "# Students are expected to use spark core RDD features to solve this problem. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://ip-10-0-0-68.ec2.internal:4043\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.3.0.cloudera2</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>pairRDD_demo</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local appName=pairRDD_demo>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Rohit, Photography',\n",
       " u'Rohit, Photo Creation',\n",
       " u'Rohit, Adobe Effects',\n",
       " u'Rohit, Video Editing',\n",
       " u'Rohit, Youtube video making',\n",
       " u'',\n",
       " u'Rahul, Photography',\n",
       " u'Rahul, Instagram growth',\n",
       " u'Rahul, Photo Creation',\n",
       " u'Rahul, digital marketing',\n",
       " u'Rahul, Blogging',\n",
       " u'',\n",
       " u'Sachin, Photo Creation',\n",
       " u'Sachin, Blogging',\n",
       " u'Sachin, Adobe Effects',\n",
       " u'Sachin, Video Editing',\n",
       " u'Sachin, Youtube video making',\n",
       " u'',\n",
       " u'Saurav, Youtube video making',\n",
       " u'Saurav, UI/UX designs',\n",
       " u'Saurav, Instagram growth',\n",
       " u'Saurav, Web development',\n",
       " u'Saurav, Blogging',\n",
       " u'',\n",
       " u'Irfan, UI/UX designs',\n",
       " u'Irfan, Web development',\n",
       " u'Irfan, Blogging',\n",
       " u'Irfan, Adobe Effects',\n",
       " u'Irfan, Video Editing']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file1 = sc.textFile(\"UserPins.txt\")\n",
    "file1.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Rohit, Photography',\n",
       " u'Rohit, Web development',\n",
       " u'',\n",
       " u'Rahul, Photography',\n",
       " u'Rahul, Instagram growth',\n",
       " u'',\n",
       " u'Sachin, Photography',\n",
       " u'Sachin, Instagram growth',\n",
       " u'Sachin, Adobe Effects',\n",
       " u'',\n",
       " u'Irfan, Photography',\n",
       " u'Irfan, Instagram growth']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file2 = sc.textFile(\"UserSearch.txt\")\n",
    "file2.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Rohit, Photography',\n",
       " u'Rohit, Photo Creation',\n",
       " u'Rohit, Adobe Effects',\n",
       " u'Rohit, Video Editing',\n",
       " u'Rohit, Youtube video making',\n",
       " u'Rahul, Photography',\n",
       " u'Rahul, Instagram growth',\n",
       " u'Rahul, Photo Creation',\n",
       " u'Rahul, digital marketing',\n",
       " u'Rahul, Blogging',\n",
       " u'Sachin, Photo Creation',\n",
       " u'Sachin, Blogging',\n",
       " u'Sachin, Adobe Effects',\n",
       " u'Sachin, Video Editing',\n",
       " u'Sachin, Youtube video making',\n",
       " u'Saurav, Youtube video making',\n",
       " u'Saurav, UI/UX designs',\n",
       " u'Saurav, Instagram growth',\n",
       " u'Saurav, Web development',\n",
       " u'Saurav, Blogging',\n",
       " u'Irfan, UI/UX designs',\n",
       " u'Irfan, Web development',\n",
       " u'Irfan, Blogging',\n",
       " u'Irfan, Adobe Effects',\n",
       " u'Irfan, Video Editing']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file1_without_empty_line = file1.filter(lambda x:x!=\"\")\n",
    "file1_without_empty_line.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Rohit, Photography',\n",
       " u'Rohit, Web development',\n",
       " u'Rahul, Photography',\n",
       " u'Rahul, Instagram growth',\n",
       " u'Sachin, Photography',\n",
       " u'Sachin, Instagram growth',\n",
       " u'Sachin, Adobe Effects',\n",
       " u'Irfan, Photography',\n",
       " u'Irfan, Instagram growth']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file2_without_empty_line = file2.filter(lambda x:x!=\"\")\n",
    "file2_without_empty_line.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'Rohit', u'Photography'],\n",
       " [u'Rohit', u'Photo Creation'],\n",
       " [u'Rohit', u'Adobe Effects'],\n",
       " [u'Rohit', u'Video Editing'],\n",
       " [u'Rohit', u'Youtube video making'],\n",
       " [u'Rahul', u'Photography'],\n",
       " [u'Rahul', u'Instagram growth'],\n",
       " [u'Rahul', u'Photo Creation'],\n",
       " [u'Rahul', u'digital marketing'],\n",
       " [u'Rahul', u'Blogging'],\n",
       " [u'Sachin', u'Photo Creation'],\n",
       " [u'Sachin', u'Blogging'],\n",
       " [u'Sachin', u'Adobe Effects'],\n",
       " [u'Sachin', u'Video Editing'],\n",
       " [u'Sachin', u'Youtube video making'],\n",
       " [u'Saurav', u'Youtube video making'],\n",
       " [u'Saurav', u'UI/UX designs'],\n",
       " [u'Saurav', u'Instagram growth'],\n",
       " [u'Saurav', u'Web development'],\n",
       " [u'Saurav', u'Blogging'],\n",
       " [u'Irfan', u'UI/UX designs'],\n",
       " [u'Irfan', u'Web development'],\n",
       " [u'Irfan', u'Blogging'],\n",
       " [u'Irfan', u'Adobe Effects'],\n",
       " [u'Irfan', u'Video Editing']]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file1_array = file1_without_empty_line.map(lambda x: x.split(\", \"))\n",
    "file1_array.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'Rohit', u'Photography'],\n",
       " [u'Rohit', u'Web development'],\n",
       " [u'Rahul', u'Photography'],\n",
       " [u'Rahul', u'Instagram growth'],\n",
       " [u'Sachin', u'Photography'],\n",
       " [u'Sachin', u'Instagram growth'],\n",
       " [u'Sachin', u'Adobe Effects'],\n",
       " [u'Irfan', u'Photography'],\n",
       " [u'Irfan', u'Instagram growth']]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file2_array = file2_without_empty_line.map(lambda x: x.split(\", \"))\n",
    "file2_array.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Rohit', u'Photography'),\n",
       " (u'Rohit', u'Photo Creation'),\n",
       " (u'Rohit', u'Adobe Effects'),\n",
       " (u'Rohit', u'Video Editing'),\n",
       " (u'Rohit', u'Youtube video making'),\n",
       " (u'Rahul', u'Photography'),\n",
       " (u'Rahul', u'Instagram growth'),\n",
       " (u'Rahul', u'Photo Creation'),\n",
       " (u'Rahul', u'digital marketing'),\n",
       " (u'Rahul', u'Blogging'),\n",
       " (u'Sachin', u'Photo Creation'),\n",
       " (u'Sachin', u'Blogging'),\n",
       " (u'Sachin', u'Adobe Effects'),\n",
       " (u'Sachin', u'Video Editing'),\n",
       " (u'Sachin', u'Youtube video making'),\n",
       " (u'Saurav', u'Youtube video making'),\n",
       " (u'Saurav', u'UI/UX designs'),\n",
       " (u'Saurav', u'Instagram growth'),\n",
       " (u'Saurav', u'Web development'),\n",
       " (u'Saurav', u'Blogging'),\n",
       " (u'Irfan', u'UI/UX designs'),\n",
       " (u'Irfan', u'Web development'),\n",
       " (u'Irfan', u'Blogging'),\n",
       " (u'Irfan', u'Adobe Effects'),\n",
       " (u'Irfan', u'Video Editing')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userInfo = file1_array.map(lambda x: (x[0],x[1]))\n",
    "userInfo.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Rohit', u'Photography'),\n",
       " (u'Rohit', u'Web development'),\n",
       " (u'Rahul', u'Photography'),\n",
       " (u'Rahul', u'Instagram growth'),\n",
       " (u'Sachin', u'Photography'),\n",
       " (u'Sachin', u'Instagram growth'),\n",
       " (u'Sachin', u'Adobe Effects'),\n",
       " (u'Irfan', u'Photography'),\n",
       " (u'Irfan', u'Instagram growth')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linkInfo = file2_array.map(lambda x: (x[0],x[1]))\n",
    "linkInfo.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Sachin', u'Adobe Effects'),\n",
       " (u'Rahul', u'Instagram growth'),\n",
       " (u'Rohit', u'Photography'),\n",
       " (u'Rahul', u'Photography')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_interest = linkInfo.intersection(userInfo)\n",
    "old_interest.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Irfan', u'Photography'),\n",
       " (u'Sachin', u'Instagram growth'),\n",
       " (u'Sachin', u'Photography'),\n",
       " (u'Rohit', u'Web development'),\n",
       " (u'Irfan', u'Instagram growth')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_interest = linkInfo.subtract(old_interest)\n",
    "new_interest.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_interest_sorted = new_interest.sortByKey()\n",
    "new_interest_sorted.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}