Paired+RDD's.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################                    Word count problem                 #####################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = sc.textFile(\"abc.txt\")\n",
    "lines.collect()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_empty_lines = lines.filter(lambda x: x!=\"\")\n",
    "non_empty_lines.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Download',\n",
       " u'Apache',\n",
       " u'Spark\\u2122',\n",
       " u'Choose',\n",
       " u'a',\n",
       " u'Spark',\n",
       " u'release:',\n",
       " u'',\n",
       " u'2.4.6',\n",
       " u'(Jun',\n",
       " u'05',\n",
       " u'2020)',\n",
       " u'Choose',\n",
       " u'a',\n",
       " u'package',\n",
       " u'type:',\n",
       " u'',\n",
       " u'Pre-built',\n",
       " u'for',\n",
       " u'Apache',\n",
       " u'Hadoop',\n",
       " u'2.7',\n",
       " u'Download',\n",
       " u'Spark:',\n",
       " u'spark-3.0.0-preview2-bin-hadoop2.7.tgz',\n",
       " u'Verify',\n",
       " u'this',\n",
       " u'release',\n",
       " u'using',\n",
       " u'the',\n",
       " u'3.0.0-preview2',\n",
       " u'signatures,',\n",
       " u'checksums',\n",
       " u'and',\n",
       " u'project',\n",
       " u'release',\n",
       " u'KEYS.',\n",
       " u'Note',\n",
       " u'that,',\n",
       " u'Spark',\n",
       " u'is',\n",
       " u'pre-built',\n",
       " u'with',\n",
       " u'Scala',\n",
       " u'2.11',\n",
       " u'except',\n",
       " u'version',\n",
       " u'2.4.2,',\n",
       " u'which',\n",
       " u'is',\n",
       " u'pre-built',\n",
       " u'with',\n",
       " u'Scala',\n",
       " u'2.12.',\n",
       " u'Latest',\n",
       " u'Preview',\n",
       " u'Release',\n",
       " u'Preview',\n",
       " u'releases,',\n",
       " u'as',\n",
       " u'the',\n",
       " u'name',\n",
       " u'suggests,',\n",
       " u'are',\n",
       " u'releases',\n",
       " u'for',\n",
       " u'previewing',\n",
       " u'upcoming',\n",
       " u'features.',\n",
       " u'Unlike',\n",
       " u'nightly',\n",
       " u'packages,',\n",
       " u'preview',\n",
       " u'releases',\n",
       " u'have',\n",
       " u'been',\n",
       " u'audited',\n",
       " u'by',\n",
       " u'the',\n",
       " u'project\\u2019s',\n",
       " u'management',\n",
       " u'committee',\n",
       " u'to',\n",
       " u'satisfy',\n",
       " u'the',\n",
       " u'legal',\n",
       " u'requirements',\n",
       " u'of',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation\\u2019s',\n",
       " u'release',\n",
       " u'policy.',\n",
       " u'Preview',\n",
       " u'releases',\n",
       " u'are',\n",
       " u'not',\n",
       " u'meant',\n",
       " u'to',\n",
       " u'be',\n",
       " u'functional,',\n",
       " u'i.e.',\n",
       " u'they',\n",
       " u'can',\n",
       " u'and',\n",
       " u'highly',\n",
       " u'likely',\n",
       " u'will',\n",
       " u'contain',\n",
       " u'critical',\n",
       " u'bugs',\n",
       " u'or',\n",
       " u'documentation',\n",
       " u'errors.',\n",
       " u'The',\n",
       " u'latest',\n",
       " u'preview',\n",
       " u'release',\n",
       " u'is',\n",
       " u'Spark',\n",
       " u'3.0.0-preview2,',\n",
       " u'published',\n",
       " u'on',\n",
       " u'Dec',\n",
       " u'23,',\n",
       " u'2019.',\n",
       " u'You',\n",
       " u'can',\n",
       " u'select',\n",
       " u'and',\n",
       " u'download',\n",
       " u'it',\n",
       " u'above.',\n",
       " u'Link',\n",
       " u'with',\n",
       " u'Spark',\n",
       " u'Spark',\n",
       " u'artifacts',\n",
       " u'are',\n",
       " u'hosted',\n",
       " u'in',\n",
       " u'Maven',\n",
       " u'Central.',\n",
       " u'You',\n",
       " u'can',\n",
       " u'add',\n",
       " u'a',\n",
       " u'Maven',\n",
       " u'dependency',\n",
       " u'with',\n",
       " u'the',\n",
       " u'following',\n",
       " u'coordinates:',\n",
       " u'groupId:',\n",
       " u'org.apache.spark',\n",
       " u'artifactId:',\n",
       " u'spark-core_2.11',\n",
       " u'version:',\n",
       " u'2.4.6',\n",
       " u'Installing',\n",
       " u'with',\n",
       " u'PyPi',\n",
       " u'PySpark',\n",
       " u'is',\n",
       " u'now',\n",
       " u'available',\n",
       " u'in',\n",
       " u'pypi.',\n",
       " u'To',\n",
       " u'install',\n",
       " u'just',\n",
       " u'run',\n",
       " u'pip',\n",
       " u'install',\n",
       " u'pyspark.',\n",
       " u'Release',\n",
       " u'Notes',\n",
       " u'for',\n",
       " u'Stable',\n",
       " u'Releases',\n",
       " u'Spark',\n",
       " u'3.0.0-preview2',\n",
       " u'(Dec',\n",
       " u'23',\n",
       " u'2019)',\n",
       " u'Spark',\n",
       " u'2.4.6',\n",
       " u'(Jun',\n",
       " u'05',\n",
       " u'2020)',\n",
       " u'Archived',\n",
       " u'Releases',\n",
       " u'As',\n",
       " u'new',\n",
       " u'Spark',\n",
       " u'releases',\n",
       " u'come',\n",
       " u'out',\n",
       " u'for',\n",
       " u'each',\n",
       " u'development',\n",
       " u'stream,',\n",
       " u'previous',\n",
       " u'ones',\n",
       " u'will',\n",
       " u'be',\n",
       " u'archived,',\n",
       " u'but',\n",
       " u'they',\n",
       " u'are',\n",
       " u'still',\n",
       " u'available',\n",
       " u'at',\n",
       " u'Spark',\n",
       " u'release',\n",
       " u'archives.',\n",
       " u'Apache',\n",
       " u'Spark,',\n",
       " u'Spark,',\n",
       " u'Apache,',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'feather',\n",
       " u'logo,',\n",
       " u'and',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'Spark',\n",
       " u'project',\n",
       " u'logo',\n",
       " u'are',\n",
       " u'either',\n",
       " u'registered',\n",
       " u'trademarks',\n",
       " u'or',\n",
       " u'trademarks',\n",
       " u'of',\n",
       " u'The',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation',\n",
       " u'in',\n",
       " u'the',\n",
       " u'United',\n",
       " u'States',\n",
       " u'and',\n",
       " u'other',\n",
       " u'countries.',\n",
       " u'See',\n",
       " u'guidance',\n",
       " u'on',\n",
       " u'use',\n",
       " u'of',\n",
       " u'Apache',\n",
       " u'Spark',\n",
       " u'trademarks.',\n",
       " u'All',\n",
       " u'other',\n",
       " u'marks',\n",
       " u'mentioned',\n",
       " u'may',\n",
       " u'be',\n",
       " u'trademarks',\n",
       " u'or',\n",
       " u'registered',\n",
       " u'trademarks',\n",
       " u'of',\n",
       " u'their',\n",
       " u'respective',\n",
       " u'owners.',\n",
       " u'Copyright',\n",
       " u'\\xa9',\n",
       " u'2018',\n",
       " u'The',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation,',\n",
       " u'Licensed',\n",
       " u'under',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'License,',\n",
       " u'Version',\n",
       " u'2.0.This',\n",
       " u'is',\n",
       " u'a',\n",
       " u'text',\n",
       " u'File.',\n",
       " u'This',\n",
       " u'is',\n",
       " u'a',\n",
       " u'text',\n",
       " u'file.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = non_empty_lines.flatMap(lambda x: (x.split(\" \")))\n",
    "words.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paired_rdd = words.map(lambda x:(x,1))\n",
    "paired_rdd.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'', 2),\n",
       " (u'United', 1),\n",
       " (u'3.0.0-preview2', 2),\n",
       " (u'just', 1),\n",
       " (u'PySpark', 1),\n",
       " (u'Verify', 1),\n",
       " (u'dependency', 1),\n",
       " (u'Hadoop', 1),\n",
       " (u'Release', 2),\n",
       " (u'policy.', 1),\n",
       " (u'artifactId:', 1),\n",
       " (u'committee', 1),\n",
       " (u'mentioned', 1),\n",
       " (u'still', 1),\n",
       " (u'previous', 1),\n",
       " (u'Foundation', 1),\n",
       " (u'Copyright', 1),\n",
       " (u'23', 1),\n",
       " (u'guidance', 1),\n",
       " (u'spark-3.0.0-preview2-bin-hadoop2.7.tgz', 1),\n",
       " (u'except', 1),\n",
       " (u'to', 2),\n",
       " (u'add', 1),\n",
       " (u'critical', 1),\n",
       " (u'2018', 1),\n",
       " (u'marks', 1),\n",
       " (u'Apache', 10),\n",
       " (u'Software', 3),\n",
       " (u'2.11', 1),\n",
       " (u'meant', 1),\n",
       " (u'Notes', 1),\n",
       " (u'Foundation\\u2019s', 1),\n",
       " (u'KEYS.', 1),\n",
       " (u'ones', 1),\n",
       " (u'they', 2),\n",
       " (u'not', 1),\n",
       " (u'using', 1),\n",
       " (u'now', 1),\n",
       " (u'previewing', 1),\n",
       " (u'name', 1),\n",
       " (u'Apache,', 1),\n",
       " (u'groupId:', 1),\n",
       " (u'either', 1),\n",
       " (u'contain', 1),\n",
       " (u'release', 5),\n",
       " (u'signatures,', 1),\n",
       " (u'audited', 1),\n",
       " (u'2.12.', 1),\n",
       " (u'Archived', 1),\n",
       " (u'(Jun', 2),\n",
       " (u'Scala', 2),\n",
       " (u'2.0.This', 1),\n",
       " (u'likely', 1),\n",
       " (u'Version', 1),\n",
       " (u'Choose', 2),\n",
       " (u'hosted', 1),\n",
       " (u'download', 1),\n",
       " (u'checksums', 1),\n",
       " (u'2.7', 1),\n",
       " (u'out', 1),\n",
       " (u'that,', 1),\n",
       " (u'will', 2),\n",
       " (u'for', 4),\n",
       " (u'05', 2),\n",
       " (u'stream,', 1),\n",
       " (u'legal', 1),\n",
       " (u'Maven', 2),\n",
       " (u'version', 1),\n",
       " (u'Link', 1),\n",
       " (u'new', 1),\n",
       " (u'2019.', 1),\n",
       " (u'2019)', 1),\n",
       " (u'3.0.0-preview2,', 1),\n",
       " (u'satisfy', 1),\n",
       " (u'Spark:', 1),\n",
       " (u'run', 1),\n",
       " (u'This', 1),\n",
       " (u'org.apache.spark', 1),\n",
       " (u'Pre-built', 1),\n",
       " (u'archives.', 1),\n",
       " (u'trademarks', 4),\n",
       " (u'Spark,', 2),\n",
       " (u'come', 1),\n",
       " (u'by', 1),\n",
       " (u'on', 2),\n",
       " (u'pyspark.', 1),\n",
       " (u'package', 1),\n",
       " (u'2020)', 2),\n",
       " (u'countries.', 1),\n",
       " (u'File.', 1),\n",
       " (u'above.', 1),\n",
       " (u'functional,', 1),\n",
       " (u'pypi.', 1),\n",
       " (u'published', 1),\n",
       " (u'Download', 2),\n",
       " (u'or', 3),\n",
       " (u'and', 5),\n",
       " (u'errors.', 1),\n",
       " (u'each', 1),\n",
       " (u'PyPi', 1),\n",
       " (u'suggests,', 1),\n",
       " (u'highly', 1),\n",
       " (u'Note', 1),\n",
       " (u'been', 1),\n",
       " (u'Preview', 3),\n",
       " (u'packages,', 1),\n",
       " (u'file.', 1),\n",
       " (u'select', 1),\n",
       " (u'release:', 1),\n",
       " (u'use', 1),\n",
       " (u'management', 1),\n",
       " (u'i.e.', 1),\n",
       " (u'artifacts', 1),\n",
       " (u'available', 2),\n",
       " (u'version:', 1),\n",
       " (u'Central.', 1),\n",
       " (u'their', 1),\n",
       " (u'feather', 1),\n",
       " (u'preview', 2),\n",
       " (u'2.4.2,', 1),\n",
       " (u'upcoming', 1),\n",
       " (u'registered', 2),\n",
       " (u'under', 1),\n",
       " (u'but', 1),\n",
       " (u'releases,', 1),\n",
       " (u'spark-core_2.11', 1),\n",
       " (u'be', 3),\n",
       " (u'Dec', 1),\n",
       " (u'with', 5),\n",
       " (u'2.4.6', 3),\n",
       " (u'Latest', 1),\n",
       " (u'Foundation,', 1),\n",
       " (u'this', 1),\n",
       " (u'project', 2),\n",
       " (u'See', 1),\n",
       " (u'can', 3),\n",
       " (u'install', 2),\n",
       " (u'of', 4),\n",
       " (u'following', 1),\n",
       " (u'are', 5),\n",
       " (u'23,', 1),\n",
       " (u'features.', 1),\n",
       " (u'owners.', 1),\n",
       " (u'is', 6),\n",
       " (u'Stable', 1),\n",
       " (u'it', 1),\n",
       " (u'pre-built', 2),\n",
       " (u'To', 1),\n",
       " (u'as', 1),\n",
       " (u'logo,', 1),\n",
       " (u'at', 1),\n",
       " (u'have', 1),\n",
       " (u'logo', 1),\n",
       " (u'pip', 1),\n",
       " (u'coordinates:', 1),\n",
       " (u'You', 2),\n",
       " (u'\\xa9', 1),\n",
       " (u'respective', 1),\n",
       " (u'requirements', 1),\n",
       " (u'releases', 4),\n",
       " (u'License,', 1),\n",
       " (u'trademarks.', 1),\n",
       " (u'other', 2),\n",
       " (u'which', 1),\n",
       " (u'Spark', 11),\n",
       " (u'development', 1),\n",
       " (u'type:', 1),\n",
       " (u'Unlike', 1),\n",
       " (u'may', 1),\n",
       " (u'Installing', 1),\n",
       " (u'project\\u2019s', 1),\n",
       " (u'nightly', 1),\n",
       " (u'(Dec', 1),\n",
       " (u'Spark\\u2122', 1),\n",
       " (u'The', 3),\n",
       " (u'archived,', 1),\n",
       " (u'a', 5),\n",
       " (u'All', 1),\n",
       " (u'in', 3),\n",
       " (u'Releases', 2),\n",
       " (u'documentation', 1),\n",
       " (u'bugs', 1),\n",
       " (u'Licensed', 1),\n",
       " (u'States', 1),\n",
       " (u'As', 1),\n",
       " (u'text', 2),\n",
       " (u'the', 9),\n",
       " (u'latest', 1)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_with_count = paired_rdd.reduceByKey(lambda x,y:x+y)\n",
    "words_with_count.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Finding the word count with out using reduceByKey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Download Apache Spark\\u2122',\n",
       " u'Choose a Spark release: ',\n",
       " u'2.4.6 (Jun 05 2020)',\n",
       " u'',\n",
       " u'',\n",
       " u'Choose a package type: ',\n",
       " u'Pre-built for Apache Hadoop 2.7',\n",
       " u'',\n",
       " u'',\n",
       " u'Download Spark: spark-3.0.0-preview2-bin-hadoop2.7.tgz',\n",
       " u'',\n",
       " u'Verify this release using the 3.0.0-preview2 signatures, checksums and project release KEYS.',\n",
       " u'',\n",
       " u'Note that, Spark is pre-built with Scala 2.11 except version 2.4.2, which is pre-built with Scala 2.12.',\n",
       " u'',\n",
       " u'Latest Preview Release',\n",
       " u'Preview releases, as the name suggests, are releases for previewing upcoming features. Unlike nightly packages, preview releases have been audited by the project\\u2019s management committee to satisfy the legal requirements of Apache Software Foundation\\u2019s release policy. Preview releases are not meant to be functional, i.e. they can and highly likely will contain critical bugs or documentation errors. The latest preview release is Spark 3.0.0-preview2, published on Dec 23, 2019. You can select and download it above.',\n",
       " u'',\n",
       " u'Link with Spark',\n",
       " u'Spark artifacts are hosted in Maven Central. You can add a Maven dependency with the following coordinates:',\n",
       " u'',\n",
       " u'groupId: org.apache.spark',\n",
       " u'artifactId: spark-core_2.11',\n",
       " u'version: 2.4.6',\n",
       " u'Installing with PyPi',\n",
       " u'PySpark is now available in pypi. To install just run pip install pyspark.',\n",
       " u'',\n",
       " u'Release Notes for Stable Releases',\n",
       " u'Spark 3.0.0-preview2 (Dec 23 2019)',\n",
       " u'Spark 2.4.6 (Jun 05 2020)',\n",
       " u'Archived Releases',\n",
       " u'As new Spark releases come out for each development stream, previous ones will be archived, but they are still available at Spark release archives.',\n",
       " u'',\n",
       " u'Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. See guidance on use of Apache Spark trademarks. All other marks mentioned may be trademarks or registered trademarks of their respective owners. Copyright \\xa9 2018 The Apache Software Foundation, Licensed under the Apache License, Version 2.0.This is a text File.',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'This is a text file.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = sc.textFile(\"abc.txt\")\n",
    "lines.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Download Apache Spark\\u2122',\n",
       " u'Choose a Spark release: ',\n",
       " u'2.4.6 (Jun 05 2020)',\n",
       " u'Choose a package type: ',\n",
       " u'Pre-built for Apache Hadoop 2.7',\n",
       " u'Download Spark: spark-3.0.0-preview2-bin-hadoop2.7.tgz',\n",
       " u'Verify this release using the 3.0.0-preview2 signatures, checksums and project release KEYS.',\n",
       " u'Note that, Spark is pre-built with Scala 2.11 except version 2.4.2, which is pre-built with Scala 2.12.',\n",
       " u'Latest Preview Release',\n",
       " u'Preview releases, as the name suggests, are releases for previewing upcoming features. Unlike nightly packages, preview releases have been audited by the project\\u2019s management committee to satisfy the legal requirements of Apache Software Foundation\\u2019s release policy. Preview releases are not meant to be functional, i.e. they can and highly likely will contain critical bugs or documentation errors. The latest preview release is Spark 3.0.0-preview2, published on Dec 23, 2019. You can select and download it above.',\n",
       " u'Link with Spark',\n",
       " u'Spark artifacts are hosted in Maven Central. You can add a Maven dependency with the following coordinates:',\n",
       " u'groupId: org.apache.spark',\n",
       " u'artifactId: spark-core_2.11',\n",
       " u'version: 2.4.6',\n",
       " u'Installing with PyPi',\n",
       " u'PySpark is now available in pypi. To install just run pip install pyspark.',\n",
       " u'Release Notes for Stable Releases',\n",
       " u'Spark 3.0.0-preview2 (Dec 23 2019)',\n",
       " u'Spark 2.4.6 (Jun 05 2020)',\n",
       " u'Archived Releases',\n",
       " u'As new Spark releases come out for each development stream, previous ones will be archived, but they are still available at Spark release archives.',\n",
       " u'Apache Spark, Spark, Apache, the Apache feather logo, and the Apache Spark project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. See guidance on use of Apache Spark trademarks. All other marks mentioned may be trademarks or registered trademarks of their respective owners. Copyright \\xa9 2018 The Apache Software Foundation, Licensed under the Apache License, Version 2.0.This is a text File.',\n",
       " u'This is a text file.']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_empty_lines = lines.filter(lambda x: x!=\"\")\n",
    "non_empty_lines.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'Download',\n",
       " u'Apache',\n",
       " u'Spark\\u2122',\n",
       " u'Choose',\n",
       " u'a',\n",
       " u'Spark',\n",
       " u'release:',\n",
       " u'',\n",
       " u'2.4.6',\n",
       " u'(Jun',\n",
       " u'05',\n",
       " u'2020)',\n",
       " u'Choose',\n",
       " u'a',\n",
       " u'package',\n",
       " u'type:',\n",
       " u'',\n",
       " u'Pre-built',\n",
       " u'for',\n",
       " u'Apache',\n",
       " u'Hadoop',\n",
       " u'2.7',\n",
       " u'Download',\n",
       " u'Spark:',\n",
       " u'spark-3.0.0-preview2-bin-hadoop2.7.tgz',\n",
       " u'Verify',\n",
       " u'this',\n",
       " u'release',\n",
       " u'using',\n",
       " u'the',\n",
       " u'3.0.0-preview2',\n",
       " u'signatures,',\n",
       " u'checksums',\n",
       " u'and',\n",
       " u'project',\n",
       " u'release',\n",
       " u'KEYS.',\n",
       " u'Note',\n",
       " u'that,',\n",
       " u'Spark',\n",
       " u'is',\n",
       " u'pre-built',\n",
       " u'with',\n",
       " u'Scala',\n",
       " u'2.11',\n",
       " u'except',\n",
       " u'version',\n",
       " u'2.4.2,',\n",
       " u'which',\n",
       " u'is',\n",
       " u'pre-built',\n",
       " u'with',\n",
       " u'Scala',\n",
       " u'2.12.',\n",
       " u'Latest',\n",
       " u'Preview',\n",
       " u'Release',\n",
       " u'Preview',\n",
       " u'releases,',\n",
       " u'as',\n",
       " u'the',\n",
       " u'name',\n",
       " u'suggests,',\n",
       " u'are',\n",
       " u'releases',\n",
       " u'for',\n",
       " u'previewing',\n",
       " u'upcoming',\n",
       " u'features.',\n",
       " u'Unlike',\n",
       " u'nightly',\n",
       " u'packages,',\n",
       " u'preview',\n",
       " u'releases',\n",
       " u'have',\n",
       " u'been',\n",
       " u'audited',\n",
       " u'by',\n",
       " u'the',\n",
       " u'project\\u2019s',\n",
       " u'management',\n",
       " u'committee',\n",
       " u'to',\n",
       " u'satisfy',\n",
       " u'the',\n",
       " u'legal',\n",
       " u'requirements',\n",
       " u'of',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation\\u2019s',\n",
       " u'release',\n",
       " u'policy.',\n",
       " u'Preview',\n",
       " u'releases',\n",
       " u'are',\n",
       " u'not',\n",
       " u'meant',\n",
       " u'to',\n",
       " u'be',\n",
       " u'functional,',\n",
       " u'i.e.',\n",
       " u'they',\n",
       " u'can',\n",
       " u'and',\n",
       " u'highly',\n",
       " u'likely',\n",
       " u'will',\n",
       " u'contain',\n",
       " u'critical',\n",
       " u'bugs',\n",
       " u'or',\n",
       " u'documentation',\n",
       " u'errors.',\n",
       " u'The',\n",
       " u'latest',\n",
       " u'preview',\n",
       " u'release',\n",
       " u'is',\n",
       " u'Spark',\n",
       " u'3.0.0-preview2,',\n",
       " u'published',\n",
       " u'on',\n",
       " u'Dec',\n",
       " u'23,',\n",
       " u'2019.',\n",
       " u'You',\n",
       " u'can',\n",
       " u'select',\n",
       " u'and',\n",
       " u'download',\n",
       " u'it',\n",
       " u'above.',\n",
       " u'Link',\n",
       " u'with',\n",
       " u'Spark',\n",
       " u'Spark',\n",
       " u'artifacts',\n",
       " u'are',\n",
       " u'hosted',\n",
       " u'in',\n",
       " u'Maven',\n",
       " u'Central.',\n",
       " u'You',\n",
       " u'can',\n",
       " u'add',\n",
       " u'a',\n",
       " u'Maven',\n",
       " u'dependency',\n",
       " u'with',\n",
       " u'the',\n",
       " u'following',\n",
       " u'coordinates:',\n",
       " u'groupId:',\n",
       " u'org.apache.spark',\n",
       " u'artifactId:',\n",
       " u'spark-core_2.11',\n",
       " u'version:',\n",
       " u'2.4.6',\n",
       " u'Installing',\n",
       " u'with',\n",
       " u'PyPi',\n",
       " u'PySpark',\n",
       " u'is',\n",
       " u'now',\n",
       " u'available',\n",
       " u'in',\n",
       " u'pypi.',\n",
       " u'To',\n",
       " u'install',\n",
       " u'just',\n",
       " u'run',\n",
       " u'pip',\n",
       " u'install',\n",
       " u'pyspark.',\n",
       " u'Release',\n",
       " u'Notes',\n",
       " u'for',\n",
       " u'Stable',\n",
       " u'Releases',\n",
       " u'Spark',\n",
       " u'3.0.0-preview2',\n",
       " u'(Dec',\n",
       " u'23',\n",
       " u'2019)',\n",
       " u'Spark',\n",
       " u'2.4.6',\n",
       " u'(Jun',\n",
       " u'05',\n",
       " u'2020)',\n",
       " u'Archived',\n",
       " u'Releases',\n",
       " u'As',\n",
       " u'new',\n",
       " u'Spark',\n",
       " u'releases',\n",
       " u'come',\n",
       " u'out',\n",
       " u'for',\n",
       " u'each',\n",
       " u'development',\n",
       " u'stream,',\n",
       " u'previous',\n",
       " u'ones',\n",
       " u'will',\n",
       " u'be',\n",
       " u'archived,',\n",
       " u'but',\n",
       " u'they',\n",
       " u'are',\n",
       " u'still',\n",
       " u'available',\n",
       " u'at',\n",
       " u'Spark',\n",
       " u'release',\n",
       " u'archives.',\n",
       " u'Apache',\n",
       " u'Spark,',\n",
       " u'Spark,',\n",
       " u'Apache,',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'feather',\n",
       " u'logo,',\n",
       " u'and',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'Spark',\n",
       " u'project',\n",
       " u'logo',\n",
       " u'are',\n",
       " u'either',\n",
       " u'registered',\n",
       " u'trademarks',\n",
       " u'or',\n",
       " u'trademarks',\n",
       " u'of',\n",
       " u'The',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation',\n",
       " u'in',\n",
       " u'the',\n",
       " u'United',\n",
       " u'States',\n",
       " u'and',\n",
       " u'other',\n",
       " u'countries.',\n",
       " u'See',\n",
       " u'guidance',\n",
       " u'on',\n",
       " u'use',\n",
       " u'of',\n",
       " u'Apache',\n",
       " u'Spark',\n",
       " u'trademarks.',\n",
       " u'All',\n",
       " u'other',\n",
       " u'marks',\n",
       " u'mentioned',\n",
       " u'may',\n",
       " u'be',\n",
       " u'trademarks',\n",
       " u'or',\n",
       " u'registered',\n",
       " u'trademarks',\n",
       " u'of',\n",
       " u'their',\n",
       " u'respective',\n",
       " u'owners.',\n",
       " u'Copyright',\n",
       " u'\\xa9',\n",
       " u'2018',\n",
       " u'The',\n",
       " u'Apache',\n",
       " u'Software',\n",
       " u'Foundation,',\n",
       " u'Licensed',\n",
       " u'under',\n",
       " u'the',\n",
       " u'Apache',\n",
       " u'License,',\n",
       " u'Version',\n",
       " u'2.0.This',\n",
       " u'is',\n",
       " u'a',\n",
       " u'text',\n",
       " u'File.',\n",
       " u'This',\n",
       " u'is',\n",
       " u'a',\n",
       " u'text',\n",
       " u'file.']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = non_empty_lines.flatMap(lambda x: (x.split(\" \")))\n",
    "words.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {u'': 2,\n",
       "             u'(Dec': 1,\n",
       "             u'(Jun': 2,\n",
       "             u'05': 2,\n",
       "             u'2.0.This': 1,\n",
       "             u'2.11': 1,\n",
       "             u'2.12.': 1,\n",
       "             u'2.4.2,': 1,\n",
       "             u'2.4.6': 3,\n",
       "             u'2.7': 1,\n",
       "             u'2018': 1,\n",
       "             u'2019)': 1,\n",
       "             u'2019.': 1,\n",
       "             u'2020)': 2,\n",
       "             u'23': 1,\n",
       "             u'23,': 1,\n",
       "             u'3.0.0-preview2': 2,\n",
       "             u'3.0.0-preview2,': 1,\n",
       "             u'All': 1,\n",
       "             u'Apache': 10,\n",
       "             u'Apache,': 1,\n",
       "             u'Archived': 1,\n",
       "             u'As': 1,\n",
       "             u'Central.': 1,\n",
       "             u'Choose': 2,\n",
       "             u'Copyright': 1,\n",
       "             u'Dec': 1,\n",
       "             u'Download': 2,\n",
       "             u'File.': 1,\n",
       "             u'Foundation': 1,\n",
       "             u'Foundation,': 1,\n",
       "             u'Foundation\\u2019s': 1,\n",
       "             u'Hadoop': 1,\n",
       "             u'Installing': 1,\n",
       "             u'KEYS.': 1,\n",
       "             u'Latest': 1,\n",
       "             u'License,': 1,\n",
       "             u'Licensed': 1,\n",
       "             u'Link': 1,\n",
       "             u'Maven': 2,\n",
       "             u'Note': 1,\n",
       "             u'Notes': 1,\n",
       "             u'Pre-built': 1,\n",
       "             u'Preview': 3,\n",
       "             u'PyPi': 1,\n",
       "             u'PySpark': 1,\n",
       "             u'Release': 2,\n",
       "             u'Releases': 2,\n",
       "             u'Scala': 2,\n",
       "             u'See': 1,\n",
       "             u'Software': 3,\n",
       "             u'Spark': 11,\n",
       "             u'Spark,': 2,\n",
       "             u'Spark:': 1,\n",
       "             u'Spark\\u2122': 1,\n",
       "             u'Stable': 1,\n",
       "             u'States': 1,\n",
       "             u'The': 3,\n",
       "             u'This': 1,\n",
       "             u'To': 1,\n",
       "             u'United': 1,\n",
       "             u'Unlike': 1,\n",
       "             u'Verify': 1,\n",
       "             u'Version': 1,\n",
       "             u'You': 2,\n",
       "             u'a': 5,\n",
       "             u'above.': 1,\n",
       "             u'add': 1,\n",
       "             u'and': 5,\n",
       "             u'archived,': 1,\n",
       "             u'archives.': 1,\n",
       "             u'are': 5,\n",
       "             u'artifactId:': 1,\n",
       "             u'artifacts': 1,\n",
       "             u'as': 1,\n",
       "             u'at': 1,\n",
       "             u'audited': 1,\n",
       "             u'available': 2,\n",
       "             u'be': 3,\n",
       "             u'been': 1,\n",
       "             u'bugs': 1,\n",
       "             u'but': 1,\n",
       "             u'by': 1,\n",
       "             u'can': 3,\n",
       "             u'checksums': 1,\n",
       "             u'come': 1,\n",
       "             u'committee': 1,\n",
       "             u'contain': 1,\n",
       "             u'coordinates:': 1,\n",
       "             u'countries.': 1,\n",
       "             u'critical': 1,\n",
       "             u'dependency': 1,\n",
       "             u'development': 1,\n",
       "             u'documentation': 1,\n",
       "             u'download': 1,\n",
       "             u'each': 1,\n",
       "             u'either': 1,\n",
       "             u'errors.': 1,\n",
       "             u'except': 1,\n",
       "             u'feather': 1,\n",
       "             u'features.': 1,\n",
       "             u'file.': 1,\n",
       "             u'following': 1,\n",
       "             u'for': 4,\n",
       "             u'functional,': 1,\n",
       "             u'groupId:': 1,\n",
       "             u'guidance': 1,\n",
       "             u'have': 1,\n",
       "             u'highly': 1,\n",
       "             u'hosted': 1,\n",
       "             u'i.e.': 1,\n",
       "             u'in': 3,\n",
       "             u'install': 2,\n",
       "             u'is': 6,\n",
       "             u'it': 1,\n",
       "             u'just': 1,\n",
       "             u'latest': 1,\n",
       "             u'legal': 1,\n",
       "             u'likely': 1,\n",
       "             u'logo': 1,\n",
       "             u'logo,': 1,\n",
       "             u'management': 1,\n",
       "             u'marks': 1,\n",
       "             u'may': 1,\n",
       "             u'meant': 1,\n",
       "             u'mentioned': 1,\n",
       "             u'name': 1,\n",
       "             u'new': 1,\n",
       "             u'nightly': 1,\n",
       "             u'not': 1,\n",
       "             u'now': 1,\n",
       "             u'of': 4,\n",
       "             u'on': 2,\n",
       "             u'ones': 1,\n",
       "             u'or': 3,\n",
       "             u'org.apache.spark': 1,\n",
       "             u'other': 2,\n",
       "             u'out': 1,\n",
       "             u'owners.': 1,\n",
       "             u'package': 1,\n",
       "             u'packages,': 1,\n",
       "             u'pip': 1,\n",
       "             u'policy.': 1,\n",
       "             u'pre-built': 2,\n",
       "             u'preview': 2,\n",
       "             u'previewing': 1,\n",
       "             u'previous': 1,\n",
       "             u'project': 2,\n",
       "             u'project\\u2019s': 1,\n",
       "             u'published': 1,\n",
       "             u'pypi.': 1,\n",
       "             u'pyspark.': 1,\n",
       "             u'registered': 2,\n",
       "             u'release': 5,\n",
       "             u'release:': 1,\n",
       "             u'releases': 4,\n",
       "             u'releases,': 1,\n",
       "             u'requirements': 1,\n",
       "             u'respective': 1,\n",
       "             u'run': 1,\n",
       "             u'satisfy': 1,\n",
       "             u'select': 1,\n",
       "             u'signatures,': 1,\n",
       "             u'spark-3.0.0-preview2-bin-hadoop2.7.tgz': 1,\n",
       "             u'spark-core_2.11': 1,\n",
       "             u'still': 1,\n",
       "             u'stream,': 1,\n",
       "             u'suggests,': 1,\n",
       "             u'text': 2,\n",
       "             u'that,': 1,\n",
       "             u'the': 9,\n",
       "             u'their': 1,\n",
       "             u'they': 2,\n",
       "             u'this': 1,\n",
       "             u'to': 2,\n",
       "             u'trademarks': 4,\n",
       "             u'trademarks.': 1,\n",
       "             u'type:': 1,\n",
       "             u'under': 1,\n",
       "             u'upcoming': 1,\n",
       "             u'use': 1,\n",
       "             u'using': 1,\n",
       "             u'version': 1,\n",
       "             u'version:': 1,\n",
       "             u'which': 1,\n",
       "             u'will': 2,\n",
       "             u'with': 5,\n",
       "             u'\\xa9': 1})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words.countByValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}