12_ntb_GroupbyWithPySpark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPlZQFZiROMtjovRwYQKOz2",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/CienciaDados/blob/main/12_ntb_GroupbyWithPySpark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "jThZQVJTeVUr"
      },
      "outputs": [],
      "source": [
        "\n",
        "from pyspark.sql import SparkSession\n",
        "import pyspark .sql.functions as F"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Montando Drive\n",
        "\n",
        "-\n"
      ],
      "metadata": {
        "id": "YS7Pd--mev9B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZgZ4YvyR7udB",
        "outputId": "d59e264c-bfff-436d-d9c1-bb1fee79677c"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "spark = (\n",
        "    SparkSession.builder.appName('PySpark - LucianaSampaio')\n",
        "    .config('spark.sql.repl.eagerEval.enabled', True)\n",
        "    .getOrCreate()\n",
        ")"
      ],
      "metadata": {
        "id": "dD7NN2Zq8UaO"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark.read.csv('/content/drive/MyDrive/.Dataset/resume_data.csv', sep=',', encoding='UTF-8', header=True, inferSchema=True)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 688
        },
        "id": "X_oD02gU8egH",
        "outputId": "8f0874eb-156b-4e75-aab2-9ff47b02ecb4"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+----------------+-------------------+--------------+----------------------+--------------------------+------------+------------+-------------+--------------------+--------------------+---------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|             address|    career_objective|              skills|educational_institution_name|        degree_names|   passing_years|educational_results|  result_types|major_field_of_studies|professional_company_names|company_urls| start_dates|    end_dates|related_skils_in_job|           positions|locations|    responsibilities|extra_curricular_activity_types|extra_curricular_organization_names|extra_curricular_organization_links|role_positions|languages|proficiency_levels|certification_providers|certification_skills|online_links|issue_dates|expiry_dates|﻿job_position_name|educationaL_requirements|experiencere_requirement|age_requirement|responsibilities.1|skills_required|matched_score|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+----------------+-------------------+--------------+----------------------+--------------------------+------------+------------+-------------+--------------------+--------------------+---------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|                NULL|Big data analytic...|['Big Data', 'Had...|        ['The Amity Schoo...|          ['B.Tech']|        ['2019']|            ['N/A']|        [None]|       ['Electronics']|             ['Coca-COla']|      [None]|['Nov 2019']|['Till Date']|      [['Big Data']]|['Big Data Analyst']|  ['N/A']|   Technical Support|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Troubleshooting|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|       Collaboration|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|       Documentation|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|   System Monitoring|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "| Software Deployment|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Training & Mentor...|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Industry Trends|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|        Field Visits|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|\",,,,,,,,,,,,Seni...|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Troubleshooting|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|       Collaboration|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|       Documentation|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|   System Monitoring|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "| Software Deployment|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Training & Mentor...|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Industry Trends|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|        Field Visits|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|              ,,0.85|                NULL|                NULL|                        NULL|                NULL|            NULL|               NULL|          NULL|                  NULL|                      NULL|        NULL|        NULL|         NULL|                NULL|                NULL|     NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|Fresher looking t...|['Data Analysis',...|        ['Delhi Universit...|['B.Sc (Maths)', ...|['2015', '2018']|     ['N/A', 'N/A']|['N/A', 'N/A']|  ['Mathematics', '...|       ['BIB Consultancy']|     ['N/A']|['Sep 2019']|['Till Date']|[['Data Analysis'...|['Business Analyst']|  ['N/A']|Machine Learning ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+----------------+-------------------+--------------+----------------------+--------------------------+------------+------------+-------------+--------------------+--------------------+---------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>address</th><th>career_objective</th><th>skills</th><th>educational_institution_name</th><th>degree_names</th><th>passing_years</th><th>educational_results</th><th>result_types</th><th>major_field_of_studies</th><th>professional_company_names</th><th>company_urls</th><th>start_dates</th><th>end_dates</th><th>related_skils_in_job</th><th>positions</th><th>locations</th><th>responsibilities</th><th>extra_curricular_activity_types</th><th>extra_curricular_organization_names</th><th>extra_curricular_organization_links</th><th>role_positions</th><th>languages</th><th>proficiency_levels</th><th>certification_providers</th><th>certification_skills</th><th>online_links</th><th>issue_dates</th><th>expiry_dates</th><th>﻿job_position_name</th><th>educationaL_requirements</th><th>experiencere_requirement</th><th>age_requirement</th><th>responsibilities.1</th><th>skills_required</th><th>matched_score</th></tr>\n",
              "<tr><td>NULL</td><td>Big data analytic...</td><td>['Big Data', 'Had...</td><td>['The Amity Schoo...</td><td>['B.Tech']</td><td>['2019']</td><td>['N/A']</td><td>[None]</td><td>['Electronics']</td><td>['Coca-COla']</td><td>[None]</td><td>['Nov 2019']</td><td>['Till Date']</td><td>[['Big Data']]</td><td>['Big Data Analyst']</td><td>['N/A']</td><td>Technical Support</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Troubleshooting</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Collaboration</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Documentation</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>System Monitoring</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Software Deployment</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Training &amp; Mentor...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Industry Trends</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Field Visits</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>&quot;,,,,,,,,,,,,Seni...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Troubleshooting</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Collaboration</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Documentation</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>System Monitoring</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Software Deployment</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Training &amp; Mentor...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Industry Trends</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Field Visits</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>,,0.85</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>Fresher looking t...</td><td>['Data Analysis',...</td><td>['Delhi Universit...</td><td>['B.Sc (Maths)', ...</td><td>['2015', '2018']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['Mathematics', '...</td><td>['BIB Consultancy']</td><td>['N/A']</td><td>['Sep 2019']</td><td>['Till Date']</td><td>[['Data Analysis'...</td><td>['Business Analyst']</td><td>['N/A']</td><td>Machine Learning ...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-uNA_5EE86qw",
        "outputId": "067a04d6-bc71-4cf1-bd52-8cf8f2d999e2"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- address: string (nullable = true)\n",
            " |-- career_objective: string (nullable = true)\n",
            " |-- skills: string (nullable = true)\n",
            " |-- educational_institution_name: string (nullable = true)\n",
            " |-- degree_names: string (nullable = true)\n",
            " |-- passing_years: string (nullable = true)\n",
            " |-- educational_results: string (nullable = true)\n",
            " |-- result_types: string (nullable = true)\n",
            " |-- major_field_of_studies: string (nullable = true)\n",
            " |-- professional_company_names: string (nullable = true)\n",
            " |-- company_urls: string (nullable = true)\n",
            " |-- start_dates: string (nullable = true)\n",
            " |-- end_dates: string (nullable = true)\n",
            " |-- related_skils_in_job: string (nullable = true)\n",
            " |-- positions: string (nullable = true)\n",
            " |-- locations: string (nullable = true)\n",
            " |-- responsibilities: string (nullable = true)\n",
            " |-- extra_curricular_activity_types: string (nullable = true)\n",
            " |-- extra_curricular_organization_names: string (nullable = true)\n",
            " |-- extra_curricular_organization_links: string (nullable = true)\n",
            " |-- role_positions: string (nullable = true)\n",
            " |-- languages: string (nullable = true)\n",
            " |-- proficiency_levels: string (nullable = true)\n",
            " |-- certification_providers: string (nullable = true)\n",
            " |-- certification_skills: string (nullable = true)\n",
            " |-- online_links: string (nullable = true)\n",
            " |-- issue_dates: string (nullable = true)\n",
            " |-- expiry_dates: string (nullable = true)\n",
            " |-- ﻿job_position_name: string (nullable = true)\n",
            " |-- educationaL_requirements: string (nullable = true)\n",
            " |-- experiencere_requirement: string (nullable = true)\n",
            " |-- age_requirement: string (nullable = true)\n",
            " |-- responsibilities.1: string (nullable = true)\n",
            " |-- skills_required: string (nullable = true)\n",
            " |-- matched_score: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ecmm7fxX9AUX",
        "outputId": "736a7d16-1125-4b3e-c14a-377be5e6a233"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "212008"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.distinct().show()\n",
        "print(\"Distinct Count: \" + str(df.distinct().count()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uxFlwvHA9EGw",
        "outputId": "9811c69e-148b-4f67-f25d-604f06110cdc"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
            "|             address|    career_objective|              skills|educational_institution_name|        degree_names|       passing_years|educational_results|      result_types|major_field_of_studies|professional_company_names|        company_urls|         start_dates|           end_dates|related_skils_in_job|           positions|           locations|    responsibilities|extra_curricular_activity_types|extra_curricular_organization_names|extra_curricular_organization_links|role_positions|languages|proficiency_levels|certification_providers|certification_skills|online_links|issue_dates|expiry_dates|﻿job_position_name|educationaL_requirements|experiencere_requirement|age_requirement|responsibilities.1|skills_required|matched_score|\n",
            "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
            "|         Soft Skills|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|              ,,0.75|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|\",['Award'],['N/A...|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|  Campaign Execution|                SEO\"|        ['Workshop']|        ['Software Lifecy...|              [None]|              [None]|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|An enthusiast, wi...|['Data science', ...|        ['St. Subhash Col...|            ['B.CA']|            ['2020']|            ['N/A']|           ['N/A']|               ['N/A']|      ['BB House of Dev...|             ['N/A']|        ['Nov 2019']|        ['Feb 2020']|[['Machine Learni...|  ['Intern Trainee']|             ['N/A']|   Technical Support|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|          MS Office\"|         0.416666667|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|                NULL|['Adobe Creative ...|        ['Texas A&M Unive...|['Bachelor of Sci...|            ['1996']|            ['N/A']|           ['N/A']|               ['N/A']|      ['Privacy Data Sy...|['N/A', 'N/A', 'N...|['07/2007', '01/2...|['09/2015', '12/2...|[['business devel...|['Director of Bus...|['N/A', 'Corporat...|Data Platform Design|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|                NULL|['biology', 'budg...|        ['Istanbul Techni...|['Bachelor of Sci...|    ['1999', '1994']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Petrochemical a...|      ['Company Name', ...|[None, None, None...|['January 2007', ...|['Current', 'Janu...|[None, None, None...|['N/A', 'Process ...|['N/A', 'N/A', 'N...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|                NULL|['API', 'ATL', 'a...|        ['Illinois Instit...|      ['M.S', 'B.E']|      ['N/A', 'N/A']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Computer Scienc...|      ['Company Name', ...|[None, None, None...|['09/2009', '11/2...|['Current', '08/2...|[['Visual C++', '...|['Software Engine...|['City, State', '...|     Design Creation|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|           Swift UI\"|         0.633333333|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|Adept at extracti...|['Data Analysis',...|        ['Maulana Abul Un...|          ['B.tech']|            ['2019']|            ['N/A']|           ['N/A']|                ['IT']|             ['Tata Sons']|             ['N/A']|        ['Apr 2019']|       ['Till date']|[['Data Analysis'...| ['Analyst Support']|             ['N/A']|Administrative Su...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|Other Assigned Ta...|['Artificial Inte...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|     Executive - VAT|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|                NULL|['Microsoft Proje...|        ['Indiana Univers...|['Master of Busin...|  [None, None, None]| [None, None, None]|[None, None, None]|  [None, 'Electrica...|      ['Company Name', ...|[None, None, None...|['03/2015', '08/2...|['Current', '03/2...|[['Leadership', '...|['Multi-Skilled E...|['City, State', '...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|Passionate, motiv...|['Machine Learnin...|        ['Chadrasekhar Un...|            ['B.Sc']|            ['2018']|             [None]|            [None]|        ['Statistics']|      ['Kinsgley Co.', ...|        [None, None]|['Jul 2019', 'Jun...|['Ongoing', 'May ...|[['Machine Learni...|['Automation Engi...|      ['N/A', 'N/A']|Machinery Mainten...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|                NULL|                NULL|['Budget developm...|        ['WORCESTER STATE...|      ['B.S', 'B.S']|    ['1991', '1987']|    ['3.1', '2.89']|    ['GPA', 'GPA']|  ['Management info...|      ['Company Name', ...|        [None, None]|['01/2000', '01/1...|['Current', '06/2...|        [None, None]|['Engineering Man...|['City , State', ...|Open-Source Techn...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|Other Assigned Ta...|                NULL|                NULL|                        NULL|                NULL|['English', 'Fren...|     ['N/A', 'N/A']|              NULL|                  NULL|                      NULL|                NULL|                NULL|     Executive - VAT|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|  Campaign Execution|                SEO\"|             ['N/A']|                     ['N/A']|              [None]|['Data science on...|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|                NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|     Task Execution\"|['Research Fellow...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|Manager- Human Re...|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "| Team Collaboration\"|['Home Improvemen...|      ['N/A', 'N/A']|                [None, None]|['N/A', 'Particip...|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|         AI Engineer|Bachelors or Mast...|                NULL|                NULL|Machine Learning ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "|Task Assignment &...|['MACHINE LEARNIN...|             ['N/A']|                     ['N/A']|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|                NULL|System Administra...|     Bachelor/Honors|        4 to 5 years|                NULL|Hardware & Networ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
            "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "Distinct Count: 16683\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.dropDuplicates()\n",
        "df.count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A_EeO7yR9jiW",
        "outputId": "2edd183e-462e-4f07-d3d9-04855deb5960"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "16683"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col, to_date\n",
        "\n",
        "# Lista de colunas que precisam ser convertidas\n",
        "date_columns = ['start_dates', 'end_dates', 'issue_dates', 'expiry_dates']\n",
        "\n",
        "# Loop para converter cada coluna em formato de data\n",
        "for date_col in date_columns:\n",
        "    df = df.withColumn(date_col, to_date(col(date_col), \"yyyy-MM-dd\"))\n"
      ],
      "metadata": {
        "id": "_0h9uq9cUQAR"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 879
        },
        "id": "BZUTREQwUeT3",
        "outputId": "3ff60450-fc2f-462d-8f57-065d86a49423"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|             address|    career_objective|              skills|educational_institution_name|        degree_names|       passing_years|educational_results|      result_types|major_field_of_studies|professional_company_names|        company_urls|start_dates|end_dates|related_skils_in_job|           positions|           locations|    responsibilities|extra_curricular_activity_types|extra_curricular_organization_names|extra_curricular_organization_links|role_positions|languages|proficiency_levels|certification_providers|certification_skills|online_links|issue_dates|expiry_dates|﻿job_position_name|educationaL_requirements|experiencere_requirement|age_requirement|responsibilities.1|skills_required|matched_score|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|         Soft Skills|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|              ,,0.75|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|\",['Award'],['N/A...|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|  Campaign Execution|                SEO\"|        ['Workshop']|        ['Software Lifecy...|              [None]|              [None]|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|An enthusiast, wi...|['Data science', ...|        ['St. Subhash Col...|            ['B.CA']|            ['2020']|            ['N/A']|           ['N/A']|               ['N/A']|      ['BB House of Dev...|             ['N/A']|       NULL|     NULL|[['Machine Learni...|  ['Intern Trainee']|             ['N/A']|   Technical Support|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|          MS Office\"|         0.416666667|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Adobe Creative ...|        ['Texas A&M Unive...|['Bachelor of Sci...|            ['1996']|            ['N/A']|           ['N/A']|               ['N/A']|      ['Privacy Data Sy...|['N/A', 'N/A', 'N...|       NULL|     NULL|[['business devel...|['Director of Bus...|['N/A', 'Corporat...|Data Platform Design|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['biology', 'budg...|        ['Istanbul Techni...|['Bachelor of Sci...|    ['1999', '1994']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Petrochemical a...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[None, None, None...|['N/A', 'Process ...|['N/A', 'N/A', 'N...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['API', 'ATL', 'a...|        ['Illinois Instit...|      ['M.S', 'B.E']|      ['N/A', 'N/A']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Computer Scienc...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[['Visual C++', '...|['Software Engine...|['City, State', '...|     Design Creation|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|           Swift UI\"|         0.633333333|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|Adept at extracti...|['Data Analysis',...|        ['Maulana Abul Un...|          ['B.tech']|            ['2019']|            ['N/A']|           ['N/A']|                ['IT']|             ['Tata Sons']|             ['N/A']|       NULL|     NULL|[['Data Analysis'...| ['Analyst Support']|             ['N/A']|Administrative Su...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Other Assigned Ta...|['Artificial Inte...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Microsoft Proje...|        ['Indiana Univers...|['Master of Busin...|  [None, None, None]| [None, None, None]|[None, None, None]|  [None, 'Electrica...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[['Leadership', '...|['Multi-Skilled E...|['City, State', '...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|Passionate, motiv...|['Machine Learnin...|        ['Chadrasekhar Un...|            ['B.Sc']|            ['2018']|             [None]|            [None]|        ['Statistics']|      ['Kinsgley Co.', ...|        [None, None]|       NULL|     NULL|[['Machine Learni...|['Automation Engi...|      ['N/A', 'N/A']|Machinery Mainten...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Budget developm...|        ['WORCESTER STATE...|      ['B.S', 'B.S']|    ['1991', '1987']|    ['3.1', '2.89']|    ['GPA', 'GPA']|  ['Management info...|      ['Company Name', ...|        [None, None]|       NULL|     NULL|        [None, None]|['Engineering Man...|['City , State', ...|Open-Source Techn...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Other Assigned Ta...|                NULL|                NULL|                        NULL|                NULL|['English', 'Fren...|     ['N/A', 'N/A']|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|  Campaign Execution|                SEO\"|             ['N/A']|                     ['N/A']|              [None]|['Data science on...|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Task Execution\"|['Research Fellow...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "| Team Collaboration\"|['Home Improvemen...|      ['N/A', 'N/A']|                [None, None]|['N/A', 'Particip...|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|Bachelors or Mast...|                NULL|                NULL|Machine Learning ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Task Assignment &...|['MACHINE LEARNIN...|             ['N/A']|                     ['N/A']|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|     Bachelor/Honors|        4 to 5 years|                NULL|Hardware & Networ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>address</th><th>career_objective</th><th>skills</th><th>educational_institution_name</th><th>degree_names</th><th>passing_years</th><th>educational_results</th><th>result_types</th><th>major_field_of_studies</th><th>professional_company_names</th><th>company_urls</th><th>start_dates</th><th>end_dates</th><th>related_skils_in_job</th><th>positions</th><th>locations</th><th>responsibilities</th><th>extra_curricular_activity_types</th><th>extra_curricular_organization_names</th><th>extra_curricular_organization_links</th><th>role_positions</th><th>languages</th><th>proficiency_levels</th><th>certification_providers</th><th>certification_skills</th><th>online_links</th><th>issue_dates</th><th>expiry_dates</th><th>﻿job_position_name</th><th>educationaL_requirements</th><th>experiencere_requirement</th><th>age_requirement</th><th>responsibilities.1</th><th>skills_required</th><th>matched_score</th></tr>\n",
              "<tr><td>Soft Skills</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>,,0.75</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>&quot;,['Award'],['N/A...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Campaign Execution</td><td> SEO&quot;</td><td>['Workshop']</td><td>['Software Lifecy...</td><td>[None]</td><td>[None]</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Marketing Officer</td><td>Bachelor/Honors</td><td>At least 1 year</td><td>NULL</td><td>Digital Marketing...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>An enthusiast, wi...</td><td>['Data science', ...</td><td>['St. Subhash Col...</td><td>['B.CA']</td><td>['2020']</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>['BB House of Dev...</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>[['Machine Learni...</td><td>['Intern Trainee']</td><td>['N/A']</td><td>Technical Support</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>MS Office&quot;</td><td>0.416666667</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Adobe Creative ...</td><td>['Texas A&amp;M Unive...</td><td>['Bachelor of Sci...</td><td>['1996']</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>['Privacy Data Sy...</td><td>['N/A', 'N/A', 'N...</td><td>NULL</td><td>NULL</td><td>[['business devel...</td><td>['Director of Bus...</td><td>['N/A', 'Corporat...</td><td>Data Platform Design</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['biology', 'budg...</td><td>['Istanbul Techni...</td><td>['Bachelor of Sci...</td><td>['1999', '1994']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['Petrochemical a...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[None, None, None...</td><td>['N/A', 'Process ...</td><td>['N/A', 'N/A', 'N...</td><td>Supervision</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['API', 'ATL', 'a...</td><td>['Illinois Instit...</td><td>['M.S', 'B.E']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['Computer Scienc...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[['Visual C++', '...</td><td>['Software Engine...</td><td>['City, State', '...</td><td>Design Creation</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Swift UI&quot;</td><td>0.633333333</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>Adept at extracti...</td><td>['Data Analysis',...</td><td>['Maulana Abul Un...</td><td>['B.tech']</td><td>['2019']</td><td>['N/A']</td><td>['N/A']</td><td>['IT']</td><td>['Tata Sons']</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>[['Data Analysis'...</td><td>['Analyst Support']</td><td>['N/A']</td><td>Administrative Su...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Other Assigned Ta...</td><td>['Artificial Inte...</td><td>['N/A']</td><td>[None]</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>BBA in Accounting...</td><td>1 to 3 years</td><td>NULL</td><td>Mushak Forms Main...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Microsoft Proje...</td><td>['Indiana Univers...</td><td>['Master of Busin...</td><td>[None, None, None]</td><td>[None, None, None]</td><td>[None, None, None]</td><td>[None, 'Electrica...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[['Leadership', '...</td><td>['Multi-Skilled E...</td><td>['City, State', '...</td><td>Supervision</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>Passionate, motiv...</td><td>['Machine Learnin...</td><td>['Chadrasekhar Un...</td><td>['B.Sc']</td><td>['2018']</td><td>[None]</td><td>[None]</td><td>['Statistics']</td><td>['Kinsgley Co.', ...</td><td>[None, None]</td><td>NULL</td><td>NULL</td><td>[['Machine Learni...</td><td>['Automation Engi...</td><td>['N/A', 'N/A']</td><td>Machinery Mainten...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Budget developm...</td><td>['WORCESTER STATE...</td><td>['B.S', 'B.S']</td><td>['1991', '1987']</td><td>['3.1', '2.89']</td><td>['GPA', 'GPA']</td><td>['Management info...</td><td>['Company Name', ...</td><td>[None, None]</td><td>NULL</td><td>NULL</td><td>[None, None]</td><td>['Engineering Man...</td><td>['City , State', ...</td><td>Open-Source Techn...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Other Assigned Ta...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>['English', 'Fren...</td><td>['N/A', 'N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>BBA in Accounting...</td><td>1 to 3 years</td><td>NULL</td><td>Mushak Forms Main...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Campaign Execution</td><td> SEO&quot;</td><td>['N/A']</td><td>['N/A']</td><td>[None]</td><td>['Data science on...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Marketing Officer</td><td>Bachelor/Honors</td><td>At least 1 year</td><td>NULL</td><td>Digital Marketing...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Task Execution&quot;</td><td>['Research Fellow...</td><td>['N/A']</td><td>[None]</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Team Collaboration&quot;</td><td>['Home Improvemen...</td><td>['N/A', 'N/A']</td><td>[None, None]</td><td>['N/A', 'Particip...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Bachelors or Mast...</td><td>NULL</td><td>NULL</td><td>Machine Learning ...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Task Assignment &amp;...</td><td>['MACHINE LEARNIN...</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Bachelor/Honors</td><td>4 to 5 years</td><td>NULL</td><td>Hardware &amp; Networ...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.select(\"start_dates\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EQSrwuLGUruU",
        "outputId": "e147658f-46db-4b8a-95da-47b00efc83c2"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+\n",
            "|start_dates|\n",
            "+-----------+\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "|       NULL|\n",
            "+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import when\n",
        "\n",
        "for date_col in date_columns:\n",
        "    df = df.withColumn(\n",
        "        date_col,\n",
        "        when(col(date_col).rlike(\"^[0-9]{4}-[0-9]{2}-[0-9]{2}$\"), to_date(col(date_col), \"yyyy-MM-dd\"))\n",
        "        .when(col(date_col).rlike(\"^[A-Za-z]{3} \\\\d{4}$\"), to_date(col(date_col), \"MMM yyyy\"))\n",
        "        .otherwise(None)\n",
        "    )\n"
      ],
      "metadata": {
        "id": "bder-sHwU6S-"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col\n",
        "\n",
        "\n",
        "df.filter(col(\"start_dates\") == '2019').select(col(\"start_dates\")).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DX6o7fKLU9O6",
        "outputId": "1f9ce2cf-be54-40ad-d117-a8aecc56f988"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+\n",
            "|start_dates|\n",
            "+-----------+\n",
            "+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col\n",
        "\n",
        "df.select(col(\"start_dates\")).filter(col(\"start_dates\") == '2020').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IfgklKZUVfUk",
        "outputId": "89a7b997-f10f-426f-f2e7-dbb820ef1e64"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+\n",
            "|start_dates|\n",
            "+-----------+\n",
            "+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 879
        },
        "id": "8IuqiFkeVBuK",
        "outputId": "dbb0a3bc-49fd-47be-d97b-3c95f1554e85"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|             address|    career_objective|              skills|educational_institution_name|        degree_names|       passing_years|educational_results|      result_types|major_field_of_studies|professional_company_names|        company_urls|start_dates|end_dates|related_skils_in_job|           positions|           locations|    responsibilities|extra_curricular_activity_types|extra_curricular_organization_names|extra_curricular_organization_links|role_positions|languages|proficiency_levels|certification_providers|certification_skills|online_links|issue_dates|expiry_dates|﻿job_position_name|educationaL_requirements|experiencere_requirement|age_requirement|responsibilities.1|skills_required|matched_score|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "|         Soft Skills|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|              ,,0.75|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|\",['Award'],['N/A...|                NULL|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|  Campaign Execution|                SEO\"|        ['Workshop']|        ['Software Lifecy...|              [None]|              [None]|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|An enthusiast, wi...|['Data science', ...|        ['St. Subhash Col...|            ['B.CA']|            ['2020']|            ['N/A']|           ['N/A']|               ['N/A']|      ['BB House of Dev...|             ['N/A']|       NULL|     NULL|[['Machine Learni...|  ['Intern Trainee']|             ['N/A']|   Technical Support|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|          MS Office\"|         0.416666667|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Adobe Creative ...|        ['Texas A&M Unive...|['Bachelor of Sci...|            ['1996']|            ['N/A']|           ['N/A']|               ['N/A']|      ['Privacy Data Sy...|['N/A', 'N/A', 'N...|       NULL|     NULL|[['business devel...|['Director of Bus...|['N/A', 'Corporat...|Data Platform Design|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['biology', 'budg...|        ['Istanbul Techni...|['Bachelor of Sci...|    ['1999', '1994']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Petrochemical a...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[None, None, None...|['N/A', 'Process ...|['N/A', 'N/A', 'N...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['API', 'ATL', 'a...|        ['Illinois Instit...|      ['M.S', 'B.E']|      ['N/A', 'N/A']|     ['N/A', 'N/A']|    ['N/A', 'N/A']|  ['Computer Scienc...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[['Visual C++', '...|['Software Engine...|['City, State', '...|     Design Creation|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|           Swift UI\"|         0.633333333|                NULL|                        NULL|                NULL|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|Adept at extracti...|['Data Analysis',...|        ['Maulana Abul Un...|          ['B.tech']|            ['2019']|            ['N/A']|           ['N/A']|                ['IT']|             ['Tata Sons']|             ['N/A']|       NULL|     NULL|[['Data Analysis'...| ['Analyst Support']|             ['N/A']|Administrative Su...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Other Assigned Ta...|['Artificial Inte...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Microsoft Proje...|        ['Indiana Univers...|['Master of Busin...|  [None, None, None]| [None, None, None]|[None, None, None]|  [None, 'Electrica...|      ['Company Name', ...|[None, None, None...|       NULL|     NULL|[['Leadership', '...|['Multi-Skilled E...|['City, State', '...|         Supervision|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|Passionate, motiv...|['Machine Learnin...|        ['Chadrasekhar Un...|            ['B.Sc']|            ['2018']|             [None]|            [None]|        ['Statistics']|      ['Kinsgley Co.', ...|        [None, None]|       NULL|     NULL|[['Machine Learni...|['Automation Engi...|      ['N/A', 'N/A']|Machinery Mainten...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|                NULL|                NULL|['Budget developm...|        ['WORCESTER STATE...|      ['B.S', 'B.S']|    ['1991', '1987']|    ['3.1', '2.89']|    ['GPA', 'GPA']|  ['Management info...|      ['Company Name', ...|        [None, None]|       NULL|     NULL|        [None, None]|['Engineering Man...|['City , State', ...|Open-Source Techn...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Other Assigned Ta...|                NULL|                NULL|                        NULL|                NULL|['English', 'Fren...|     ['N/A', 'N/A']|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|BBA in Accounting...|        1 to 3 years|                NULL|Mushak Forms Main...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|  Campaign Execution|                SEO\"|             ['N/A']|                     ['N/A']|              [None]|['Data science on...|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|   Marketing Officer|     Bachelor/Honors|     At least 1 year|                NULL|           Digital Marketing...|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|     Task Execution\"|['Research Fellow...|             ['N/A']|                      [None]|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|                NULL|                NULL|                NULL|                NULL|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "| Team Collaboration\"|['Home Improvemen...|      ['N/A', 'N/A']|                [None, None]|['N/A', 'Particip...|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|Bachelors or Mast...|                NULL|                NULL|Machine Learning ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "|Task Assignment &...|['MACHINE LEARNIN...|             ['N/A']|                     ['N/A']|             ['N/A']|                NULL|               NULL|              NULL|                  NULL|                      NULL|                NULL|       NULL|     NULL|     Bachelor/Honors|        4 to 5 years|                NULL|Hardware & Networ...|                           NULL|                               NULL|                               NULL|          NULL|     NULL|              NULL|                   NULL|                NULL|        NULL|       NULL|        NULL|              NULL|                    NULL|                    NULL|           NULL|              NULL|           NULL|         NULL|\n",
              "+--------------------+--------------------+--------------------+----------------------------+--------------------+--------------------+-------------------+------------------+----------------------+--------------------------+--------------------+-----------+---------+--------------------+--------------------+--------------------+--------------------+-------------------------------+-----------------------------------+-----------------------------------+--------------+---------+------------------+-----------------------+--------------------+------------+-----------+------------+------------------+------------------------+------------------------+---------------+------------------+---------------+-------------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>address</th><th>career_objective</th><th>skills</th><th>educational_institution_name</th><th>degree_names</th><th>passing_years</th><th>educational_results</th><th>result_types</th><th>major_field_of_studies</th><th>professional_company_names</th><th>company_urls</th><th>start_dates</th><th>end_dates</th><th>related_skils_in_job</th><th>positions</th><th>locations</th><th>responsibilities</th><th>extra_curricular_activity_types</th><th>extra_curricular_organization_names</th><th>extra_curricular_organization_links</th><th>role_positions</th><th>languages</th><th>proficiency_levels</th><th>certification_providers</th><th>certification_skills</th><th>online_links</th><th>issue_dates</th><th>expiry_dates</th><th>﻿job_position_name</th><th>educationaL_requirements</th><th>experiencere_requirement</th><th>age_requirement</th><th>responsibilities.1</th><th>skills_required</th><th>matched_score</th></tr>\n",
              "<tr><td>Soft Skills</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>,,0.75</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>&quot;,['Award'],['N/A...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Campaign Execution</td><td> SEO&quot;</td><td>['Workshop']</td><td>['Software Lifecy...</td><td>[None]</td><td>[None]</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Marketing Officer</td><td>Bachelor/Honors</td><td>At least 1 year</td><td>NULL</td><td>Digital Marketing...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>An enthusiast, wi...</td><td>['Data science', ...</td><td>['St. Subhash Col...</td><td>['B.CA']</td><td>['2020']</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>['BB House of Dev...</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>[['Machine Learni...</td><td>['Intern Trainee']</td><td>['N/A']</td><td>Technical Support</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>MS Office&quot;</td><td>0.416666667</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Adobe Creative ...</td><td>['Texas A&amp;M Unive...</td><td>['Bachelor of Sci...</td><td>['1996']</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>['Privacy Data Sy...</td><td>['N/A', 'N/A', 'N...</td><td>NULL</td><td>NULL</td><td>[['business devel...</td><td>['Director of Bus...</td><td>['N/A', 'Corporat...</td><td>Data Platform Design</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['biology', 'budg...</td><td>['Istanbul Techni...</td><td>['Bachelor of Sci...</td><td>['1999', '1994']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['Petrochemical a...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[None, None, None...</td><td>['N/A', 'Process ...</td><td>['N/A', 'N/A', 'N...</td><td>Supervision</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['API', 'ATL', 'a...</td><td>['Illinois Instit...</td><td>['M.S', 'B.E']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['N/A', 'N/A']</td><td>['Computer Scienc...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[['Visual C++', '...</td><td>['Software Engine...</td><td>['City, State', '...</td><td>Design Creation</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Swift UI&quot;</td><td>0.633333333</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>Adept at extracti...</td><td>['Data Analysis',...</td><td>['Maulana Abul Un...</td><td>['B.tech']</td><td>['2019']</td><td>['N/A']</td><td>['N/A']</td><td>['IT']</td><td>['Tata Sons']</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>[['Data Analysis'...</td><td>['Analyst Support']</td><td>['N/A']</td><td>Administrative Su...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Other Assigned Ta...</td><td>['Artificial Inte...</td><td>['N/A']</td><td>[None]</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>BBA in Accounting...</td><td>1 to 3 years</td><td>NULL</td><td>Mushak Forms Main...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Microsoft Proje...</td><td>['Indiana Univers...</td><td>['Master of Busin...</td><td>[None, None, None]</td><td>[None, None, None]</td><td>[None, None, None]</td><td>[None, 'Electrica...</td><td>['Company Name', ...</td><td>[None, None, None...</td><td>NULL</td><td>NULL</td><td>[['Leadership', '...</td><td>['Multi-Skilled E...</td><td>['City, State', '...</td><td>Supervision</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>Passionate, motiv...</td><td>['Machine Learnin...</td><td>['Chadrasekhar Un...</td><td>['B.Sc']</td><td>['2018']</td><td>[None]</td><td>[None]</td><td>['Statistics']</td><td>['Kinsgley Co.', ...</td><td>[None, None]</td><td>NULL</td><td>NULL</td><td>[['Machine Learni...</td><td>['Automation Engi...</td><td>['N/A', 'N/A']</td><td>Machinery Mainten...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>NULL</td><td>NULL</td><td>['Budget developm...</td><td>['WORCESTER STATE...</td><td>['B.S', 'B.S']</td><td>['1991', '1987']</td><td>['3.1', '2.89']</td><td>['GPA', 'GPA']</td><td>['Management info...</td><td>['Company Name', ...</td><td>[None, None]</td><td>NULL</td><td>NULL</td><td>[None, None]</td><td>['Engineering Man...</td><td>['City , State', ...</td><td>Open-Source Techn...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Other Assigned Ta...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>['English', 'Fren...</td><td>['N/A', 'N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>BBA in Accounting...</td><td>1 to 3 years</td><td>NULL</td><td>Mushak Forms Main...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Campaign Execution</td><td> SEO&quot;</td><td>['N/A']</td><td>['N/A']</td><td>[None]</td><td>['Data science on...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Marketing Officer</td><td>Bachelor/Honors</td><td>At least 1 year</td><td>NULL</td><td>Digital Marketing...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Task Execution&quot;</td><td>['Research Fellow...</td><td>['N/A']</td><td>[None]</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Team Collaboration&quot;</td><td>['Home Improvemen...</td><td>['N/A', 'N/A']</td><td>[None, None]</td><td>['N/A', 'Particip...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Bachelors or Mast...</td><td>NULL</td><td>NULL</td><td>Machine Learning ...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "<tr><td>Task Assignment &amp;...</td><td>['MACHINE LEARNIN...</td><td>['N/A']</td><td>['N/A']</td><td>['N/A']</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>Bachelor/Honors</td><td>4 to 5 years</td><td>NULL</td><td>Hardware &amp; Networ...</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td><td>NULL</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}