09_ComandosPySpark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/data-engineering/blob/main/9_ComandosPySpark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "NUvuS6Jhsofb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Instalando o PySpark**"
      ],
      "metadata": {
        "id": "MKnM9UWSNEwF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "55LYoRrmwnrW",
        "outputId": "43c605ec-b435-4f49-b3ed-b3373b8a329f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.2.1.tar.gz (281.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.4 MB 35 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.3\n",
            "  Downloading py4j-0.10.9.3-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 56.6 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.2.1-py2.py3-none-any.whl size=281853642 sha256=af8369bba16c2ddaafba1b1a91d935f8cc0fe32e9c6caeabf37969f288f0d45f\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/f5/07/7cd8017084dce4e93e84e92efd1e1d5334db05f2e83bcef74f\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.3 pyspark-3.2.1\n"
          ]
        }
      ],
      "source": [
        "# Instalando o PySpark\n",
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Tratamento dos Daodos**"
      ],
      "metadata": {
        "id": "r0-F1toVNJ11"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Importando a Lib\n",
        "from pyspark.sql import SparkSession\n",
        "\n",
        "# Ignorando avisos\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "k060uqQwwsxE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Criando a seção do Spark (API)\n",
        "Spark = SparkSession.builder.appName('Dataframe').getOrCreate()\n",
        "\n",
        "# Verificando\n",
        "Spark"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "id": "VLbfbBE6zYDw",
        "outputId": "ca047728-cdfc-43d0-8ac1-62d279142f7f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f369b50be10>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://07ff874420cf:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.2.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Dataframe</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Lendo um Arquivo CSV\n",
        "Base_Dados_Spark = Spark.read.csv( 'Dados_TesteSpark.csv', header=True, inferSchema=True, sep=';' )"
      ],
      "metadata": {
        "id": "or-hkZ3mznfj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Verificando a estrutura da Base\n",
        "Base_Dados_Spark.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WSa9-iB2z5Is",
        "outputId": "01251611-0711-4032-ad30-d8f6b095f73b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Numero_01: integer (nullable = true)\n",
            " |-- Numero_02: integer (nullable = true)\n",
            " |-- Numero_03: integer (nullable = true)\n",
            " |-- Numero_04: integer (nullable = true)\n",
            " |-- Classe: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Verificando o tipo da informação\n",
        "type( Base_Dados_Spark )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E8nML-zN0DRd",
        "outputId": "67fb6bc9-f69a-4cc1-ec35-446eb31d87b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.sql.dataframe.DataFrame"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Verificando primeiros registros\n",
        "Base_Dados_Spark.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BVFTMTPg0H8H",
        "outputId": "1f73165d-cc38-41f2-a59c-b20c32daee2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Numero_01=316, Numero_02=706, Numero_03=617, Numero_04=803, Classe=1),\n",
              " Row(Numero_01=306, Numero_02=580, Numero_03=452, Numero_04=91, Classe=1),\n",
              " Row(Numero_01=705, Numero_02=628, Numero_03=4, Numero_04=665, Classe=1),\n",
              " Row(Numero_01=303, Numero_02=48, Numero_03=746, Numero_04=956, Classe=2),\n",
              " Row(Numero_01=39, Numero_02=489, Numero_03=89, Numero_04=702, Classe=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Mostrando os valores\n",
        "Base_Dados_Spark.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3_9uqel50Lx6",
        "outputId": "a7fb87dd-5dcb-4ec7-f22a-cff7d9c80a36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      303|       48|      746|      956|     2|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      791|      136|      974|       48|     3|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      723|      880|       88|      357|     2|\n",
            "|      787|       81|      412|      793|     2|\n",
            "|      221|      477|      568|      330|     2|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      146|        4|      224|       27|     3|\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      396|      812|      678|      700|     3|\n",
            "|      754|      635|        9|      679|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      151|       98|      925|      262|     3|\n",
            "|      691|      635|      756|       46|     2|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Selecionado algumas colunas\n",
        "Base_Dados_Spark.select( ['Numero_01','Numero_02'] ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Umiroola0Ql2",
        "outputId": "c567593c-5ca4-45d1-8249-2ed2b5442340"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+\n",
            "|Numero_01|Numero_02|\n",
            "+---------+---------+\n",
            "|      316|      706|\n",
            "|      306|      580|\n",
            "|      705|      628|\n",
            "|      303|       48|\n",
            "|       39|      489|\n",
            "|      791|      136|\n",
            "|      874|      931|\n",
            "|      723|      880|\n",
            "|      787|       81|\n",
            "|      221|      477|\n",
            "|      726|      430|\n",
            "|      146|        4|\n",
            "|      982|      140|\n",
            "|      553|      420|\n",
            "|      281|      555|\n",
            "|      396|      812|\n",
            "|      754|      635|\n",
            "|      970|      152|\n",
            "|      151|       98|\n",
            "|      691|      635|\n",
            "+---------+---------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Veriricando a coluna\n",
        "Base_Dados_Spark['Numero_01']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IlnCmjON0bXU",
        "outputId": "9b2d00cf-2ef6-4e98-e990-d04607c8f9e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Column<'Numero_01'>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Analisando a esutrura dos tipos dos dados\n",
        "Base_Dados_Spark.dtypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2QbSFN9q00Xe",
        "outputId": "9ee80c9f-d018-4b96-a9ee-2a4a02db45eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Numero_01', 'int'),\n",
              " ('Numero_02', 'int'),\n",
              " ('Numero_03', 'int'),\n",
              " ('Numero_04', 'int'),\n",
              " ('Classe', 'int')]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Verificando dados estatisticos\n",
        "Base_Dados_Spark.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M7McSMyY04ip",
        "outputId": "1d99bbeb-3d48-4ac5-f946-859271716aef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+-----------------+------------------+----------------+------------------+\n",
            "|summary|        Numero_01|        Numero_02|         Numero_03|       Numero_04|            Classe|\n",
            "+-------+-----------------+-----------------+------------------+----------------+------------------+\n",
            "|  count|              100|              100|                98|             100|               100|\n",
            "|   mean|           491.27|           514.68|454.66326530612247|           539.3|              2.02|\n",
            "| stddev|279.5811750357013|288.2428437441197|272.47744421460413|304.324139394465|0.7518327774528889|\n",
            "|    min|                8|                4|                 4|              24|                 1|\n",
            "|    max|              998|              995|               974|             999|                 3|\n",
            "+-------+-----------------+-----------------+------------------+----------------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Adicionando uma coluna\n",
        "Base_Dados_Spark = Base_Dados_Spark.withColumn('Numero_05', Base_Dados_Spark['Numero_04'] + 2 )\n",
        "\n",
        "# Verificando\n",
        "Base_Dados_Spark.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OEHCQzUr0-Mu",
        "outputId": "0ab054dd-43a3-4426-f392-8fa608230fae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Numero_01=316, Numero_02=706, Numero_03=617, Numero_04=803, Classe=1, Numero_05=805),\n",
              " Row(Numero_01=306, Numero_02=580, Numero_03=452, Numero_04=91, Classe=1, Numero_05=93),\n",
              " Row(Numero_01=705, Numero_02=628, Numero_03=4, Numero_04=665, Classe=1, Numero_05=667),\n",
              " Row(Numero_01=303, Numero_02=48, Numero_03=746, Numero_04=956, Classe=2, Numero_05=958),\n",
              " Row(Numero_01=39, Numero_02=489, Numero_03=89, Numero_04=702, Classe=1, Numero_05=704)]"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Excluindo a coluna\n",
        "Base_Dados_Spark = Base_Dados_Spark.drop('Numero_05')\n",
        "\n",
        "# Veriricando\n",
        "Base_Dados_Spark.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "80LbzW0A1POQ",
        "outputId": "42228682-0a9b-46a1-e264-6df8c1e8b6c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Numero_01=316, Numero_02=706, Numero_03=617, Numero_04=803, Classe=1),\n",
              " Row(Numero_01=306, Numero_02=580, Numero_03=452, Numero_04=91, Classe=1),\n",
              " Row(Numero_01=705, Numero_02=628, Numero_03=4, Numero_04=665, Classe=1),\n",
              " Row(Numero_01=303, Numero_02=48, Numero_03=746, Numero_04=956, Classe=2),\n",
              " Row(Numero_01=39, Numero_02=489, Numero_03=89, Numero_04=702, Classe=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Renomeando as colunas\n",
        "Base_Dados_Spark.withColumnRenamed( 'Numero_01', 'Coluna_01').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qakw7abC1ap4",
        "outputId": "368da35d-c3fe-4b10-b055-f154a8e50bb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Coluna_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      303|       48|      746|      956|     2|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      791|      136|      974|       48|     3|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      723|      880|       88|      357|     2|\n",
            "|      787|       81|      412|      793|     2|\n",
            "|      221|      477|      568|      330|     2|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      146|        4|      224|       27|     3|\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      396|      812|      678|      700|     3|\n",
            "|      754|      635|        9|      679|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      151|       98|      925|      262|     3|\n",
            "|      691|      635|      756|       46|     2|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Retornar eliminado as linhas vazias/nulas\n",
        "Base_Dados_Spark.na.drop().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TF4pV_ge1ixe",
        "outputId": "638b8241-24ba-404e-da8f-b5188646764f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      303|       48|      746|      956|     2|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      791|      136|      974|       48|     3|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      723|      880|       88|      357|     2|\n",
            "|      787|       81|      412|      793|     2|\n",
            "|      221|      477|      568|      330|     2|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      146|        4|      224|       27|     3|\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      396|      812|      678|      700|     3|\n",
            "|      754|      635|        9|      679|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      151|       98|      925|      262|     3|\n",
            "|      691|      635|      756|       46|     2|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# # Retornar eliminado as linhas vazias/nulas de uma coluna especifica\n",
        "Base_Dados_Spark.na.drop(how='any', subset=['Numero_02']).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NMZRnQg52a3c",
        "outputId": "d144073c-c083-4127-c748-d154ce771f70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      303|       48|      746|      956|     2|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      791|      136|      974|       48|     3|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      723|      880|       88|      357|     2|\n",
            "|      787|       81|      412|      793|     2|\n",
            "|      221|      477|      568|      330|     2|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      146|        4|      224|       27|     3|\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      396|      812|      678|      700|     3|\n",
            "|      754|      635|        9|      679|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      151|       98|      925|      262|     3|\n",
            "|      691|      635|      756|       46|     2|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Preenchendo o valor ausente\n",
        "Base_Dados_Spark.na.fill('Qualquer_Valor', ['Numero_01','Numero_02'] ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "In9WoShp3FyE",
        "outputId": "4648b6f7-86d1-400b-e695-705161621ed1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      303|       48|      746|      956|     2|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      791|      136|      974|       48|     3|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      723|      880|       88|      357|     2|\n",
            "|      787|       81|      412|      793|     2|\n",
            "|      221|      477|      568|      330|     2|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      146|        4|      224|       27|     3|\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      396|      812|      678|      700|     3|\n",
            "|      754|      635|        9|      679|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      151|       98|      925|      262|     3|\n",
            "|      691|      635|      756|       46|     2|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Função para inserir\n",
        "from pyspark.ml.feature import Imputer\n",
        "\n",
        "# Estimador de imputação para completar valores faltantes, usando a média, mediana ou moda\n",
        "# Parametros para preencher [ mean, median or mode ]\n",
        "Inserir = Imputer(\n",
        "  inputCols=['Numero_01', 'Numero_02', 'Numero_03', 'Numero_04'],\n",
        "  outputCols=[ '{}_imputed'.format(Loop) for Loop in ['Numero_01', 'Numero_02', 'Numero_03', 'Numero_04'] ]\n",
        "  ).setStrategy('mean')"
      ],
      "metadata": {
        "id": "CxVIlXVI32DJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Adiciona colunas de imputação ao df\n",
        "Inserir.fit( Base_Dados_Spark ).transform( Base_Dados_Spark ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V5Go4cUO4QgH",
        "outputId": "5113a57c-371f-441d-c297-776b42c58d61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+-----------------+-----------------+-----------------+-----------------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|Numero_01_imputed|Numero_02_imputed|Numero_03_imputed|Numero_04_imputed|\n",
            "+---------+---------+---------+---------+------+-----------------+-----------------+-----------------+-----------------+\n",
            "|      316|      706|      617|      803|     1|              316|              706|              617|              803|\n",
            "|      306|      580|      452|       91|     1|              306|              580|              452|               91|\n",
            "|      705|      628|        4|      665|     1|              705|              628|                4|              665|\n",
            "|      303|       48|      746|      956|     2|              303|               48|              746|              956|\n",
            "|       39|      489|       89|      702|     1|               39|              489|               89|              702|\n",
            "|      791|      136|      974|       48|     3|              791|              136|              974|               48|\n",
            "|      874|      931|      386|      963|     1|              874|              931|              386|              963|\n",
            "|      723|      880|       88|      357|     2|              723|              880|               88|              357|\n",
            "|      787|       81|      412|      793|     2|              787|               81|              412|              793|\n",
            "|      221|      477|      568|      330|     2|              221|              477|              568|              330|\n",
            "|      726|      430|      434|      632|     1|              726|              430|              434|              632|\n",
            "|      146|        4|      224|       27|     3|              146|                4|              224|               27|\n",
            "|      982|      140|       47|      878|     2|              982|              140|               47|              878|\n",
            "|      553|      420|      184|      519|     1|              553|              420|              184|              519|\n",
            "|      281|      555|      379|      862|     1|              281|              555|              379|              862|\n",
            "|      396|      812|      678|      700|     3|              396|              812|              678|              700|\n",
            "|      754|      635|        9|      679|     2|              754|              635|                9|              679|\n",
            "|      970|      152|      191|      984|     3|              970|              152|              191|              984|\n",
            "|      151|       98|      925|      262|     3|              151|               98|              925|              262|\n",
            "|      691|      635|      756|       46|     2|              691|              635|              756|               46|\n",
            "+---------+---------+---------+---------+------+-----------------+-----------------+-----------------+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Filtrar um valor\n",
        "Base_Dados_Spark.filter('Numero_01 >= 900 ').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "30QTCfx94bSe",
        "outputId": "066e3fbc-2d8c-423d-dffa-f732e3e12505"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      982|      140|       47|      878|     2|\n",
            "|      970|      152|      191|      984|     3|\n",
            "|      980|      924|      193|      895|     2|\n",
            "|      938|      237|      333|      830|     2|\n",
            "|      916|      229|      334|      557|     3|\n",
            "|      968|      143|      458|      666|     2|\n",
            "|      998|      856|       74|      464|     2|\n",
            "|      973|      389|      102|      389|     1|\n",
            "+---------+---------+---------+---------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Filtrar um valor e colunas\n",
        "Base_Dados_Spark.filter('Numero_01 >= 1000 ').select(['Numero_01', 'Numero_02']).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aAPKuyji5Ocz",
        "outputId": "19a2efbe-4ca3-4640-e2a8-02602bdd899e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+\n",
            "|Numero_01|Numero_02|\n",
            "+---------+---------+\n",
            "+---------+---------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Filtrando usando o and\n",
        "Base_Dados_Spark.filter(\n",
        "    ( Base_Dados_Spark['Numero_02'] == 1000) &\n",
        "    (Base_Dados_Spark['Numero_01'] == 494 ) ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NMQSo-FH5YRE",
        "outputId": "f3024c27-c649-45e4-dadc-75e0258cd5b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "+---------+---------+---------+---------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Filtrando usando o OR\n",
        "Base_Dados_Spark.filter(\n",
        "    ( Base_Dados_Spark['Numero_02'] == 1000) |\n",
        "    (Base_Dados_Spark['Numero_01'] == 494 ) ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-NeYqmOz5kRd",
        "outputId": "25b9e534-afab-443f-d453-8c4366dc2991"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "+---------+---------+---------+---------+------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agrupando valores usando a Soma\n",
        "Base_Dados_Spark.groupBy('Classe').sum().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVQrMcVm6FNf",
        "outputId": "aa0169ed-ce68-4fb0-dd2d-b6da405a3a29"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------+--------------+--------------+--------------+-----------+\n",
            "|Classe|sum(Numero_01)|sum(Numero_02)|sum(Numero_03)|sum(Numero_04)|sum(Classe)|\n",
            "+------+--------------+--------------+--------------+--------------+-----------+\n",
            "|     1|         12943|         13147|         11978|         15180|         27|\n",
            "|     3|         14077|         14180|         13601|         16782|         87|\n",
            "|     2|         22107|         24141|         18978|         21968|         88|\n",
            "+------+--------------+--------------+--------------+--------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agrupando valores usando a Media\n",
        "Base_Dados_Spark.groupBy('Classe').avg().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmKlDDTX6RGu",
        "outputId": "852d50a5-3cdb-4147-b720-b140630eb15c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "|Classe|    avg(Numero_01)|   avg(Numero_02)|   avg(Numero_03)|    avg(Numero_04)|avg(Classe)|\n",
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "|     1| 479.3703703703704|486.9259259259259|460.6923076923077| 562.2222222222222|        1.0|\n",
            "|     3|485.41379310344826|488.9655172413793|            469.0| 578.6896551724138|        3.0|\n",
            "|     2| 502.4318181818182|548.6590909090909|441.3488372093023|499.27272727272725|        2.0|\n",
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agrupando valores usando a Media\n",
        "Base_Dados_Spark.groupBy('Classe').mean().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ifS4ZMSi6d6k",
        "outputId": "de8aa20a-f7b9-4330-d35d-3b10f7907b00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "|Classe|    avg(Numero_01)|   avg(Numero_02)|   avg(Numero_03)|    avg(Numero_04)|avg(Classe)|\n",
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "|     1| 479.3703703703704|486.9259259259259|460.6923076923077| 562.2222222222222|        1.0|\n",
            "|     3|485.41379310344826|488.9655172413793|            469.0| 578.6896551724138|        3.0|\n",
            "|     2| 502.4318181818182|548.6590909090909|441.3488372093023|499.27272727272725|        2.0|\n",
            "+------+------------------+-----------------+-----------------+------------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agrupando valores contando\n",
        "Base_Dados_Spark.groupBy('Classe').count().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "djbGlrvs6mC7",
        "outputId": "a39ada6f-b2d9-4b13-9060-c46ced06ea6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-----+\n",
            "|Classe|count|\n",
            "+------+-----+\n",
            "|     1|   27|\n",
            "|     3|   29|\n",
            "|     2|   44|\n",
            "+------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Agrupando o argumento agg\n",
        "Base_Dados_Spark.agg( {'Classe':'count'} ).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wbg_2ixQ6p-C",
        "outputId": "4ff0aaef-4804-417b-d524-89bce64a21d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+\n",
            "|count(Classe)|\n",
            "+-------------+\n",
            "|          100|\n",
            "+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Verificando as colunas\n",
        "Base_Dados_Spark.columns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oh4QLPxo6w0R",
        "outputId": "6e5e248e-09dd-4b16-fdfc-ff3f58eaa95f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Numero_01', 'Numero_02', 'Numero_03', 'Numero_04', 'Classe']"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Realizando um JOIN\n",
        "\n",
        "# Divindindo os dados em 2 partes\n",
        "Parte_1 = Base_Dados_Spark.filter( ( Base_Dados_Spark['Numero_02'] > 500 ) )\n",
        "Parte_2 = Base_Dados_Spark.filter( ( Base_Dados_Spark['Numero_02'] <= 500 ) )\n",
        "\n",
        "# Aplicando o comnado\n",
        "Base_Join = Parte_1.join(Parte_2, Parte_1.Classe == Parte_2.Classe, 'inner')\n",
        "\n",
        "Base_Join.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1u0JCGt6xXD",
        "outputId": "9a772744-cbad-492d-eabe-601dff30d5e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+---------+---------+---------+---------+------+\n",
            "|      316|      706|      617|      803|     1|      355|      355|      355|      355|     1|\n",
            "|      316|      706|      617|      803|     1|      973|      389|      102|      389|     1|\n",
            "|      316|      706|      617|      803|     1|      833|      176|     null|      151|     1|\n",
            "|      316|      706|      617|      803|     1|      580|      372|       23|      507|     1|\n",
            "|      316|      706|      617|      803|     1|      427|      378|      798|      810|     1|\n",
            "|      316|      706|      617|      803|     1|      526|      207|      517|      887|     1|\n",
            "|      316|      706|      617|      803|     1|      737|      351|      660|      450|     1|\n",
            "|      316|      706|      617|      803|     1|      171|      327|      242|      754|     1|\n",
            "|      316|      706|      617|      803|     1|      216|       57|      829|      788|     1|\n",
            "|      316|      706|      617|      803|     1|      366|       31|      182|      507|     1|\n",
            "|      316|      706|      617|      803|     1|      518|      107|      840|       54|     1|\n",
            "|      316|      706|      617|      803|     1|      553|      420|      184|      519|     1|\n",
            "|      316|      706|      617|      803|     1|      726|      430|      434|      632|     1|\n",
            "|      316|      706|      617|      803|     1|       39|      489|       89|      702|     1|\n",
            "|      306|      580|      452|       91|     1|      355|      355|      355|      355|     1|\n",
            "|      306|      580|      452|       91|     1|      973|      389|      102|      389|     1|\n",
            "|      306|      580|      452|       91|     1|      833|      176|     null|      151|     1|\n",
            "|      306|      580|      452|       91|     1|      580|      372|       23|      507|     1|\n",
            "|      306|      580|      452|       91|     1|      427|      378|      798|      810|     1|\n",
            "|      306|      580|      452|       91|     1|      526|      207|      517|      887|     1|\n",
            "+---------+---------+---------+---------+------+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Ordenando por coluna\n",
        "Base_Dados_Spark.sort('Classe').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MWuzijCf8UZ9",
        "outputId": "7d198e07-d491-43e3-d57a-5c875c7d953b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|\n",
            "+---------+---------+---------+---------+------+\n",
            "|      518|      107|      840|       54|     1|\n",
            "|      737|      351|      660|      450|     1|\n",
            "|       19|      667|      480|       65|     1|\n",
            "|      306|      580|      452|       91|     1|\n",
            "|       39|      489|       89|      702|     1|\n",
            "|      874|      931|      386|      963|     1|\n",
            "|      726|      430|      434|      632|     1|\n",
            "|      553|      420|      184|      519|     1|\n",
            "|      281|      555|      379|      862|     1|\n",
            "|      316|      706|      617|      803|     1|\n",
            "|      705|      628|        4|      665|     1|\n",
            "|      366|       31|      182|      507|     1|\n",
            "|      363|      733|      560|      760|     1|\n",
            "|      734|      721|      619|      201|     1|\n",
            "|      216|       57|      829|      788|     1|\n",
            "|       48|      838|      518|      936|     1|\n",
            "|      171|      327|      242|      754|     1|\n",
            "|      580|      516|       51|      805|     1|\n",
            "|      526|      207|      517|      887|     1|\n",
            "|      552|      790|      956|      511|     1|\n",
            "+---------+---------+---------+---------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Pré-Processamento - MinMax**"
      ],
      "metadata": {
        "id": "n-P155K2N0Oy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Função de Vetor\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Selecionado os dados\n",
        "Amostra = Base_Dados_Spark\n",
        "\n",
        "# Vetorizando os dados\n",
        "Dados_Amostra = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03', 'Numero_04', 'Classe'],\n",
        "    outputCol='Correlação' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Dados_Amostral = Dados_Amostra.transform( Amostra )\n",
        "\n",
        "# Função da regressão Linear\n",
        "from pyspark.ml.feature import MinMaxScaler\n",
        "\n",
        "# Definindo os parametros da função\n",
        "Funcao_MinMax = MinMaxScaler(outputCol='Valor_Esclada_MixMAx')\n",
        "\n",
        "# Setando a coluna para ser aplicado\n",
        "Funcao_MinMax.setInputCol('Correlação')\n",
        "\n",
        "# Fitando o modelo\n",
        "Modelo_MinMax = Funcao_MinMax.fit( Dados_Amostral )\n",
        "\n",
        "Modelo_MinMax.transform(Dados_Amostral).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e4lDCwAHLr_D",
        "outputId": "cfee3de6-4982-4e94-b6a5-90c7cbc813c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+--------------------+--------------------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|          Correlação|Valor_Esclada_MixMAx|\n",
            "+---------+---------+---------+---------+------+--------------------+--------------------+\n",
            "|      316|      706|      617|      803|     1|[316.0,706.0,617....|[0.31111111111111...|\n",
            "|      306|      580|      452|       91|     1|[306.0,580.0,452....|[0.30101010101010...|\n",
            "|      705|      628|        4|      665|     1|[705.0,628.0,4.0,...|[0.70404040404040...|\n",
            "|      303|       48|      746|      956|     2|[303.0,48.0,746.0...|[0.29797979797979...|\n",
            "|       39|      489|       89|      702|     1|[39.0,489.0,89.0,...|[0.03131313131313...|\n",
            "|      791|      136|      974|       48|     3|[791.0,136.0,974....|[0.79090909090909...|\n",
            "|      874|      931|      386|      963|     1|[874.0,931.0,386....|[0.87474747474747...|\n",
            "|      723|      880|       88|      357|     2|[723.0,880.0,88.0...|[0.72222222222222...|\n",
            "|      787|       81|      412|      793|     2|[787.0,81.0,412.0...|[0.78686868686868...|\n",
            "|      221|      477|      568|      330|     2|[221.0,477.0,568....|[0.21515151515151...|\n",
            "|      726|      430|      434|      632|     1|[726.0,430.0,434....|[0.72525252525252...|\n",
            "|      146|        4|      224|       27|     3|[146.0,4.0,224.0,...|[0.13939393939393...|\n",
            "|      982|      140|       47|      878|     2|[982.0,140.0,47.0...|[0.98383838383838...|\n",
            "|      553|      420|      184|      519|     1|[553.0,420.0,184....|[0.55050505050505...|\n",
            "|      281|      555|      379|      862|     1|[281.0,555.0,379....|[0.27575757575757...|\n",
            "|      396|      812|      678|      700|     3|[396.0,812.0,678....|[0.39191919191919...|\n",
            "|      754|      635|        9|      679|     2|[754.0,635.0,9.0,...|[0.75353535353535...|\n",
            "|      970|      152|      191|      984|     3|[970.0,152.0,191....|[0.97171717171717...|\n",
            "|      151|       98|      925|      262|     3|[151.0,98.0,925.0...|[0.14444444444444...|\n",
            "|      691|      635|      756|       46|     2|[691.0,635.0,756....|[0.68989898989898...|\n",
            "+---------+---------+---------+---------+------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Correlação**"
      ],
      "metadata": {
        "id": "2mhyC8h0L5hy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Calcurando a Correlação\n",
        "\n",
        "# Gerando uma amostra devido termos +1mm de registros\n",
        "Amostra = Base_Dados_Spark\n",
        "\n",
        "# Vetorizando os dados\n",
        "Dados_Correlacao = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03', 'Numero_04', 'Classe'],\n",
        "    outputCol='Correlação' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Saida_Correlcao = Dados_Correlacao.transform( Amostra )\n",
        "\n",
        "# Função da Correlação\n",
        "from pyspark.ml.stat import Correlation\n",
        "\n",
        "# Aplicando a Função da correlação de Pearson\n",
        "Correlacao_Pearson = Correlation.corr( Saida_Correlcao, 'Correlação', 'pearson').collect()\n",
        "# Plotando o valor\n",
        "print( str(Correlacao_Pearson).replace('nan', 'NaN') )\n",
        "\n",
        "# Aplicando a Função da correlação de Spearman\n",
        "Correlacao_Spearman = Correlation.corr(Saida_Correlcao, 'Correlação', method='spearman').collect()[0][0]\n",
        "# Plotando o valor\n",
        "print( '\\n', str(Correlacao_Spearman).replace('nan', 'NaN') );"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IoiD3peIE99G",
        "outputId": "ed09772e-9c32-4c08-8c09-a370c7729c75"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Row(pearson(Correlação)=DenseMatrix(5, 5, [1.0, -0.0734, -0.163, 0.074, 0.0005, -0.0734, 1.0, 0.1117, ..., -0.1151, 1.0, 0.0149, 0.0005, -0.0076, 0.0191, 0.0149, 1.0], False))]\n",
            "\n",
            " DenseMatrix([[ 1.00000000e+00, -6.45503431e-02, -1.62613794e-01,\n",
            "               7.04068128e-02,  4.80629708e-03],\n",
            "             [-6.45503431e-02,  1.00000000e+00,  1.26782296e-01,\n",
            "               1.05897232e-01, -7.16473292e-04],\n",
            "             [-1.62613794e-01,  1.26782296e-01,  1.00000000e+00,\n",
            "              -9.97495284e-02,  8.78672168e-03],\n",
            "             [ 7.04068128e-02,  1.05897232e-01, -9.97495284e-02,\n",
            "               1.00000000e+00,  2.28341986e-02],\n",
            "             [ 4.80629708e-03, -7.16473292e-04,  8.78672168e-03,\n",
            "               2.28341986e-02,  1.00000000e+00]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Regressão Linear**"
      ],
      "metadata": {
        "id": "HhA8povGMD5A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Regressão Linear\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Separar os dados de caracteristcas e previsor\n",
        "Selecao_Dados = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03'], outputCol='Variaveis_Caracteristicas' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Saida = Selecao_Dados.transform( Base_Dados_Spark )\n",
        "\n",
        "# Veriricando\n",
        "Saida.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xT_YyZO79gLL",
        "outputId": "2e04211c-35d0-46b1-a39e-27301700fa7f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|Variaveis_Caracteristicas|\n",
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "|      316|      706|      617|      803|     1|      [316.0,706.0,617.0]|\n",
            "|      306|      580|      452|       91|     1|      [306.0,580.0,452.0]|\n",
            "|      705|      628|        4|      665|     1|        [705.0,628.0,4.0]|\n",
            "|      303|       48|      746|      956|     2|       [303.0,48.0,746.0]|\n",
            "|       39|      489|       89|      702|     1|        [39.0,489.0,89.0]|\n",
            "|      791|      136|      974|       48|     3|      [791.0,136.0,974.0]|\n",
            "|      874|      931|      386|      963|     1|      [874.0,931.0,386.0]|\n",
            "|      723|      880|       88|      357|     2|       [723.0,880.0,88.0]|\n",
            "|      787|       81|      412|      793|     2|       [787.0,81.0,412.0]|\n",
            "|      221|      477|      568|      330|     2|      [221.0,477.0,568.0]|\n",
            "|      726|      430|      434|      632|     1|      [726.0,430.0,434.0]|\n",
            "|      146|        4|      224|       27|     3|        [146.0,4.0,224.0]|\n",
            "|      982|      140|       47|      878|     2|       [982.0,140.0,47.0]|\n",
            "|      553|      420|      184|      519|     1|      [553.0,420.0,184.0]|\n",
            "|      281|      555|      379|      862|     1|      [281.0,555.0,379.0]|\n",
            "|      396|      812|      678|      700|     3|      [396.0,812.0,678.0]|\n",
            "|      754|      635|        9|      679|     2|        [754.0,635.0,9.0]|\n",
            "|      970|      152|      191|      984|     3|      [970.0,152.0,191.0]|\n",
            "|      151|       98|      925|      262|     3|       [151.0,98.0,925.0]|\n",
            "|      691|      635|      756|       46|     2|      [691.0,635.0,756.0]|\n",
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Veriricando\n",
        "Saida.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hRxU8ukWE0et",
        "outputId": "4f832ca0-acf0-42d8-d995-65cdbb6a1638"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "|Numero_01|Numero_02|Numero_03|Numero_04|Classe|Variaveis_Caracteristicas|\n",
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "|      316|      706|      617|      803|     1|      [316.0,706.0,617.0]|\n",
            "|      306|      580|      452|       91|     1|      [306.0,580.0,452.0]|\n",
            "|      705|      628|        4|      665|     1|        [705.0,628.0,4.0]|\n",
            "|      303|       48|      746|      956|     2|       [303.0,48.0,746.0]|\n",
            "|       39|      489|       89|      702|     1|        [39.0,489.0,89.0]|\n",
            "|      791|      136|      974|       48|     3|      [791.0,136.0,974.0]|\n",
            "|      874|      931|      386|      963|     1|      [874.0,931.0,386.0]|\n",
            "|      723|      880|       88|      357|     2|       [723.0,880.0,88.0]|\n",
            "|      787|       81|      412|      793|     2|       [787.0,81.0,412.0]|\n",
            "|      221|      477|      568|      330|     2|      [221.0,477.0,568.0]|\n",
            "|      726|      430|      434|      632|     1|      [726.0,430.0,434.0]|\n",
            "|      146|        4|      224|       27|     3|        [146.0,4.0,224.0]|\n",
            "|      982|      140|       47|      878|     2|       [982.0,140.0,47.0]|\n",
            "|      553|      420|      184|      519|     1|      [553.0,420.0,184.0]|\n",
            "|      281|      555|      379|      862|     1|      [281.0,555.0,379.0]|\n",
            "|      396|      812|      678|      700|     3|      [396.0,812.0,678.0]|\n",
            "|      754|      635|        9|      679|     2|        [754.0,635.0,9.0]|\n",
            "|      970|      152|      191|      984|     3|      [970.0,152.0,191.0]|\n",
            "|      151|       98|      925|      262|     3|       [151.0,98.0,925.0]|\n",
            "|      691|      635|      756|       46|     2|      [691.0,635.0,756.0]|\n",
            "+---------+---------+---------+---------+------+-------------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Selecionado a coluna previsor\n",
        "Dados_Modelo = Saida.select('Variaveis_Caracteristicas', 'Numero_04')\n",
        "\n",
        "# verificando\n",
        "Dados_Modelo.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "geVcgkub956s",
        "outputId": "fc5b5b8d-0e15-4663-a3e9-dc473a90113c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------+---------+\n",
            "|Variaveis_Caracteristicas|Numero_04|\n",
            "+-------------------------+---------+\n",
            "|      [316.0,706.0,617.0]|      803|\n",
            "|      [306.0,580.0,452.0]|       91|\n",
            "|        [705.0,628.0,4.0]|      665|\n",
            "|       [303.0,48.0,746.0]|      956|\n",
            "|        [39.0,489.0,89.0]|      702|\n",
            "|      [791.0,136.0,974.0]|       48|\n",
            "|      [874.0,931.0,386.0]|      963|\n",
            "|       [723.0,880.0,88.0]|      357|\n",
            "|       [787.0,81.0,412.0]|      793|\n",
            "|      [221.0,477.0,568.0]|      330|\n",
            "|      [726.0,430.0,434.0]|      632|\n",
            "|        [146.0,4.0,224.0]|       27|\n",
            "|       [982.0,140.0,47.0]|      878|\n",
            "|      [553.0,420.0,184.0]|      519|\n",
            "|      [281.0,555.0,379.0]|      862|\n",
            "|      [396.0,812.0,678.0]|      700|\n",
            "|        [754.0,635.0,9.0]|      679|\n",
            "|      [970.0,152.0,191.0]|      984|\n",
            "|       [151.0,98.0,925.0]|      262|\n",
            "|      [691.0,635.0,756.0]|       46|\n",
            "+-------------------------+---------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Treinamento do Modelo\n",
        "\n",
        "# Função da regressão Linear\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "\n",
        "# Separando os dados de Teste e Treino\n",
        "Dados_Treino, Dados_Teste = Dados_Modelo.randomSplit( [0.75, 0.25] )\n",
        "\n",
        "# Aplicando as features para o modelo linear\n",
        "Funcao_Linear = LinearRegression( featuresCol='Variaveis_Caracteristicas', labelCol='Numero_04')\n",
        "\n",
        "# Fitando o Modelo\n",
        "Funcao_Linear = Funcao_Linear.fit(Dados_Treino)\n",
        "\n",
        "# Veriricando os coeficientes\n",
        "Funcao_Linear.coefficients"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WK-vgO0s-1XL",
        "outputId": "77f72342-3322-4c92-f4d5-362765a2bcd8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DenseVector([0.1345, 0.2464, -0.1219])"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fazendo a previsões dos dados de Teste\n",
        "Previoes = Funcao_Linear.evaluate( Dados_Teste )\n",
        "\n",
        "# Mostrando as previsões\n",
        "Previoes.predictions.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3NsXS5ZI_Z3w",
        "outputId": "e98479d1-bef2-44ef-ae8a-286d9fd7bc9f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------+---------+------------------+\n",
            "|Variaveis_Caracteristicas|Numero_04|        prediction|\n",
            "+-------------------------+---------+------------------+\n",
            "|        [39.0,489.0,89.0]|      702| 505.6934710791836|\n",
            "|       [49.0,953.0,418.0]|      876|  581.269095129581|\n",
            "|      [126.0,154.0,137.0]|      889|428.97986313855927|\n",
            "|      [171.0,327.0,242.0]|      754|464.86228797793945|\n",
            "|      [177.0,945.0,401.0]|       77| 598.5838567503675|\n",
            "|      [262.0,763.0,680.0]|      289| 531.1379102159367|\n",
            "|      [316.0,706.0,617.0]|      803| 532.0349298953695|\n",
            "|      [329.0,796.0,733.0]|      600| 541.8175203144399|\n",
            "|       [413.0,47.0,633.0]|      962| 380.7195629598334|\n",
            "|      [619.0,207.0,445.0]|      926| 470.7797055052423|\n",
            "|       [723.0,880.0,88.0]|      357| 694.1594591995411|\n",
            "|      [734.0,721.0,619.0]|      201| 591.6997195553748|\n",
            "|      [862.0,506.0,500.0]|      512| 570.4385433103265|\n",
            "|      [968.0,143.0,458.0]|      666|500.35474763967875|\n",
            "+-------------------------+---------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Regressão Logistica"
      ],
      "metadata": {
        "id": "h00-emk7VQnu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# -- Construção Modelo Logistico\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Separar os dados de caracteristcas e previsor\n",
        "Selecao_Dados = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03'], outputCol='Variaveis_Caracteristicas' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Saida = Selecao_Dados.transform( Base_Dados_Spark )\n",
        "\n",
        "# Selecionado a coluna previsor\n",
        "Dados_Modelo = Saida.select('Variaveis_Caracteristicas', 'Classe')\n",
        "\n",
        "# Veriricando\n",
        "Dados_Modelo.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yQQhlE9lUUTh",
        "outputId": "1aa31fb5-769f-4011-e7bb-b68e49bf53ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Variaveis_Caracteristicas=DenseVector([316.0, 706.0, 617.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([306.0, 580.0, 452.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([705.0, 628.0, 4.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([303.0, 48.0, 746.0]), Classe=2),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([39.0, 489.0, 89.0]), Classe=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Função Logistica\n",
        "from pyspark.ml.classification import LogisticRegression\n",
        "\n",
        "# Separando os dados de Teste e Treino\n",
        "Dados_Treino, Dados_Teste = Dados_Modelo.randomSplit( [0.75, 0.25] )\n",
        "\n",
        "# Aplicando as features para o modelo linea\n",
        "Funcao_Logistica = LogisticRegression(featuresCol = 'Variaveis_Caracteristicas', labelCol = 'Classe', maxIter=1000)\n",
        "\n",
        "# Fitando o Modelo\n",
        "Funcao_Logistica = Funcao_Logistica.fit(Dados_Treino)\n",
        "\n",
        "# Fazendo a previsões dos dados de Teste\n",
        "Previoes = Funcao_Logistica.evaluate( Dados_Teste )\n",
        "\n",
        "# Mostrando as previsões\n",
        "Previoes.predictions.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "64WIO0MzUUn-",
        "outputId": "8bd156c9-cc32-4d06-d27e-c71ee2d1c66c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|Variaveis_Caracteristicas|Classe|       rawPrediction|         probability|prediction|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|       [48.0,838.0,518.0]|     1|[-16.907459680451...|[4.95840595284761...|       2.0|\n",
            "|       [49.0,953.0,418.0]|     3|[-16.824641484571...|[5.36377553415517...|       2.0|\n",
            "|      [171.0,327.0,242.0]|     1|[-17.508217077354...|[2.41376346539577...|       3.0|\n",
            "|      [262.0,763.0,680.0]|     2|[-17.004938076398...|[4.47928410764775...|       2.0|\n",
            "|      [280.0,694.0,248.0]|     2|[-17.191531797229...|[3.55014073947933...|       2.0|\n",
            "|      [287.0,679.0,919.0]|     2|[-17.029572889327...|[4.40589136530800...|       2.0|\n",
            "|      [299.0,154.0,344.0]|     2|[-17.687451599033...|[1.90060591658064...|       3.0|\n",
            "|       [303.0,48.0,746.0]|     2|[-17.682940920292...|[1.89950053022615...|       3.0|\n",
            "|      [325.0,784.0,233.0]|     2|[-17.124144355503...|[3.82376451743130...|       2.0|\n",
            "|      [355.0,817.0,894.0]|     1|[-16.926455150734...|[4.93245981800289...|       2.0|\n",
            "|      [363.0,733.0,560.0]|     1|[-17.098064549541...|[4.00120290126989...|       2.0|\n",
            "|      [392.0,951.0,950.0]|     3|[-16.795606951791...|[5.68584493256866...|       2.0|\n",
            "|      [407.0,509.0,679.0]|     2|[-17.294359683659...|[3.18291697518008...|       2.0|\n",
            "|       [413.0,47.0,633.0]|     2|[-17.749431659974...|[1.73574875736086...|       3.0|\n",
            "|      [558.0,810.0,418.0]|     2|[-17.125243170880...|[3.83609978878100...|       2.0|\n",
            "|      [578.0,459.0,481.0]|     3|[-17.449898237052...|[2.60692683122368...|       2.0|\n",
            "|      [667.0,967.0,607.0]|     2|[-16.960275848684...|[4.62823953423381...|       2.0|\n",
            "|      [691.0,635.0,756.0]|     2|[-17.245206820038...|[3.37731658987232...|       2.0|\n",
            "|      [723.0,549.0,539.0]|     3|[-17.395334110280...|[2.79302648149763...|       2.0|\n",
            "|       [723.0,880.0,88.0]|     2|[-17.199448815722...|[3.45438339493292...|       2.0|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Floresta Decisão"
      ],
      "metadata": {
        "id": "ayV-B0ciaRtl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# -- Construção Modelo Logistico\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Separar os dados de caracteristcas e previsor\n",
        "Selecao_Dados = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03'], outputCol='Variaveis_Caracteristicas' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Saida = Selecao_Dados.transform( Base_Dados_Spark )\n",
        "\n",
        "# Selecionado a coluna previsor\n",
        "Dados_Modelo = Saida.select('Variaveis_Caracteristicas', 'Classe')\n",
        "\n",
        "# Veriricando\n",
        "Dados_Modelo.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HAg7ICaFVkpH",
        "outputId": "5543c0f3-f96b-4136-b2cb-1646ccf0db16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Variaveis_Caracteristicas=DenseVector([316.0, 706.0, 617.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([306.0, 580.0, 452.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([705.0, 628.0, 4.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([303.0, 48.0, 746.0]), Classe=2),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([39.0, 489.0, 89.0]), Classe=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Função Random Forest\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "\n",
        "# Separando os dados de Teste e Treino\n",
        "Dados_Treino, Dados_Teste = Dados_Modelo.randomSplit( [0.75, 0.25] )\n",
        "\n",
        "# Aplicando as features para o modelo linea\n",
        "Floresta_Decisao = RandomForestClassifier(numTrees=100, maxDepth=2,\n",
        "                                          featuresCol = 'Variaveis_Caracteristicas',\n",
        "                                          labelCol = 'Classe')\n",
        "\n",
        "# Fitando o Modelo\n",
        "Floresta_Decisao = Floresta_Decisao.fit(Dados_Treino)\n",
        "\n",
        "# Fazendo a previsões dos dados de Teste\n",
        "Previoes = Floresta_Decisao.evaluate( Dados_Teste )\n",
        "\n",
        "# Mostrando as previsões\n",
        "Previoes.predictions.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Poxr4zgrVo41",
        "outputId": "2f8f65a1-d3df-4992-ebef-0479e9e7e0a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|Variaveis_Caracteristicas|Classe|       rawPrediction|         probability|prediction|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|       [19.0,667.0,480.0]|     1|[0.0,18.780937689...|[0.0,0.1878093768...|       3.0|\n",
            "|       [48.0,838.0,518.0]|     1|[0.0,12.606309650...|[0.0,0.1260630965...|       3.0|\n",
            "|       [79.0,983.0,746.0]|     2|[0.0,17.572142649...|[0.0,0.1757214264...|       3.0|\n",
            "|      [287.0,679.0,919.0]|     2|[0.0,47.072560590...|[0.0,0.4707256059...|       1.0|\n",
            "|       [303.0,48.0,746.0]|     2|[0.0,45.244959173...|[0.0,0.4524495917...|       1.0|\n",
            "|      [305.0,255.0,447.0]|     2|[0.0,27.808429281...|[0.0,0.2780842928...|       2.0|\n",
            "|      [329.0,796.0,733.0]|     2|[0.0,43.052830279...|[0.0,0.4305283027...|       1.0|\n",
            "|      [395.0,182.0,827.0]|     2|[0.0,60.200983714...|[0.0,0.6020098371...|       1.0|\n",
            "|      [402.0,890.0,207.0]|     2|[0.0,18.625380283...|[0.0,0.1862538028...|       2.0|\n",
            "|      [473.0,370.0,149.0]|     2|[0.0,27.649587153...|[0.0,0.2764958715...|       2.0|\n",
            "|      [485.0,182.0,919.0]|     2|[0.0,47.323135848...|[0.0,0.4732313584...|       1.0|\n",
            "|      [553.0,420.0,184.0]|     1|[0.0,27.619986331...|[0.0,0.2761998633...|       2.0|\n",
            "|      [578.0,459.0,481.0]|     3|[0.0,26.484213424...|[0.0,0.2648421342...|       2.0|\n",
            "|       [580.0,516.0,51.0]|     1|[0.0,25.790745015...|[0.0,0.2579074501...|       2.0|\n",
            "|      [628.0,405.0,110.0]|     2|[0.0,28.064697869...|[0.0,0.2806469786...|       2.0|\n",
            "|      [681.0,815.0,955.0]|     3|[0.0,46.651022687...|[0.0,0.4665102268...|       1.0|\n",
            "|      [691.0,635.0,756.0]|     2|[0.0,43.061239730...|[0.0,0.4306123973...|       1.0|\n",
            "|        [705.0,628.0,4.0]|     1|[0.0,22.224655135...|[0.0,0.2222465513...|       2.0|\n",
            "|      [719.0,749.0,370.0]|     2|[0.0,22.523702978...|[0.0,0.2252370297...|       2.0|\n",
            "|      [737.0,351.0,660.0]|     1|[0.0,27.432170227...|[0.0,0.2743217022...|       2.0|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Naive Bayes"
      ],
      "metadata": {
        "id": "Pqrn-tbva_sg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# -- Construção Modelo Logistico\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Separar os dados de caracteristcas e previsor\n",
        "Selecao_Dados = VectorAssembler(\n",
        "    inputCols=['Numero_01', 'Numero_02', 'Numero_03'], outputCol='Variaveis_Caracteristicas' )\n",
        "\n",
        "# Aplicando a transformação\n",
        "Saida = Selecao_Dados.transform( Base_Dados_Spark )\n",
        "\n",
        "# Selecionado a coluna previsor\n",
        "Dados_Modelo = Saida.select('Variaveis_Caracteristicas', 'Classe')\n",
        "\n",
        "# Veriricando\n",
        "Dados_Modelo.head(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Dt6sBs5aNnn",
        "outputId": "d78f02a7-8ab8-4d50-bf1a-ed7d6de7e9aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Variaveis_Caracteristicas=DenseVector([316.0, 706.0, 617.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([306.0, 580.0, 452.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([705.0, 628.0, 4.0]), Classe=1),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([303.0, 48.0, 746.0]), Classe=2),\n",
              " Row(Variaveis_Caracteristicas=DenseVector([39.0, 489.0, 89.0]), Classe=1)]"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Função Random Forest\n",
        "from pyspark.ml.classification import NaiveBayes\n",
        "\n",
        "# Separando os dados de Teste e Treino\n",
        "Dados_Treino, Dados_Teste = Dados_Modelo.randomSplit( [0.75, 0.25] )\n",
        "\n",
        "# Aplicando as features para o modelo linea\n",
        "Funcao_NaiveBayes = NaiveBayes(smoothing=1.0, modelType='multinomial',\n",
        "                                          featuresCol = 'Variaveis_Caracteristicas',\n",
        "                                          labelCol = 'Classe')\n",
        "\n",
        "# Fitando o Modelo\n",
        "Funcao_NaiveBayes = Funcao_NaiveBayes.fit(Dados_Treino)\n",
        "\n",
        "# Fazendo a previsões dos dados de Teste\n",
        "Previoes = Funcao_NaiveBayes.transform( Dados_Teste )\n",
        "\n",
        "# Mostrando as previsões\n",
        "Previoes.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-RgOTzz4bBek",
        "outputId": "692c3a8a-5e79-436b-b7b5-7d1813283df5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|Variaveis_Caracteristicas|Classe|       rawPrediction|         probability|prediction|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "|       [13.0,935.0,631.0]|     3|[-1750.5841243277...|[8.89630790655601...|       1.0|\n",
            "|       [19.0,667.0,480.0]|     1|[-1292.8352707934...|[8.77046157546633...|       1.0|\n",
            "|        [39.0,489.0,89.0]|     1|[-682.97208892578...|[3.12444944059513...|       1.0|\n",
            "|       [88.0,753.0,446.0]|     3|[-1424.5895225806...|[1.31308277748837...|       1.0|\n",
            "|      [125.0,391.0,196.0]|     3|[-786.28920135591...|[9.64617431452898...|       1.0|\n",
            "|        [146.0,4.0,224.0]|     3|[-412.06704398172...|[2.20543726329240...|       2.0|\n",
            "|       [187.0,435.0,24.0]|     3|[-710.66117265844...|[1.31937063528319...|       1.0|\n",
            "|      [299.0,154.0,344.0]|     2|[-876.38108574521...|[9.79314175512917...|       2.0|\n",
            "|      [305.0,255.0,447.0]|     2|[-1109.0335988374...|[8.07848567107883...|       2.0|\n",
            "|      [336.0,707.0,643.0]|     2|[-1860.2039198272...|[3.38788850181533...|       1.0|\n",
            "|      [377.0,567.0,411.0]|     2|[-1491.7424484167...|[1.90158789585352...|       1.0|\n",
            "|      [395.0,182.0,827.0]|     2|[-1547.7432547716...|[1.46620880808286...|       2.0|\n",
            "|      [619.0,207.0,445.0]|     3|[-1392.4524733568...|[9.81673898901764...|       2.0|\n",
            "|      [636.0,824.0,626.0]|     3|[-2294.3276513193...|[3.71453205292662...|       1.0|\n",
            "|      [667.0,967.0,607.0]|     2|[-2464.7997237973...|[5.54637317300686...|       1.0|\n",
            "|      [693.0,995.0,280.0]|     2|[-2160.3651900849...|[1.22375338155470...|       1.0|\n",
            "|       [723.0,880.0,88.0]|     2|[-1852.1443576105...|[1.01629297111159...|       1.0|\n",
            "|      [756.0,507.0,618.0]|     3|[-2064.3042843831...|[6.51099866136853...|       2.0|\n",
            "|      [780.0,751.0,180.0]|     3|[-1873.2218824617...|[5.99479613328689...|       1.0|\n",
            "|      [862.0,506.0,500.0]|     3|[-2046.3930298398...|[0.00574385534861...|       2.0|\n",
            "+-------------------------+------+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Construindo modelo de Cluster\n",
        "\n",
        "# Função do Kmeans\n",
        "from pyspark.ml.clustering import KMeans\n",
        "\n",
        "# Parametro do modelo\n",
        "Funcao_KMeans = KMeans(k=2)\n",
        "\n",
        "# Ajustando Nome das Colunas de Treino\n",
        "Dados_Treino_Cluster = Dados_Treino.withColumnRenamed( 'Variaveis_Caracteristicas', 'features')\n",
        "Dados_Treino_Cluster = Dados_Treino_Cluster.withColumnRenamed( 'Classe', 'weighCol')\n",
        "\n",
        "# Fitar o Modelo\n",
        "Modelo = Funcao_KMeans.fit( Dados_Treino_Cluster )"
      ],
      "metadata": {
        "id": "_Kp0GXc-EYb9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Ajustando Nome das Colunas de Treino\n",
        "Dados_Teste_Cluster = Dados_Teste.withColumnRenamed( 'Variaveis_Caracteristicas', 'features')\n",
        "Dados_Teste_Cluster = Dados_Teste_Cluster.withColumnRenamed( 'Classe', 'weighCol')\n",
        "\n",
        "# Fazendo previsão do primeiro registro\n",
        "print( 'Previsão do Modelo:', Modelo.predict( Dados_Teste_Cluster.head().features ) )\n",
        "\n",
        "print( 'Valore Real:', Dados_Teste_Cluster.select('weighCol').head() )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fnRANh0TEc2p",
        "outputId": "fdc674f2-901d-4cc9-959d-4b762f4cb66b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previsão do Modelo: 0\n",
            "Valore Real: Row(weighCol=3)\n"
          ]
        }
      ]
    }
  ]
}