08_Data_engineering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNBNfzTRrPY2GmOjAdctgDf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/data-engineering/blob/main/08_Data_engineering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Instala biblioteca"
      ],
      "metadata": {
        "id": "4svy65wMZypd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SYtHq6FUSzYS",
        "outputId": "63468cc9-0b87-40b2-a542-0a0ca55eabb4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.10/dist-packages (3.5.4)\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n"
          ]
        }
      ],
      "source": [
        "pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%markdown\n",
        "# Passos:\n",
        "- arquitetura dados\n",
        "- ingestao dados\n",
        "- carregar na bronze\n",
        "- promover para silver\n",
        "- consumo na gold\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 169
        },
        "id": "EV30PsqbVv-A",
        "outputId": "2dfd0cf2-21ee-4c64-b0ce-7915d7114fa3"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ],
            "text/markdown": "# Passos: \n- arquitetura dados \n- ingestao dados \n- carregar na bronze \n- promover para silver\n- consumo na gold\n\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "# Inicializar Spark\n",
        "spark = SparkSession.builder.appName(\"UserIngestion\").getOrCreate()\n",
        "\n",
        "# Função fictícia para ingestão de dados\n",
        "def ingest_user_reg():\n",
        "    global data # Declare data as global to access it outside the function\n",
        "    data = [\n",
        "        (1, \"Luciana\", \"2023-01-01\"),\n",
        "        (2, \"Angelica\", \"2023-01-02\"),\n",
        "        (3, \"Cathy\", \"2023-01-03\")\n",
        "    ]\n",
        "    columns = [\"user_id\", \"name\", \"registration_date\"]\n",
        "\n",
        "    # Criar DataFrame e registrar como tabela\n",
        "    df = spark.createDataFrame(data, columns)\n",
        "    df.createOrReplaceTempView(\"registered_users\")\n",
        "\n"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "BFYWtbT9XKn4"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Chamar a função de ingestão\n",
        "ingest_user_reg()"
      ],
      "metadata": {
        "id": "Ci26BGVWXNSf"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Exibir os dados da tabela 'registered_users'\n",
        "display(spark.table(\"registered_users\"))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "bUwD9sKuXOwx",
        "outputId": "b7c89bbd-924e-4137-9abe-08bf4e594d90"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "DataFrame[user_id: bigint, name: string, registration_date: string]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Now you can access 'data' here\n",
        "display(data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "FMGabzJiXRuL",
        "outputId": "4375e4bf-a9d0-41d3-a4ae-ce2643170fb1"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[(1, 'Luciana', '2023-01-01'),\n",
              " (2, 'Angelica', '2023-01-02'),\n",
              " (3, 'Cathy', '2023-01-03')]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(data[0][1]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dcg63dmWaGE1",
        "outputId": "9444da28-4a0c-4cb6-cd84-67d2d7d45ea1"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'str'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "syQO_gNEaT73"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}