HIVE connect (3).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "74c478b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import libraries \n",
    "\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.preprocessing import QuantileTransformer\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from numpy import int64\n",
    "from sklearn import svm\n",
    "from sklearn.svm import SVC \n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.metrics import classification_report, accuracy_score  \n",
    "from sklearn.metrics import precision_score, recall_score \n",
    "from sklearn.metrics import f1_score, matthews_corrcoef \n",
    "from sklearn.metrics import confusion_matrix \n",
    "\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "from sklearn.utils import resample\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt # plotting\n",
    "import numpy as np # linear algebra\n",
    "import os # accessing directory structure\n",
    "import pandas as pd # data processing\n",
    "from pandas.plotting import scatter_matrix\n",
    "#import library psycopyg2\n",
    "import psycopg2\n",
    "#import library pandas\n",
    "import pandas as pd\n",
    "#import library sqlio\n",
    "import pandas.io.sql as sqlio\n",
    "import sasl\n",
    "import thrift_sasl\n",
    "from pyhive import hive\n",
    "#from impala.dbapi import connect\n",
    "from hdfs import InsecureClient\n",
    "from pyhive import hive\n",
    "from sqlalchemy import Column, String, Integer\n",
    "import serial\n",
    "import time\n",
    "from flask import Flask, flash, redirect, render_template, request, session, url_for\n",
    "from flask_sqlalchemy import SQLAlchemy\n",
    "from flask_session import Session\n",
    "from passlib.apps import custom_app_context as pwd_context\n",
    "from tempfile import gettempdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bf08ca08",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not connect to any of [('172.27.27.60', 10000)]\n"
     ]
    },
    {
     "ename": "TTransportException",
     "evalue": "Could not connect to any of [('172.27.27.60', 10000)]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTTransportException\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-7441a2bd595e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# prints the result set's schema\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m#results = cursor.fetchall() # default.client_failure1_data_hive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConnection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhost\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'172.27.27.60'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpassword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Hive#Pa55'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0musername\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'smicro'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'CUSTOM'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;31m#dataframe = pd.read_sql(\"SELECT * FROM failure1_data_hive\", conn)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mdataframe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_sql\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"SELECT * FROM demo_client_data_hive\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pyhive/hive.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, host, port, scheme, username, database, auth, configuration, kerberos_service_name, password, check_hostname, ssl_cert, thrift_transport)\u001b[0m\n\u001b[1;32m    241\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 243\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    244\u001b[0m             open_session_req = ttypes.TOpenSessionReq(\n\u001b[1;32m    245\u001b[0m                 \u001b[0mclient_protocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol_version\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/thrift_sasl/__init__.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     72\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misOpen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trans\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msasl\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/thrift/transport/TSocket.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    144\u001b[0m                                                           addrs))\n\u001b[1;32m    145\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mTTransportException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTTransportException\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNOT_OPEN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msz\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTTransportException\u001b[0m: Could not connect to any of [('172.27.27.60', 10000)]"
     ]
    }
   ],
   "source": [
    "#cursor = conn.cursor()\n",
    "#cursor.execute('SELECT * FROM test_parquet')\n",
    "#print(cursor.execute) \n",
    "# prints the result set's schema\n",
    "#results = cursor.fetchall() # default.client_failure1_data_hive\n",
    "conn = hive.Connection(host='172.27.27.60', port=10000, password='Hive#Pa55', username='smicro', auth='CUSTOM')\n",
    "#dataframe = pd.read_sql(\"SELECT * FROM failure1_data_hive\", conn)\n",
    "dataframe = pd.read_sql(\"SELECT * FROM demo_client_data_hive\", conn)\n",
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9d46efaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['demo_client_data_hive.date', 'demo_client_data_hive.serial_number',\n",
       "       'demo_client_data_hive.model', 'demo_client_data_hive.capacity_bytes',\n",
       "       'demo_client_data_hive.failure',\n",
       "       'demo_client_data_hive.smart_1_normalized',\n",
       "       'demo_client_data_hive.smart_1_raw',\n",
       "       'demo_client_data_hive.smart_2_normalized',\n",
       "       'demo_client_data_hive.smart_2_raw',\n",
       "       'demo_client_data_hive.smart_3_normalized',\n",
       "       ...\n",
       "       'demo_client_data_hive.smart_250_normalized',\n",
       "       'demo_client_data_hive.smart_250_raw',\n",
       "       'demo_client_data_hive.smart_251_normalized',\n",
       "       'demo_client_data_hive.smart_251_raw',\n",
       "       'demo_client_data_hive.smart_252_normalized',\n",
       "       'demo_client_data_hive.smart_252_raw',\n",
       "       'demo_client_data_hive.smart_254_normalized',\n",
       "       'demo_client_data_hive.smart_254_raw',\n",
       "       'demo_client_data_hive.smart_255_normalized',\n",
       "       'demo_client_data_hive.smart_255_raw'],\n",
       "      dtype='object', length=149)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8495149b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/smicro/.local/lib/python3.6/site-packages/pyspark/context.py:238: FutureWarning: Python 3.6 support is deprecated in Spark 3.2.\n",
      "  FutureWarning\n",
      "/home/smicro/.local/lib/python3.6/site-packages/pyspark/sql/context.py:604: FutureWarning: HiveContext is deprecated in Spark 2.0.0. Please use SparkSession.builder.enableHiveSupport().getOrCreate() instead.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "ename": "AnalysisException",
     "evalue": "org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAnalysisException\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-30cb36113c37>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msql\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mHiveContext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mhive_context\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mHiveContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhive_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"demo_client_data_hive\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pyspark/sql/context.py\u001b[0m in \u001b[0;36mtable\u001b[0;34m(self, tableName)\u001b[0m\n\u001b[1;32m    453\u001b[0m         \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m         \"\"\"\n\u001b[0;32m--> 455\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparkSession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtableName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdbName\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pyspark/sql/session.py\u001b[0m in \u001b[0;36mtable\u001b[0;34m(self, tableName)\u001b[0m\n\u001b[1;32m    739\u001b[0m         \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    740\u001b[0m         \"\"\"\n\u001b[0;32m--> 741\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jsparkSession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtableName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrapped\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    742\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    743\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1308\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1310\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1312\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m    115\u001b[0m                 \u001b[0;31m# Hide where the exception came from that shows a non-Pythonic\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m                 \u001b[0;31m# JVM exception message.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mconverted\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m                 \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAnalysisException\u001b[0m: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient"
     ]
    }
   ],
   "source": [
    "################################## 2nd methord usig spark\n",
    "from pyspark import SparkContext\n",
    "sc =SparkContext()\n",
    "from pyspark.sql import HiveContext\n",
    "hive_context = HiveContext(sc)\n",
    "data = hive_context.table(\"demo_client_data_hive\")\n",
    "data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9fe7303b",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'colums'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-6f0deaa88227>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolums\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pyspark/sql/dataframe.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1658\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1659\u001b[0m             raise AttributeError(\n\u001b[0;32m-> 1660\u001b[0;31m                 \"'%s' object has no attribute '%s'\" % (self.__class__.__name__, name))\n\u001b[0m\u001b[1;32m   1661\u001b[0m         \u001b[0mjc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1662\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mColumn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'colums'"
     ]
    }
   ],
   "source": [
    "data.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "b03ec6d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('id', 'int'), ('name', 'string'), ('age', 'int')]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d6a9a9e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.master('yarn').appName('pythonSpark').enableHiveSupport().getOrCreate()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5cac9fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = spark.read.load('/data/employees1.csv', format='com.databricks.spark.csv', header='true', inferSchema='true')\n",
    "df1.write.format(\"PARQUET\").saveAsTable(\"default.test_parquet\")\n",
    "df2 = spark.read.load('/data/employees2.csv', format='com.databricks.spark.csv', header='true', inferSchema='true')\n",
    "df2.write.mode(\"append\").saveAsTable(\"default.test_parquet\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}